diff --git a/.clang-format b/.clang-format index 1cc151e2adcc..6a3de86ab27a 100644 --- a/.clang-format +++ b/.clang-format @@ -481,6 +481,7 @@ ForEachMacros: - 'genradix_for_each' - 'genradix_for_each_from' - 'genradix_for_each_reverse' + - 'guard' - 'hash_for_each' - 'hash_for_each_possible' - 'hash_for_each_possible_rcu' @@ -674,6 +675,7 @@ ForEachMacros: - 'rq_list_for_each' - 'rq_list_for_each_safe' - 'sample_read_group__for_each' + - 'scoped_guard' - 'scsi_for_each_prot_sg' - 'scsi_for_each_sg' - 'sctp_for_each_hentry' diff --git a/.gitignore b/.gitignore index 38365b870822..9875120ea7bd 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ *.s *.so *.so.dbg +*.spdx.json *.su *.symtypes *.tab.[ch] diff --git a/.mailmap b/.mailmap index 7d653f7e693d..6832a9e8b15a 100644 --- a/.mailmap +++ b/.mailmap @@ -36,13 +36,14 @@ Alexander Lobakin Alexander Mikhalitsyn Alexander Mikhalitsyn Alexander Mikhalitsyn -Alexander Sverdlin -Alexander Sverdlin -Alexander Sverdlin -Alexander Sverdlin -Alexander Sverdlin -Alexander Sverdlin -Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin +Alexander Sverdlin Alexandre Belloni Alexandre Ghiti Alexei Avshalom Lazar @@ -117,6 +118,7 @@ Asutosh Das Atish Patra Atish Patra Avaneesh Kumar Dwivedi +Avri Altman Axel Dyks Axel Lin Balakrishna Godavarthi @@ -371,6 +373,7 @@ Jarkko Sakkinen Jason Gunthorpe Jason Gunthorpe Jason Gunthorpe +Jason Wang Jason Xing Javi Merino @@ -396,6 +399,7 @@ Jens Axboe Jens Axboe Jens Axboe Jens Osterkamp +Jens Wiklander Jernej Skrabec Jesper Dangaard Brouer Jesper Dangaard Brouer @@ -437,6 +441,7 @@ John Stultz Jonas Gorski Jonathan Cameron Jordan Crouse +Jorge Ramirez-Ortiz @@ -527,7 +532,8 @@ Luca Ceresoli Luca Weiss Lucas De Marchi Lukasz Luba -Luo Jie +Luo Jie +Luo Jie Lance Yang Lance Yang Maciej W. Rozycki @@ -554,6 +560,7 @@ Martyna Szapar-Mudlaw Mat Martineau Mat Martineau +Matt Coster Matthew Wilcox Matthew Wilcox Matthew Wilcox @@ -704,6 +711,10 @@ Qi Zheng Quentin Monnet Quentin Monnet Quentin Perret +Radu Rendec +Radu Rendec +Radu Rendec +Radu Rendec Rae Moar Rafael J. Wysocki Rajeev Nandan @@ -808,6 +819,7 @@ Simon Wunderlich Simon Wunderlich Simon Wunderlich Simon Wunderlich +SJ Park Sricharan Ramabadhran Srinivas Kandagatla Srinivas Kandagatla diff --git a/CREDITS b/CREDITS index e127c3074441..db8f748d7fba 100644 --- a/CREDITS +++ b/CREDITS @@ -1503,6 +1503,10 @@ N: Andy Gross E: agross@kernel.org D: Qualcomm SoC subsystem and drivers +N: Mark Gross +E: markgross@kernel.org +D: x86/mellanox platform maintenance and various x86 specific drivers + N: Grant Grundler E: grantgrundler@gmail.com W: http://obmouse.sourceforge.net/ @@ -3096,6 +3100,10 @@ N: Jens Osterkamp E: jens@de.ibm.com D: Maintainer of Spidernet network driver for Cell +N: Kent Overstreet +E: kent.overstreet@linux.dev +D: Co-authored and contributed to Memory Allocation Profiling + N: Gadi Oxman E: gadio@netvision.net.il D: Original author and maintainer of IDE/ATAPI floppy/tape drivers @@ -3373,6 +3381,10 @@ N: Anil Ravindranath E: anil_ravindranath@pmc-sierra.com D: PMC-Sierra MaxRAID driver +N: Dwaipayan Ray +E: dwaipayanray1@gmail.com +D: checkpatch improvements + N: Eric S. Raymond E: esr@thyrsus.com W: http://www.tuxedo.org/~esr/ diff --git a/Documentation/ABI/testing/configfs-thunderbolt_stream b/Documentation/ABI/testing/configfs-thunderbolt_stream new file mode 100644 index 000000000000..7abc6b73a1e4 --- /dev/null +++ b/Documentation/ABI/testing/configfs-thunderbolt_stream @@ -0,0 +1,83 @@ +What: /sys/kernel/config/thunderbolt/stream/. +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + Configuration group for a stream Thunderbolt/USB4 + service. It is possible to create groups even if there + is no connection yet to the other host. Once a + connection established and there is stream service on + the remote side that matches, this configuration is + applied to it. + + To find the service name you can run tblist from tbtools [1]: + + # tblist -A + ... + Domain 0 Route 3 Index 0: stream + + [1] https://github.com/intel/tbtools + +What: /sys/kernel/config/thunderbolt/stream/./$name +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + Creates new stream with $name and fills it with the + default values. If there is an advertised remote stream + with the same name, uses its values as the default. + +What: /sys/kernel/config/thunderbolt/stream/./$name/index +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + This matches the X in /dev/tbstreamX and allows userspace + to map the configfs directory to the corresponding character + device. + +What: /sys/kernel/config/thunderbolt/stream/./$name/in_hopid +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + In HopID used with the read path of the tunnel. Available HopIDs + for tunneling start from 8. You can pass also -1 for automatic + allocation. The allocated value can be read here. Writing 0 will + de-allocate if the stream is not in use. + + To figure out the maximum HopID you can run tbget from + tbtools for the lane adapter. For example below we check + for lane adapter number 1 (first USB4 port): + + # tbget -r 0 -a 1 -D ADP_CS_5.Max\ Input\ HopID + 19 + + This allows to use anything between 8 and 19 inclusive. + +What: /sys/kernel/config/thunderbolt/stream/./$name/out_hopid +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + Out HopID used with the write path of the tunnel. Available HopIDs + for tunneling start from 8. You can pass also -1 for automatic + allocation. The allocated value can be read here. Writing 0 will + de-allocate if the stream is not in use. See @in_hopid + for how to figure out the maximum HopID. + +What: /sys/kernel/config/thunderbolt/stream/./$name/ring_size +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + Size of the TX/RX rings. Can be adjusted between 32 and + 4096. The default is 256. + +What: /sys/kernel/config/thunderbolt/stream/./$name/throttling +Date: Sep 2026 +KernelVersion: v7.2 +Contact: Mika Westerberg +Description: + Interrupt throttling rate in ns. Lower values can give + better latency. The default is 8192 ns. diff --git a/Documentation/ABI/testing/debugfs-vfio b/Documentation/ABI/testing/debugfs-vfio index 70ec2d454686..ed2f29c3a9b4 100644 --- a/Documentation/ABI/testing/debugfs-vfio +++ b/Documentation/ABI/testing/debugfs-vfio @@ -29,3 +29,29 @@ Date: Oct 2025 KernelVersion: 6.18 Contact: Cédric Le Goater Description: Read the migration features of the vfio device. + +What: /sys/kernel/debug/vfio//pci +Date: June 2026 +KernelVersion: 7.2 +Contact: Alex Williamson +Description: This debugfs file directory is used for debugging + VFIO PCI devices. + +What: /sys/kernel/debug/vfio//pci/nointxmask +Date: June 2026 +KernelVersion: 7.2 +Contact: Alex Williamson +Description: Read the nointxmask policy latched for this device. This + policy governs whether the device may use PCI 2.3 style + INTx masking when supported, reporting a value of "N", or + requires APIC level INTx masking, reporting a value of "Y". + +What: /sys/kernel/debug/vfio//pci/disable_idle_d3 +Date: June 2026 +KernelVersion: 7.2 +Contact: Alex Williamson +Description: Read the disable_idle_d3 policy latched for this device. This + policy governs whether the device PM runtime usage count is + kept elevated while the device is bound to the driver and + unused, reporting a value of "Y", or decremented to allow the + device to enter a low power state, reporting a value of "N". diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio index 4fc9f6bd4281..4ea5598e7cd2 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio +++ b/Documentation/ABI/testing/sysfs-bus-iio @@ -764,10 +764,13 @@ Contact: linux-iio@vger.kernel.org Description: Specifies the output powerdown mode. DAC output stage is disconnected from the amplifier and + 500ohm_to_gnd: connected to ground via a 500Ohm resistor, 1kohm_to_gnd: connected to ground via an 1kOhm resistor, 2.5kohm_to_gnd: connected to ground via a 2.5kOhm resistor, + 3.85kohm_to_gnd: connected to ground via a 3.85kOhm resistor, 6kohm_to_gnd: connected to ground via a 6kOhm resistor, 7.7kohm_to_gnd: connected to ground via a 7.7kOhm resistor, + 16kohm_to_gnd: connected to ground via a 16kOhm resistor, 20kohm_to_gnd: connected to ground via a 20kOhm resistor, 32kohm_to_gnd: connected to ground via a 32kOhm resistor, 42kohm_to_gnd: connected to ground via a 42kOhm resistor, @@ -914,7 +917,7 @@ Description: Note the driver will assume the last p events requested are to be enabled where p is how many it supports (which may vary - depending on the exact set requested. So if you want to be + depending on the exact set requested). So if you want to be sure you have set what you think you have, check the contents of these attributes after everything is configured. Drivers may have to buffer any parameters so that they are consistent when @@ -970,7 +973,7 @@ Description: Note the driver will assume the last p events requested are to be enabled where p is however many it supports (which may - vary depending on the exact set requested. So if you want to be + vary depending on the exact set requested). So if you want to be sure you have set what you think you have, check the contents of these attributes after everything is configured. Drivers may have to buffer any parameters so that they are consistent when @@ -1510,21 +1513,24 @@ Contact: linux-iio@vger.kernel.org Description: Description of the scan element data storage within the buffer and hence the form in which it is read from user-space. - Form is [be|le]:[s|u]bits/storagebits[>>shift]. - be or le specifies big or little endian. s or u specifies if - signed (2's complement) or unsigned. bits is the number of bits - of data and storagebits is the space (after padding) that it - occupies in the buffer. shift if specified, is the shift that - needs to be applied prior to masking out unused bits. Some - devices put their data in the middle of the transferred elements - with additional information on both sides. Note that some - devices will have additional information in the unused bits - so to get a clean value, the bits value must be used to mask - the buffer output value appropriately. The storagebits value - also specifies the data alignment. So s48/64>>2 will be a - signed 48 bit integer stored in a 64 bit location aligned to - a 64 bit boundary. To obtain the clean value, shift right 2 - and apply a mask to zero the top 16 bits of the result. + Form is [be|le]:[f|s|u]bits/storagebits[>>shift]. + be or le specifies big or little endian. f means floating-point + (IEEE 754 binary format), s means signed (2's complement), u means + unsigned. bits is the number of bits of data and storagebits is the + space (after padding) that it occupies in the buffer; when using a + floating-point format, bits must be one of the width values defined + in the IEEE 754 standard for binary interchange formats (e.g. 16 + indicates the binary16 format for half-precision numbers). shift, + if specified, is the shift that needs to be applied prior to + masking out unused bits. Some devices put their data in the middle + of the transferred elements with additional information on both + sides. Note that some devices will have additional information in + the unused bits, so to get a clean value the bits value must be + used to mask the buffer output value appropriately. The storagebits + value also specifies the data alignment. So s48/64>>2 will be a + signed 48-bit integer stored in a 64-bit location aligned to a + 64-bit boundary. To obtain the clean value, shift right 2 and apply + a mask to zero the top 16 bits of the result. For other storage combinations this attribute will be extended appropriately. @@ -1752,6 +1758,21 @@ Description: measurement from channel Y. Units after application of scale and offset are milliamps. +What: /sys/bus/iio/devices/iio:deviceX/in_rot_quaternionaxis_raw +KernelVersion: 7.1 +Contact: linux-iio@vger.kernel.org +Description: + Raw value of {x, y, z} components of the quaternion vector. These + components represent the axis about which a rotation occurs, and are + subject to the following constraints: + + - the quaternion vector is normalized, i.e. w^2 + x^2 + y^2 + z^2 = 1 + - the rotation angle is within the [-pi, pi] range, i.e. the w + component (which represents the amount of rotation) is non-negative + + These constraints allow the w value to be calculated from the other + components: w = sqrt(1 - (x^2 + y^2 + z^2)). + What: /sys/.../iio:deviceX/in_energy_en What: /sys/.../iio:deviceX/in_distance_en What: /sys/.../iio:deviceX/in_velocity_sqrt(x^2+y^2+z^2)_en @@ -1959,6 +1980,23 @@ Description: Raw (unscaled no offset etc.) resistance reading. Units after application of scale and offset are ohms. +What: /sys/bus/iio/devices/iio:deviceX/in_coverageY_raw +KernelVersion: 7.2 +Contact: linux-iio@vger.kernel.org +Description: + Raw (unscaled no offset etc.) coverage reading. Used for sensors + that report fractional coverage as a percentage, such as leak + detectors where the value represents what portion of the sensing + element is wetted. Units after application of scale and offset are + percent. + +What: /sys/bus/iio/devices/iio:deviceX/in_coverageY_scale +KernelVersion: 7.2 +Contact: linux-iio@vger.kernel.org +Description: + Scale to be applied to in_coverageY_raw to obtain coverage + in percent. + What: /sys/bus/iio/devices/iio:deviceX/heater_enable KernelVersion: 4.1.0 Contact: linux-iio@vger.kernel.org diff --git a/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 b/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 index 28be1cabf112..d44eb5f8728b 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 +++ b/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 @@ -16,7 +16,7 @@ Description: In bridge applications, such as strain gauges and load cells, the bridge itself consumes the majority of the current in the system. To minimize the current consumption of the system, - the bridge can be disconnected (when it is not being used + the bridge can be disconnected (when it is not being used) using the bridge_switch_en attribute. What: /sys/bus/iio/devices/iio:deviceX/in_voltage2-voltage2_shorted_raw diff --git a/Documentation/ABI/testing/sysfs-bus-iio-frequency b/Documentation/ABI/testing/sysfs-bus-iio-frequency new file mode 100644 index 000000000000..46075aee7804 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-iio-frequency @@ -0,0 +1,11 @@ +What: /sys/bus/iio/devices/iio:deviceX/out_altvoltageY_frequency_resolution +KernelVersion: 3.4.0 +Contact: linux-iio@vger.kernel.org +Description: + Stores channel Y frequency resolution/channel spacing in Hz for PLL + devices. The given value directly influences the operating mode when + fractional-N synthesis is required, as it derives values for + configurable modulus parameters used in the calculation of the output + frequency. It is assumed that the algorithm used to compute the various + dividers is able to generate proper values for multiples of the channel + spacing. diff --git a/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 b/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 index 1254457a726e..76987a119feb 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 +++ b/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 @@ -1,13 +1,3 @@ -What: /sys/bus/iio/devices/iio:deviceX/out_altvoltageY_frequency_resolution -KernelVersion: 3.4.0 -Contact: linux-iio@vger.kernel.org -Description: - Stores channel Y frequency resolution/channel spacing in Hz. - The value given directly influences the MODULUS used by - the fractional-N PLL. It is assumed that the algorithm - that is used to compute the various dividers, is able to - generate proper values for multiples of channel spacing. - What: /sys/bus/iio/devices/iio:deviceX/out_altvoltageY_refin_frequency KernelVersion: 3.4.0 Contact: linux-iio@vger.kernel.org diff --git a/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd index d10e6de3adb2..991765d84201 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd +++ b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd @@ -22,7 +22,7 @@ Description: Reading this attribute gives the state of the DbC. It can be one of the following states: disabled, enabled, - initialized, connected, configured and stalled. + initialized, connected, configured or suspended. What: /sys/bus/pci/drivers/xhci_hcd/.../dbc_idVendor Date: March 2023 diff --git a/Documentation/ABI/testing/sysfs-class-power-bd71828 b/Documentation/ABI/testing/sysfs-class-power-bd71828 new file mode 100644 index 000000000000..2d451e1c8336 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-power-bd71828 @@ -0,0 +1,12 @@ +What: /sys/class/power_supply/bd71828_ac/auto_dcin_limit +Description: + Enable/Disable automatic management of input current limit + (ILIM_DCIN_EN bit). + + Possible values are: + + ============ =========================================== + 1 automatic adjustment of input current limit + 0 no adjustment of input current limit. This + helps for more unusual power sources like + solar modules. diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes index a16c54ab841b..4306966b7fcc 100644 --- a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes +++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes @@ -2,33 +2,36 @@ What: /sys/class/reboot-mode//reboot_modes Date: March 2026(TBD) KernelVersion: TBD Contact: linux-pm@vger.kernel.org - Description: +Description: This interface exposes the reboot-mode arguments registered with the reboot-mode framework. It is a read-only interface and provides a space separated list of reboot-mode arguments supported on the current platform. Example: + recovery fastboot bootloader The exact sysfs path may vary depending on the name of the driver that registers the arguments. - Example: + Example:: + /sys/class/reboot-mode/nvmem-reboot-mode/reboot_modes /sys/class/reboot-mode/syscon-reboot-mode/reboot_modes /sys/class/reboot-mode/qcom-pon/reboot_modes The supported arguments can be used by userspace to invoke device reset using the standard reboot() system - call interface, with the "argument" as string to "*arg" - parameter along with LINUX_REBOOT_CMD_RESTART2. + call interface, with the "argument" as string to ``*arg`` + parameter along with ``LINUX_REBOOT_CMD_RESTART2``. A driver can expose the supported arguments by registering them with the reboot-mode framework using the property names that follow the mode- format. Example: - mode-bootloader, mode-recovery. + + mode-bootloader, mode-recovery This attribute is useful for scripts or initramfs logic that need to programmatically determine diff --git a/Documentation/ABI/testing/sysfs-driver-dell-dw5826e-reset b/Documentation/ABI/testing/sysfs-driver-dell-dw5826e-reset new file mode 100644 index 000000000000..a665e265633f --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-dell-dw5826e-reset @@ -0,0 +1,9 @@ +What: /sys/bus/platform/devices//wwan_reset +Date: April 2026 +KernelVersion: 7.2 +Contact: Jackbb Wu +Description: + Writing to this file triggers a Platform Level Device Reset + (PLDR) of the Dell DW5826e WWAN module via an ACPI _DSM + method. This can be used to recover the modem when it is in + a frozen state and unable to respond to USB commands. diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon index 55ab45f669ac..0da739d9a816 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon @@ -251,6 +251,13 @@ Description: RO. Fan 2 speed in RPM. Only supported for particular Intel Xe graphics platforms. + On DG2 the driver always shows two fan channels, because the + FSC_READ_NUM_FANS command does not work on some cards. OEMs + decide how the fans map to tach channels, so two fans can share + one tach line. When that happens, the second channel + reads 0 RPM even though the fan is spinning. This is normal, not + a bug. + What: /sys/bus/pci/drivers/xe/.../hwmon/hwmon/fan3_input Date: March 2025 KernelVersion: 6.16 diff --git a/Documentation/ABI/testing/sysfs-driver-uniwill-laptop b/Documentation/ABI/testing/sysfs-driver-uniwill-laptop index 2397c65c969a..943f92c6b561 100644 --- a/Documentation/ABI/testing/sysfs-driver-uniwill-laptop +++ b/Documentation/ABI/testing/sysfs-driver-uniwill-laptop @@ -78,3 +78,28 @@ Description: Reading this file returns the profile names with the currently active one in brackets. + +What: /sys/bus/platform/devices/INOU0000:XX/ac_auto_boot +Date: March 2026 +KernelVersion: 7.1 +Contact: Armin Wolf +Description: + Allows userspace applications to configure if the device should boot automatically + when being connected to a power source. Writing "1"/"0" into this file + enables/disables this functionality. Enabling both AC auto boot and USB powershare + at the same time is not supported. + + Reading this file returns the current status of the AC auto boot functionality. + +What: /sys/bus/platform/devices/INOU0000:XX/usb_powershare_high +Date: March 2026 +KernelVersion: 7.1 +Contact: Armin Wolf +Description: + Allows userspace applications to configure if the device should continue to provide + power via the USB ports when hibernating or powered off. Might also increase the + power budget available to USB ports on some devices. Writing "1"/"0" into this + file enables/disables this functionality. Enabling both USB powershare and AC auto + boot at the same time is not supported. + + Reading this file returns the current status of the USB powershare functionality. diff --git a/Documentation/ABI/testing/sysfs-firmware-gsmi b/Documentation/ABI/testing/sysfs-firmware-gsmi index 7a558354c1ee..3b3c3ae6f458 100644 --- a/Documentation/ABI/testing/sysfs-firmware-gsmi +++ b/Documentation/ABI/testing/sysfs-firmware-gsmi @@ -19,7 +19,7 @@ Description: /sys/firmware/gsmi/vars: This directory has the same layout (and - underlying implementation as /sys/firmware/efi/vars. + underlying implementation) as /sys/firmware/efi/vars. See `Documentation/ABI/*/sysfs-firmware-efi-vars` for more information on how to interact with this structure. diff --git a/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities b/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities new file mode 100644 index 000000000000..964362558bb2 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities @@ -0,0 +1,19 @@ +What: /sys/firmware/sev/vulnerabilities/supported_mitigations +Date: June 2026 +Contact: linux-crypto@vger.kernel.org +Description: + Read-only interface that reports the vector of SEV-SNP + firmware vulnerability mitigations supported by the firmware. + +What: /sys/firmware/sev/vulnerabilities/verified_mitigations +Date: June 2026 +Contact: linux-crypto@vger.kernel.org +Description: + Read/write interface that reports the vector of SEV-SNP + firmware vulnerability mitigations already verified by the + firmware. Writing a vector value requests the firmware to + VERIFY the corresponding mitigation bit(s). + + The list of supported mitigations and the meaning of each + vector bit are both platform- and bug-specific and are + published as part of the AMD Security Bulletin. diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 423ec40e2e4e..1b58c029abd0 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -270,7 +270,8 @@ Description: Shows all enabled kernel features. inode_checksum, flexible_inline_xattr, quota_ino, inode_crtime, lost_found, verity, sb_checksum, casefold, readonly, compression, test_dummy_encryption_v2, - atomic_write, pin_file, encrypted_casefold, linear_lookup. + atomic_write, pin_file, encrypted_casefold, linear_lookup, + fserror. What: /sys/fs/f2fs//inject_rate Date: May 2016 @@ -1000,4 +1001,4 @@ Contact: "Chao Yu" Description: It can be used to tune priority of f2fs critical task, e.g. f2fs_ckpt, f2fs_gc threads, limitation as below: - it requires user has CAP_SYS_NICE capability. - - the range is [100, 139], by default the value is 100. + - the range is [100, 139], by default the value is 120. diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index b73e6bc28ea5..dd6b5bd76e11 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -1,26 +1,26 @@ what: /sys/kernel/mm/damon/ Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Interface for Data Access MONitoring (DAMON). Contains files for controlling DAMON. For more details on DAMON itself, please refer to Documentation/admin-guide/mm/damon/index.rst. What: /sys/kernel/mm/damon/admin/ Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Interface for privileged users of DAMON. Contains files for controlling DAMON that aimed to be used by privileged users. What: /sys/kernel/mm/damon/admin/kdamonds/nr_kdamonds Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for controlling each DAMON worker thread (kdamond) named '0' to 'N-1' under the kdamonds/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//state Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing 'on' or 'off' to this file makes the kdamond starts or stops, respectively. Reading the file returns the keywords based on the current status. Writing 'commit' to this file @@ -40,33 +40,33 @@ Description: Writing 'on' or 'off' to this file makes the kdamond starts or What: /sys/kernel/mm/damon/admin/kdamonds//pid Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the pid of the kdamond if it is running. What: /sys/kernel/mm/damon/admin/kdamonds//refresh_ms Date: Jul 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the time interval for automatic DAMON status file contents update. Writing '0' disables the update. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts/nr_contexts Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for controlling each DAMON context named '0' to 'N-1' under the contexts/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//avail_operations Date: Apr 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the available monitoring operations sets on the currently running kernel. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//operations Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a keyword for a monitoring operations set ('vaddr' for virtual address spaces monitoring, 'fvaddr' for fixed virtual address ranges monitoring, and 'paddr' for the physical address @@ -79,42 +79,42 @@ Description: Writing a keyword for a monitoring operations set ('vaddr' for What: /sys/kernel/mm/damon/admin/kdamonds//contexts//addr_unit Date: Aug 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing an integer to this file sets the 'address unit' parameter of the given operations set of the context. Reading the file returns the last-written 'address unit' value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//pause Date: Mar 2026 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a boolean keyword to this file sets the 'pause' request parameter for the context. Reading the file returns the last-written 'pause' value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/sample_us Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the sampling interval of the DAMON context in microseconds as the value. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/aggr_us Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the aggregation interval of the DAMON context in microseconds as the value. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/update_us Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the update interval of the DAMON context in microseconds as the value. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/access_bp Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the monitoring intervals auto-tuning target DAMON-observed access events ratio within the given time interval (aggrs in same directory), in bp @@ -122,7 +122,7 @@ Description: Writing a value to this file sets the monitoring intervals What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/aggrs Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the time interval to achieve the monitoring intervals auto-tuning target DAMON-observed access events ratio (access_bp in same directory) within. @@ -130,14 +130,14 @@ Description: Writing a value to this file sets the time interval to achieve What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/min_sample_us Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the minimum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/max_sample_us Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the maximum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. @@ -145,42 +145,82 @@ Description: Writing a value to this file sets the maximum value of What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/nr_regions/min WDate: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the minimum number of monitoring regions of the DAMON context as the value. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/nr_regions/max Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the maximum number of monitoring regions of the DAMON context as the value. Reading this file returns the value. +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/nr_probes +Date: May 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe named '0' to 'N-1' under the + probes/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters/nr_filters +Date: May 2026 +Contact: SJ Park +Description: Writing a number 'N' to this file creates the number of + directories for each DAMON probe filter named '0' to 'N-1' + under the filters/ directory. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//type +Date: May 2026 +Contact: SJ Park +Description: Writing to and reading from this file sets and gets the type of + the memory of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//path +Date: May 2026 +Contact: SJ Park +Description: If 'memcg' is written to the 'type' file, writing to and + reading from this file sets and gets the path to the memory + cgroup of the interest. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//matching +Date: May 2026 +Contact: SJ Park +Description: Writing 'Y' or 'N' to this file sets whether the filter is for + the memory of the 'type', or all except the 'type'. + +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/probes/

/filters//allow +Date: May 2026 +Contact: SJ Park +Description: Writing 'Y' or 'N' to this file sets whether to allow or reject + hitting the probe for the memory that satisfies the 'type' and + the 'matching' of the directory. + What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets/nr_targets Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for controlling each DAMON target of the context named '0' to 'N-1' under the contexts/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets//pid_target Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the pid of the target process if the context is for virtual address spaces monitoring, respectively. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets//obsolete_target Date: Oct 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the obsoleteness of the matching parameters commit destination target. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets//regions/nr_regions Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for setting each DAMON target memory region of the context named '0' to 'N-1' under the regions/ directory. In @@ -190,181 +230,181 @@ Description: Writing a number 'N' to this file creates the number of What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets//regions//start Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the start address of the monitoring region. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//targets//regions//end Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the end address of the monitoring region. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes/nr_schemes Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for controlling each DAMON-based operation scheme of the context named '0' to 'N-1' under the schemes/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//action Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the action of the scheme. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//target_nid Date: Jun 2024 -Contact: SeongJae Park +Contact: SJ Park Description: Action's target NUMA node id. Supported by only relevant actions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//apply_interval_us Date: Sep 2023 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a value to this file sets the action apply interval of the scheme in microseconds. Reading this file returns the value. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/sz/min Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the minimum size of the scheme's target regions in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/sz/max Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the maximum size of the scheme's target regions in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/nr_accesses/min Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the manimum 'nr_accesses' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/nr_accesses/max Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the maximum 'nr_accesses' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/age/min Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the minimum 'age' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//access_pattern/age/max Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the maximum 'age' of the scheme's target regions. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/ms Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the time quota of the scheme in milliseconds. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/bytes Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the size quota of the scheme in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/effective_bytes Date: Feb 2024 -Contact: SeongJae Park +Contact: SJ Park Description: Reading from this file gets the effective size quota of the scheme in bytes, which adjusted for the time quota and goals. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/reset_interval_ms Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the quotas charge reset interval of the scheme in milliseconds. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals/nr_goals Date: Nov 2023 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for setting automatic tuning of the scheme's aggressiveness named '0' to 'N-1' under the goals/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals//target_metric Date: Feb 2024 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the quota auto-tuning goal metric. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals//target_value Date: Nov 2023 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the target value of the goal metric. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals//current_value Date: Nov 2023 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the current value of the goal metric. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals//nid Date: Apr 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the nid parameter of the goal. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goals//path Date: Oct 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the path parameter of the goal. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/goal_tuner Date: Mar 2026 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the goal-based effective quota auto-tuning algorithm to use. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/fail_charge_num Date: Mar 2026 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the action-failed memory quota charging ratio numerator. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/fail_charge_denom Date: Mar 2026 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the action-failed memory quota charging ratio denominator. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/weights/sz_permil Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the under-quota limit regions prioritization weight for 'size' in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/weights/nr_accesses_permil Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the under-quota limit regions prioritization weight for 'nr_accesses' in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//quotas/weights/age_permil Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the under-quota limit regions prioritization weight for 'age' in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//watermarks/metric Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the metric of the watermarks for the scheme. The writable/readable keywords for this file are 'none' for disabling the watermarks @@ -373,44 +413,44 @@ Description: Writing to and reading from this file sets and gets the metric What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//watermarks/interval_us Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the metric check interval of the watermarks for the scheme in microseconds. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//watermarks/high Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the high watermark of the scheme in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//watermarks/mid Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the mid watermark of the scheme in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//watermarks/low Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the low watermark of the scheme in permil. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Directory for DAMON core layer-handled DAMOS filters. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters/nr_filters Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for setting filters of the scheme named '0' to 'N-1' under the core_filters/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//type Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the type of the memory of the interest. 'anon' for anonymous pages, 'memcg' for specific memory cgroup, 'young' for young pages, @@ -419,62 +459,62 @@ Description: Writing to and reading from this file sets and gets the type of What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//memcg_path Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'memcg' is written to the 'type' file, writing to and reading from this file sets and gets the path to the memory cgroup of the interest. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//addr_start Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'addr' is written to the 'type' file, writing to or reading from this file sets or gets the start address of the address range for the filter. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//addr_end Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'addr' is written to the 'type' file, writing to or reading from this file sets or gets the end address of the address range for the filter. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//min Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'hugepage_size' is written to the 'type' file, writing to or reading from this file sets or gets the minimum size of the hugepage for the filter. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//max Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'hugepage_size' is written to the 'type' file, writing to or reading from this file sets or gets the maximum size of the hugepage for the filter. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//damon_target_idx Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: If 'target' is written to the 'type' file, writing to or reading from this file sets or gets the index of the DAMON monitoring target of the interest. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//matching Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing 'Y' or 'N' to this file sets whether the filter is for the memory of the 'type', or all except the 'type'. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters//allow Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing 'Y' or 'N' to this file sets whether to allow or reject applying the scheme's action to the memory that satisfies the 'type' and the 'matching' of the directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//ops_filters Date: Feb 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Directory for DAMON operations set layer-handled DAMOS filters. Files under this directory works same to those of /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//core_filters @@ -482,7 +522,7 @@ Description: Directory for DAMON operations set layer-handled DAMOS filters. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//filters Date: Dec 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Directory for DAMOS filters. Files under this directory works same to those of /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//{core,ops}_filters @@ -491,14 +531,14 @@ Description: Directory for DAMOS filters. Files under this directory works What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//dests/nr_dests Date: Jul 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number 'N' to this file creates the number of directories for setting action destinations of the scheme named '0' to 'N-1' under the dests/ directory. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//dests//id Date: Jul 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the id of the DAMOS action destination. For DAMOS_MIGRATE_{HOT,COLD} actions, the destination node's node id can be written and @@ -506,98 +546,98 @@ Description: Writing to and reading from this file sets and gets the id of What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//dests//weight Date: Jul 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing to and reading from this file sets and gets the weight of the DAMOS action destination to select as the destination of each action among the destinations. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/nr_tried Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the number of regions that the action of the scheme has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/sz_tried Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the total size of regions that the action of the scheme has tried to be applied in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/nr_applied Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the number of regions that the action of the scheme has successfully applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/sz_applied Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the total size of regions that the action of the scheme has successfully applied in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/sz_ops_filter_passed Date: Dec 2024 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the total size of memory that passed DAMON operations layer-handled filters of the scheme in bytes. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/qt_exceeds Date: Mar 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the number of the exceed events of the scheme's quotas. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/nr_snapshots Date: Dec 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the total number of DAMON snapshots that the scheme has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//stats/max_nr_snapshots Date: Dec 2025 -Contact: SeongJae Park +Contact: SJ Park Description: Writing a number to this file sets the upper limit of nr_snapshots that deactivates the scheme when the limit is reached or exceeded. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions/total_bytes Date: Jul 2023 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the total amount of memory that corresponding DAMON-based Operation Scheme's action has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//start Date: Oct 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the start address of a memory region that corresponding DAMON-based Operation Scheme's action has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//end Date: Oct 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the end address of a memory region that corresponding DAMON-based Operation Scheme's action has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//nr_accesses Date: Oct 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the 'nr_accesses' of a memory region that corresponding DAMON-based Operation Scheme's action has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//age Date: Oct 2022 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the 'age' of a memory region that corresponding DAMON-based Operation Scheme's action has tried to be applied. What: /sys/kernel/mm/damon/admin/kdamonds//contexts//schemes//tried_regions//sz_filter_passed Date: Dec 2024 -Contact: SeongJae Park +Contact: SJ Park Description: Reading this file returns the size of the memory in the region that passed DAMON operations layer-handled filters of the scheme in bytes. diff --git a/Documentation/ABI/testing/sysfs-kernel-slab b/Documentation/ABI/testing/sysfs-kernel-slab index b26e4299f822..c52034e3e794 100644 --- a/Documentation/ABI/testing/sysfs-kernel-slab +++ b/Documentation/ABI/testing/sysfs-kernel-slab @@ -113,8 +113,10 @@ KernelVersion: 2.6.22 Contact: Pekka Enberg , Christoph Lameter Description: - The cpu_slabs file is read-only and displays how many cpu slabs - are active and their NUMA locality. + The cpu_slabs file is read-only. It is deprecated and always + reads "0" since the removal of per-cpu slabs in Linux 7.0. It + previously displayed how many cpu slabs were active and their + NUMA locality. The file is kept for backwards compatibility. What: /sys/kernel/slab//cpuslab_flush Date: April 2009 @@ -509,12 +511,16 @@ What: /sys/kernel/slab//slabs_cpu_partial Date: Aug 2011 Contact: Christoph Lameter Description: - This read-only file shows the number of partialli allocated - frozen slabs. + This read-only file is deprecated and always reads "0(0)" since + the removal of per-cpu partial slabs in Linux 7.0. It previously + showed the number of partially allocated frozen slabs. The file + is kept for backwards compatibility. What: /sys/kernel/slab//cpu_partial Date: Aug 2011 Contact: Christoph Lameter Description: - This read-only file shows the number of per cpu partial - pages to keep around. + This file is deprecated and always reads "0" since the removal of + per-cpu partial slabs in Linux 7.0. It previously showed the + number of per-cpu partial pages to keep around. The file is kept + for backwards compatibility. diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi index 89acb6638df8..f9825c6150b5 100644 --- a/Documentation/ABI/testing/sysfs-platform-asus-wmi +++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi @@ -58,6 +58,15 @@ Description: * 1 - overboost, * 2 - silent +What: /sys/devices/platform//keystone +Date: Jun 2026 +KernelVersion: 7.2 +Contact: "Dariusz Figzał" +Description: + Reports the Keystone dongle insert state (read-only): + * 0 - not inserted + * 1 - inserted + What: /sys/devices/platform//gpu_mux_mode Date: Aug 2022 KernelVersion: 6.1 diff --git a/Documentation/PCI/pci.rst b/Documentation/PCI/pci.rst index f4d2662871ab..be35e9a1ee75 100644 --- a/Documentation/PCI/pci.rst +++ b/Documentation/PCI/pci.rst @@ -338,7 +338,7 @@ the PCI_IRQ_MSI and PCI_IRQ_MSIX flags will fail, so try to always specify PCI_IRQ_INTX as well. Drivers that have different interrupt handlers for MSI/MSI-X and -legacy INTx should chose the right one based on the msi_enabled +legacy INTx should choose the right one based on the msi_enabled and msix_enabled flags in the pci_dev structure after calling pci_alloc_irq_vectors. diff --git a/Documentation/PCI/pciebus-howto.rst b/Documentation/PCI/pciebus-howto.rst index 375d9ce171f6..9cc133ccdeec 100644 --- a/Documentation/PCI/pciebus-howto.rst +++ b/Documentation/PCI/pciebus-howto.rst @@ -97,7 +97,7 @@ register its service with the PCI Express Port Bus driver (see section 5.2.1 & 5.2.2). It is important that a service driver initializes the pcie_port_service_driver data structure, included in header file /include/linux/pcieport_if.h, before calling these APIs. -Failure to do so will result an identity mismatch, which prevents +Failure to do so will result in an identity mismatch, which prevents the PCI Express Port Bus driver from loading a service driver. pcie_port_service_register diff --git a/Documentation/accounting/delay-accounting.rst b/Documentation/accounting/delay-accounting.rst index e209c46241b0..4bb2e72a6043 100644 --- a/Documentation/accounting/delay-accounting.rst +++ b/Documentation/accounting/delay-accounting.rst @@ -212,3 +212,46 @@ Advanced usage examples:: # ./delaytop -d secs Specify refresh interval as secs + + # ./delaytop -t type + Display only specified delay type with avg/max/timestamp + (rows sorted by MAX for that type, largest first) + + + +delaytop add delay_max fields to track the maximum delay value for each delay type +(cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy) per task. + + bash# ./delaytop -t cpu + System Pressure Information: (avg10/avg60/avg300/total) + CPU some: 0.4%/ 0.2%/ 0.1%/ 220(ms) + CPU full: 0.0%/ 0.0%/ 0.0%/ 0(ms) + Memory full: 0.0%/ 0.0%/ 0.0%/ 0(ms) + Memory some: 0.0%/ 0.0%/ 0.0%/ 0(ms) + IO full: 0.0%/ 0.0%/ 0.0%/ 12(ms) + IO some: 0.0%/ 0.0%/ 0.0%/ 13(ms) + IRQ full: 0.0%/ 0.0%/ 0.0%/ 0(ms) + [q]quit + Top 20 processes (sorted by cpu MAX delay, largest first): + PID TGID COMMAND AVG(ms) MAX(ms) MAX_TIMESTAMP + ------------------------------------------------------------------------ + 9 9 kworker/0:0-eve 0.59 16.87 2026-05-27T13:32:39 + 30 30 kworker/2:0H-kb 2.87 11.36 2026-05-27T13:32:36 + 27 27 migration/2 1.05 9.51 2026-05-27T13:32:37 + 50 50 kworker/2:1-eve 0.50 9.13 2026-05-27T13:32:37 + 15 15 rcu_preempt 0.11 8.98 2026-05-27T13:32:37 + 1 1 init 0.17 7.12 2026-05-27T13:32:38 + 67 67 scsi_eh_0 1.20 4.23 2026-05-27T13:32:37 + 23 23 ksoftirqd/1 1.12 3.77 2026-05-27T13:32:36 + 3 3 pool_workqueue_ 0.72 3.55 2026-05-27T13:32:38 + 62 62 kworker/u20:2-a 0.49 3.03 2026-05-27T13:32:37 + 2 2 kthreadd 0.18 2.82 2026-05-27T13:32:37 + 11 11 kworker/0:1 1.42 2.76 2026-05-27T13:32:36 + 39 39 kworker/u20:0-a 0.10 2.71 2026-05-27T13:32:38 + 17 17 rcu_exp_gp_kthr 0.25 2.65 2026-05-27T13:32:37 + 66 66 kworker/u20:3-e 0.38 2.55 2026-05-27T13:32:37 + 20 20 cpuhp/0 0.53 2.51 2026-05-27T13:32:37 + 28 28 ksoftirqd/2 0.59 2.48 2026-05-27T13:32:37 + 55 55 kworker/u19:1 0.88 2.42 2026-05-27T13:32:37 + 13 13 kworker/R-mm_pe 1.18 2.35 2026-05-27T13:32:36 + 54 54 kworker/3:1-eve 0.14 2.20 2026-05-27T13:32:38 diff --git a/Documentation/admin-guide/bootconfig.rst b/Documentation/admin-guide/bootconfig.rst index f712758472d5..3d6412458c8b 100644 --- a/Documentation/admin-guide/bootconfig.rst +++ b/Documentation/admin-guide/bootconfig.rst @@ -234,6 +234,87 @@ Kconfig option selected. Note that even if you set this option, you can override the embedded bootconfig by another bootconfig which attached to the initrd. +Rendering Embedded kernel.* Keys at Build Time +---------------------------------------------- + +By default, the embedded bootconfig (``CONFIG_BOOT_CONFIG_EMBED=y``) is +parsed at runtime, after ``parse_early_param()`` has already run. Early +parameter handlers (``mem=``, ``earlycon=``, ``loglevel=``, ...) therefore +cannot see values supplied via the embedded ``kernel`` subtree. + +``CONFIG_CMDLINE_FROM_BOOTCONFIG`` resolves this by rendering the +``kernel`` subtree of ``CONFIG_BOOT_CONFIG_EMBED_FILE`` into a flat cmdline +string at kernel build time (via ``tools/bootconfig -C``) and prepending +it to ``boot_command_line`` during early architecture setup, so the keys +are visible to ``parse_early_param()``. + +The option requires ``CONFIG_BOOT_CONFIG_EMBED=y``, a non-empty +``CONFIG_BOOT_CONFIG_EMBED_FILE``, ``CONFIG_CMDLINE`` to be empty, and +an architecture that selects ``CONFIG_ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG``. +Currently only x86 selects it; on other architectures the embedded +bootconfig still works, but only through the late runtime parser. + +The same ``bootconfig`` opt-in applies as elsewhere: the rendered keys +are prepended only when ``bootconfig`` (in any form) appears on the +kernel command line, or when ``CONFIG_BOOT_CONFIG_FORCE`` is set, which +defaults to ``y`` when ``CONFIG_BOOT_CONFIG_EMBED`` is set. + +For example, given:: + + kernel { + loglevel = 7 + mem = 4G + } + +the kernel boots as if ``loglevel=7 mem=4G`` had been prepended to the +bootloader command line, with the values visible to early-parsed +handlers. Comma-separated values are still expanded into multiple +cmdline entries per the bootconfig array convention -- the embedded +``kernel.earlycon = "uart8250,io,0x3f8"`` must be quoted to land as a +single ``earlycon=`` entry, exactly as for the runtime parser. + +If the rendered string would not fit in ``COMMAND_LINE_SIZE`` together +with the existing command line, the prepend is skipped and an error is +logged, so an oversized embedded bootconfig cannot brick a boot. + +Interaction with other command line and bootconfig sources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With ``CONFIG_CMDLINE_FROM_BOOTCONFIG=y`` the rendered ``kernel`` +subtree behaves like a build-time command line (similar to +``CONFIG_CMDLINE``), not like a bootconfig source. It is prepended to +``boot_command_line`` in ``setup_arch()``, before ``parse_early_param()`` +and long before the runtime parser looks at an initrd. Options can reach +the kernel from up to four places: + +- Bootloader command line: the arguments the boot loader passes. The + embedded cmdline is prepended in front of them, so for last-one-wins + parameters a bootloader option still overrides the embedded value. + Visible in /proc/cmdline. +- Embedded cmdline (this option): the rendered ``kernel`` subtree, + prepended early so it is seen by ``parse_early_param()``. Visible in + /proc/cmdline. +- Initrd bootconfig: parsed late in ``setup_boot_config()``; its + ``kernel`` keys are placed ahead of ``boot_command_line``, i.e. before + the embedded cmdline, so last-wins favors the embedded values. As a + bootconfig source, an initrd bootconfig still replaces the embedded + bootconfig. Visible in /proc/cmdline and /proc/bootconfig. +- Embedded bootconfig (runtime): parsed late, only when no initrd + bootconfig is present. Visible in /proc/cmdline and /proc/bootconfig. + +So with this option the embedded ``kernel.*`` values take precedence +over an initrd bootconfig's ``kernel.*`` values: for early parameters +the initrd is not parsed yet, and for ordinary parameters the embedded +keys land later in the command line. If you need an initrd bootconfig to +override the embedded ``kernel.*`` keys, leave this option off and rely +on the runtime parser. + +The rendered string is part of the command line, so it appears in +/proc/cmdline. It is deliberately not shown in /proc/bootconfig: that +file keeps reporting the parsed bootconfig tree -- the initrd bootconfig +if present, otherwise the embedded bootconfig -- independent of whether +build-time cmdline rendering is enabled. + Kernel parameters via Boot Config ================================= diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 993446ab66d0..b0bd591bc4bf 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1145,7 +1145,7 @@ will be referred to. All time durations are in microseconds. This file exists whether the controller is enabled or not. It always reports the following three stats, which account for all the - processes in the cgroup: + processes in the cgroup (including those in descendant cgroups): - usage_usec - user_usec @@ -1160,6 +1160,27 @@ will be referred to. All time durations are in microseconds. - nr_bursts - burst_usec + Note that the above five CFS bandwidth stats are non-hierarchical; + they only account for throttling caused by this cgroup's own bandwidth + limit, not including throttling inherited from ancestor cgroups. + + cpu.stat.local + A read-only flat-keyed file. + This file exists whether the controller is enabled or not. + + It reports the following stat when the controller is enabled: + + - throttled_usec + + Unlike the ``throttled_usec`` reported by ``cpu.stat`` which + accounts for throttling caused by this cgroup's own CFS + bandwidth limit, ``cpu.stat.local`` reports the actual + throttling time incurred by this cgroup's own runqueues, + which may include throttling inherited from ancestor + cgroup bandwidth limits. + + When the controller is not enabled, this stat is not reported. + cpu.weight A read-write single value file which exists on non-root cgroups. The default is "100". @@ -1570,7 +1591,7 @@ The following nested keys are defined. sock (npn) Amount of memory used in network transmission buffers - vmalloc (npn) + vmalloc Amount of memory used for vmap backed memory. shmem @@ -1735,7 +1756,7 @@ The following nested keys are defined. Number of pages written from zswap to swap. zswap_incomp - Number of incompressible pages currently stored in zswap + Amount of memory used by incompressible pages currently stored in zswap without compression. These pages could not be compressed to a size smaller than PAGE_SIZE, so they are stored as-is. @@ -2527,6 +2548,13 @@ Cpuset Interface Files a need to change "cpuset.mems" with active tasks, it shouldn't be done frequently. + For a multithreaded process, the threadgroup leader is + considered the owner of the group's memory. Memory policy + rebinding and migration will only happen with respect to the + threadgroup leader. To avoid unexpected results, non-leading + threads shouldn't be put into another cgroup whose "cpuset.mems" + doesn't fully overlap that of the threadgroup leader. + cpuset.mems.effective A read-only multiple values file which exists on all cpuset-enabled cgroups. @@ -2861,6 +2889,12 @@ DMEM Interface Files The semantics are the same as for the memory cgroup controller, and are calculated in the same way. + dmem.peak + A read-only nested-keyed file that exists on non-root cgroups. + + The max device memory usage recorded for the cgroup and its + descendants since the creation of the cgroup for each region. + dmem.capacity A read-only file that describes maximum region capacity. It only exists on the root cgroup. Not all memory can be diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt index 440633642fea..f544399675fa 100644 --- a/Documentation/admin-guide/devices.txt +++ b/Documentation/admin-guide/devices.txt @@ -291,7 +291,6 @@ 154 = /dev/pmu Macintosh PowerBook power manager 155 = 156 = /dev/lcd Front panel LCD display - 157 = /dev/ac Applicom Intl Profibus card 158 = /dev/nwbutton Netwinder external button 159 = /dev/nwdebug Netwinder debug interface 160 = /dev/nwflash Netwinder flash memory diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a68003c3599c..b2d7d3540ded 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1089,14 +1089,14 @@ Kernel parameters It will be ignored when crashkernel=X,high is not used or memory reserved is below 4G. crashkernel=size[KMG],cma - [KNL, X86, ppc] Reserve additional crash kernel memory from - CMA. This reservation is usable by the first system's - userspace memory and kernel movable allocations (memory - balloon, zswap). Pages allocated from this memory range - will not be included in the vmcore so this should not - be used if dumping of userspace memory is intended and - it has to be expected that some movable kernel pages - may be missing from the dump. + [KNL, X86, ARM64, RISCV, PPC] Reserve additional crash + kernel memory from CMA. This reservation is usable by + the first system's userspace memory and kernel movable + allocations (memory balloon, zswap). Pages allocated + from this memory range will not be included in the vmcore + so this should not be used if dumping of userspace memory + is intended and it has to be expected that some movable + kernel pages may be missing from the dump. A standard crashkernel reservation, as described above, is still needed to hold the crash kernel and initrd. @@ -2064,8 +2064,11 @@ Kernel parameters hugetlb_cma= [HW,CMA,EARLY] The size of a CMA area used for allocation of gigantic hugepages. Or using node format, the size of a CMA area per node can be specified. - Format: nn[KMGTPE] or (node format) - :nn[KMGTPE][,:nn[KMGTPE]] + The size can be an absolute value (e.g., 2G) or a + percentage of the total memory or node memory (e.g., 20%). + Format: nn[KMGTPE] or nn% or (node format) + :nn[KMGTPE][,:nn[KMGTPE]] or + :nn%[,:nn%] The size must be a multiple of the gigantic page size. When using node format, this applies to each per-node size. @@ -3977,6 +3980,11 @@ Kernel parameters Note that even when enabled, there are a few cases where the feature is not effective. + mempool_debug [MM] + Enable mempool debugging. This enables element + poison checking when freeing elements back to the + pool. Useful for debugging mempool corruption. + memtest= [KNL,X86,ARM,M68K,PPC,RISCV,EARLY] Enable memtest Format: default : 0 @@ -4288,13 +4296,6 @@ Kernel parameters Only applies if the softerr mount option is enabled, and the specified value is >= 0. - nfs.enable_ino64= - [NFS] enable 64-bit inode numbers. - If zero, the NFS client will fake up a 32-bit inode - number for the readdir() and stat() syscalls instead - of returning the full 64-bit number. - The default is to return 64-bit inode numbers. - nfs.idmap_cache_timeout= [NFS] set the maximum lifetime for idmapper cache entries. @@ -6420,9 +6421,9 @@ Kernel parameters reboot= [KNL] Format (x86 or x86_64): [w[arm] | c[old] | h[ard] | s[oft] | g[pio]] | d[efault] \ - [[,]s[mp]#### \ + [[,]s[mp]####] \ [[,]b[ios] | a[cpi] | k[bd] | t[riple] | e[fi] | p[ci]] \ - [[,]f[orce] + [[,]f[orce]] Where reboot_mode is one of warm (soft) or cold (hard) or gpio (prefix with 'panic_' to set mode for panic reboot only), @@ -6923,7 +6924,7 @@ Kernel parameters apic=verbose is specified. Example: apic=debug show_lapic=all - slab_debug[=options[,slabs][;[options[,slabs]]...] [MM] + slab_debug[=options[,slabs][;[options[,slabs]]...]] [MM] Enabling slab_debug allows one to determine the culprit if slab objects become corrupted. Enabling slab_debug can create guard zones around objects and diff --git a/Documentation/admin-guide/laptops/uniwill-laptop.rst b/Documentation/admin-guide/laptops/uniwill-laptop.rst index 1f3ca84c7d88..be50b45b82ef 100644 --- a/Documentation/admin-guide/laptops/uniwill-laptop.rst +++ b/Documentation/admin-guide/laptops/uniwill-laptop.rst @@ -46,11 +46,20 @@ Battery Charging Control .. warning:: Some devices do not properly implement the charging threshold interface. Forcing the driver to enable access to said interface on such devices might damage the battery [1]_. Because of this the driver will not enable said feature even when - using the ``force`` module parameter. + using the ``force`` module parameter. The charging profile interface will be + available instead. -The ``uniwill-laptop`` driver supports controlling the battery charge limit. This happens over -the standard ``charge_control_end_threshold`` power supply sysfs attribute. All values -between 1 and 100 percent are supported. +The ``uniwill-laptop`` driver supports controlling the battery charge limit. This either happens +over the standard ``charge_control_end_threshold`` or ``charge_types`` power supply sysfs attribute, +depending on the device. When using the ``charge_control_end_threshold`` sysfs attribute, all values +between 1 and 100 percent are supported. When using the ``charge_types`` sysfs attribute, the driver +supports switching between the ``Standard``, ``Trickle`` and ``Long Life`` profiles. + +Keep in mind that when using the ``charge_types`` sysfs attribute, the EC firmware will hide the +true charging status of the battery from the operating system, potentially misleading users into +thinking that the charging profile does not work. Checking the ``current_now`` sysfs attribute +tells you the true charging status of the battery even when using the ``charge_types`` sysfs +attribute (0 means that the battery is currently not charging). Additionally the driver signals the presence of battery charging issues through the standard ``health`` power supply sysfs attribute. @@ -68,6 +77,19 @@ LED class device. The default name of this LED class device is ``uniwill:multico See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details on how to control the various animation modes of the lightbar. +Keyboard Backlight +------------------ + +The ``uniwill-laptop`` driver supports controlling the keyboard backlight using the standard +LED class interface. The default name of this LED class device is ``uniwill:white:kbd_backlight`` +when the keyboard backlight supports only a single color, or ``uniwill:multicolor:kbd_backlight`` +when the keyboard backlight supports RGB colors. The maximum intensity for each color channel +in RGB mode is 50. + +Keep in mind that due to hardware design choices, the driver does not support the RGB value +``0x000000`` (black), instead it will fall back to ``0x010101`` (faint white). In order to +disable the keyboard backlight, the standard LED brightness setting has to be used instead. + Configurable TGP ---------------- @@ -76,6 +98,20 @@ allow it. See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details. +AC Auto Boot +------------ + +The ``uniwill-laptop`` driver allows the user to configure if the system should automatically +boot when being connected to a power source, see +Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details. + +USB Powershare +-------------- + +The ``uniwill-laptop`` driver allows the user to configure if the system should continue to +provide power via the USB ports when hibernating or powered off, see +Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details. + References ========== diff --git a/Documentation/admin-guide/media/bttv.rst b/Documentation/admin-guide/media/bttv.rst index 58cbaf6df694..78c3e560b806 100644 --- a/Documentation/admin-guide/media/bttv.rst +++ b/Documentation/admin-guide/media/bttv.rst @@ -1239,7 +1239,7 @@ Models: - Galaxis DVB Card C CI - Galaxis DVB Card S - Galaxis DVB Card C -- Galaxis plug.in S [neuer Name: Galaxis DVB Card S CI +- Galaxis plug.in S [new Name: Galaxis DVB Card S CI] Hauppauge ~~~~~~~~~ diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index 011296f1e7c2..b2649ea011f9 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -246,7 +246,7 @@ writing to and reading from the files. Under ``nr_regions`` directory, two files for the lower-bound and upper-bound of DAMON's monitoring regions (``min`` and ``max``, respectively), which controls the monitoring overhead, exist. You can set and get the values by -writing to and rading from the files. +writing to and reading from the files. For more details about the intervals and monitoring regions range, please refer to the Design document (:doc:`/mm/damon/design`). @@ -264,7 +264,7 @@ Please refer to the :ref:`design document of the feature ` for the internal of the tuning mechanism. Reading and writing the four files under ``intervals_goal`` directory shows and updates the tuning parameters that described in the -:ref:design doc ` with the same +:ref:`design doc ` with the same names. The tuning starts with the user-set ``sample_us`` and ``aggr_us``. The tuning-applied current values of the two intervals can be read from the ``sample_us`` and ``aggr_us`` files after writing ``update_tuned_intervals`` to @@ -377,7 +377,7 @@ schemes// In each scheme directory, nine directories (``access_pattern``, ``quotas``, ``watermarks``, ``core_filters``, ``ops_filters``, ``filters``, ``dests``, ``stats``, and ``tried_regions``) and three files (``action``, ``target_nid`` -and ``apply_interval``) exist. +and ``apply_interval_us``) exist. The ``action`` file is for setting and getting the scheme's :ref:`action `. The keywords that can be written to and read @@ -743,7 +743,7 @@ counter). Finally the tenth field (``X``) shows the ``age`` of the region (refer to :ref:`design ` for more details of the counter). -If the event was ``damon:damos_beofre_apply``, the ``perf script`` output would +If the event was ``damon:damos_before_apply``, the ``perf script`` output would be somewhat like below:: kdamond.0 47293 [000] 80801.060214: damon:damos_before_apply: ctx_idx=0 scheme_idx=0 target_idx=0 nr_regions=11 121932607488-135128711168: 0 136 diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst index 67a941903fd2..3cc15d800be1 100644 --- a/Documentation/admin-guide/mm/hugetlbpage.rst +++ b/Documentation/admin-guide/mm/hugetlbpage.rst @@ -455,7 +455,7 @@ used to change the file attributes on hugetlbfs. Also, it is important to note that no such mount command is required if applications are going to use only shmat/shmget system calls or mmap with MAP_HUGETLB. For an example of how to use mmap with MAP_HUGETLB see -:ref:`map_hugetlb ` below. +:ref:`examples ` below. Users who wish to use hugetlb memory via shared memory segment should be members of a supplementary group and system admin needs to configure that gid @@ -473,16 +473,13 @@ a hugetlb page and the length is smaller than the hugepage size. Examples ======== -.. _map_hugetlb: +.. _examples: -``map_hugetlb`` - see tools/testing/selftests/mm/map_hugetlb.c +``hugetlb-shm`` + see tools/testing/selftests/mm/hugetlb-shm.c -``hugepage-shm`` - see tools/testing/selftests/mm/hugepage-shm.c - -``hugepage-mmap`` - see tools/testing/selftests/mm/hugepage-mmap.c +``hugetlb-mmap`` + see tools/testing/selftests/mm/hugetlb-mmap.c The `libhugetlbfs`_ library provides a wide range of userspace tools to help with huge page usability, environment setup, and control. diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst index c57e61b5d8aa..20e3fe76f099 100644 --- a/Documentation/admin-guide/mm/pagemap.rst +++ b/Documentation/admin-guide/mm/pagemap.rst @@ -67,7 +67,7 @@ number of times a page is mapped. * ``/proc/kpageflags``. This file contains a 64-bit set of flags for each page, indexed by PFN. - The flags are (from ``fs/proc/page.c``, above kpageflags_read): + The flags are (from ``include/uapi/linux/kernel-page-flags.h``): 0. LOCKED 1. ERROR @@ -264,7 +264,7 @@ The ``struct pm_scan_arg`` is used as the argument of the IOCTL. provided or not. 3. The range is specified through ``start`` and ``end``. 4. The walk can abort before visiting the complete range such as the user buffer - can get full etc. The walk ending address is specified in``end_walk``. + can get full etc. The walk ending address is specified in ``walk_end``. 5. The output buffer of ``struct page_region`` array and size is specified in ``vec`` and ``vec_len``. 6. The optional maximum requested pages are specified in the ``max_pages``. @@ -275,7 +275,7 @@ Find pages which have been written and WP them as well:: struct pm_scan_arg arg = { .size = sizeof(arg), - .flags = PM_SCAN_CHECK_WPASYNC | PM_SCAN_CHECK_WPASYNC, + .flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, .. .category_mask = PAGE_IS_WRITTEN, .return_mask = PAGE_IS_WRITTEN, @@ -288,7 +288,7 @@ present or huge:: .size = sizeof(arg), .flags = 0, .. - .category_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED, + .category_mask = PAGE_IS_WRITTEN | PAGE_IS_FILE, .category_inverted = PAGE_IS_SWAPPED, .category_anyof_mask = PAGE_IS_PRESENT | PAGE_IS_HUGE, .return_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED | diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 76f4eb14e262..23f8d13c2629 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -63,7 +63,8 @@ often. THP can be enabled system wide or restricted to certain tasks or even memory ranges inside task's address space. Unless THP is completely disabled, there is ``khugepaged`` daemon that scans memory and -collapses sequences of basic pages into PMD-sized huge pages. +collapses sequences of basic pages into huge pages of either PMD size +or mTHP sizes, if the system is configured to do so. The THP behaviour is controlled via :ref:`sysfs ` interface and using madvise(2) and prctl(2) system calls. @@ -219,10 +220,10 @@ this behaviour by writing 0 to shrink_underused, and enable it by writing echo 0 > /sys/kernel/mm/transparent_hugepage/shrink_underused echo 1 > /sys/kernel/mm/transparent_hugepage/shrink_underused -khugepaged will be automatically started when PMD-sized THP is enabled +khugepaged will be automatically started when any THP size is enabled (either of the per-size anon control or the top-level control are set to "always" or "madvise"), and it'll be automatically shutdown when -PMD-sized THP is disabled (when both the per-size anon control and the +all THP sizes are disabled (when both the per-size anon control and the top-level control are "never") process THP controls @@ -265,8 +266,8 @@ Khugepaged controls ------------------- .. note:: - khugepaged currently only searches for opportunities to collapse to - PMD-sized THP and no attempt is made to collapse to other THP + khugepaged currently only searches for opportunities to collapse file/shmem + to PMD-sized THP. Only anonymous memory will attempt to collapse to other THP sizes. khugepaged runs usually at low frequency so while one may not want to @@ -296,11 +297,11 @@ allocation failure to throttle the next allocation attempt:: The khugepaged progress can be seen in the number of pages collapsed (note that this counter may not be an exact count of the number of pages collapsed, since "collapsed" could mean multiple things: (1) A PTE mapping -being replaced by a PMD mapping, or (2) All 4K physical pages replaced by -one 2M hugepage. Each may happen independently, or together, depending on -the type of memory and the failures that occur. As such, this value should -be interpreted roughly as a sign of progress, and counters in /proc/vmstat -consulted for more accurate accounting):: +being replaced by a PMD mapping, or (2) physical pages replaced by one +hugepage of various sizes (PMD-sized or mTHP). Each may happen independently, +or together, depending on the type of memory and the failures that occur. +As such, this value should be interpreted roughly as a sign of progress, +and counters in /proc/vmstat consulted for more accurate accounting):: /sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed @@ -308,16 +309,21 @@ for each pass:: /sys/kernel/mm/transparent_hugepage/khugepaged/full_scans -``max_ptes_none`` specifies how many extra small pages (that are -not already mapped) can be allocated when collapsing a group -of small pages into one large page:: +``max_ptes_none`` specifies how many empty (none/zero) pages are allowed +when collapsing a group of small pages into one large page:: /sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none -A higher value leads to use additional memory for programs. -A lower value leads to gain less thp performance. Value of -max_ptes_none can waste cpu time very little, you can -ignore it. +For PMD-sized THP collapse, this directly limits the number of empty pages +allowed in the 2MB region. + +For mTHP collapse, only 0 or (HPAGE_PMD_NR - 1) are supported. At +HPAGE_PMD_NR - 1, we collapse to the highest possible order. Any intermediate +value will emit a warning and mTHP collapse will default to max_ptes_none=0. + +A higher value allows more empty pages, potentially leading to more memory +usage but better THP performance. A lower value is more conservative and +may result in fewer THP collapses. ``max_ptes_swap`` specifies how many pages can be brought in from swap when collapsing a group of pages into a transparent huge page:: @@ -337,6 +343,15 @@ that THP is shared. Exceeding the number would block the collapse:: A higher value may increase memory footprint for some workloads. +.. note:: + For mTHP collapse, khugepaged does not support collapsing regions that + contain shared or swapped out pages, as this could lead to continuous + promotion to higher orders. The collapse will fail if any shared or + swapped PTEs are encountered during the scan. + + Currently, madvise_collapse only supports collapsing to PMD-sized THPs + and does not attempt mTHP collapses. + Boot parameters =============== @@ -639,6 +654,14 @@ anon_fault_fallback_charge instead falls back to using huge pages with lower orders or small pages even though the allocation was successful. +collapse_alloc + is incremented every time a huge page is successfully allocated for a + khugepaged collapse. + +collapse_alloc_failed + is incremented every time a huge page allocation fails during a + khugepaged collapse. + zswpout is incremented every time a huge page is swapped out to zswap in one piece without splitting. @@ -706,6 +729,20 @@ nr_anon_partially_mapped an anonymous THP as "partially mapped" and count it here, even though it is not actually partially mapped anymore. +collapse_exceed_none_pte + The number of collapse attempts that failed due to exceeding the + max_ptes_none threshold. + +collapse_exceed_swap_pte + The number of collapse attempts that failed due to exceeding the + max_ptes_swap threshold. For non-PMD orders this occurs if a mTHP range + contains at least one swap PTE. + +collapse_exceed_shared_pte + The number of collapse attempts that failed due to exceeding the + max_ptes_shared threshold. For non-PMD orders this occurs if a mTHP range + contains at least one shared PTE. + As the system ages, allocating huge pages may be expensive as the system uses memory compaction to copy data around memory to free a huge page for use. There are some counters in ``/proc/vmstat`` to help diff --git a/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst b/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst index d367ba4d744a..b43ad4d5e333 100644 --- a/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst +++ b/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst @@ -88,8 +88,15 @@ and "fabric_cluster_id" in the directory. Attributes in each directory: +``instance_id`` + This attribute is used to get die indices in userspace mapped MMIO + blocks. Indices are local to a single TPMI partition. Needed for direct + TPMI register access. + ``domain_id`` This attribute is used to get the power domain id of this instance. + Indices are unique in all TPMI partitions on a given CPU package. Can be + used to map compute dies to corresponding CPUs. ``die_id`` This attribute is used to get the Linux die id of this instance. diff --git a/Documentation/admin-guide/sysctl/crypto.rst b/Documentation/admin-guide/sysctl/crypto.rst index b707bd314a64..9a1bd53287f4 100644 --- a/Documentation/admin-guide/sysctl/crypto.rst +++ b/Documentation/admin-guide/sysctl/crypto.rst @@ -7,6 +7,42 @@ kernel configuration: .. contents:: :local: +.. _af_alg_restrict: + +af_alg_restrict +=============== + +Controls the level of restriction of AF_ALG. + +AF_ALG is a deprecated and rarely-used userspace interface that is a +frequent source of vulnerabilities. It also unnecessarily exposes a +large number of kernel implementation details. For more information +about AF_ALG, see :ref:`Documentation/crypto/userspace-if.rst +`. + +Starting in Linux v7.3, AF_ALG supports only a limited set of +algorithms by default. This sysctl allows the system administrator to +remove this restriction when needed for compatibility reasons, or to +go further and disable AF_ALG entirely. The default value is 1. + +=== ================================================================== +0 AF_ALG is unrestricted. + +1 AF_ALG is supported with a limited list of algorithms. The list + is designed for compatibility with known users such as iwd and + bluez that haven't yet been fixed to use userspace crypto code. + + Specifically, there is an allowlist for unprivileged processes + and a somewhat longer allowlist for processes that hold + CAP_SYS_ADMIN or CAP_NET_ADMIN in the initial user namespace. + + Attempts to bind() an AF_ALG socket with a disallowed algorithm + fail with ENOENT. + +2 AF_ALG is completely disabled. Attempts to create an AF_ALG + socket fail with EAFNOSUPPORT. +=== ================================================================== + fips_enabled ============ diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index b9b0c218bfb4..22cc54cac3b2 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -67,6 +67,7 @@ Currently, these files are in /proc/sys/vm: - page-cluster - page_lock_unfairness - panic_on_oom +- panic_on_unrecoverable_memory_failure - percpu_pagelist_high_fraction - stat_interval - stat_refresh @@ -925,6 +926,85 @@ panic_on_oom=2+kdump gives you very strong tool to investigate why oom happens. You can get snapshot. +panic_on_unrecoverable_memory_failure +====================================== + +When a hardware memory error (e.g. multi-bit ECC) hits a kernel page +that cannot be recovered by the memory failure handler, the default +behaviour is to ignore the error and continue operation. This is +dangerous because the corrupted data remains accessible to the kernel, +risking silent data corruption or a delayed crash when the poisoned +memory is next accessed. + +When enabled, this sysctl triggers a panic on memory failure events +hitting kernel-owned pages that the handler cannot recover: +``PageReserved`` (firmware reservations, kernel image, vDSO, zero +page, and similar memblock-reserved regions), ``PageSlab``, +``PageTable``, and ``PageLargeKmalloc``. These are owned by the +kernel and the memory failure handler cannot reliably evict their +contents. + +Other unrecoverable kernel-owned populations (vmalloc allocations, +kernel stack pages, ...) are not currently covered because the +handler has no page-type signal that distinguishes them from a +userspace folio temporarily off the LRU during migration or +compaction. Such pages still go through the standard +MF_MSG_GET_HWPOISON path: ``PG_hwpoison`` is set on them and a +delayed crash on the next access remains possible. Coverage may +grow as the handler gains stronger kernel-ownership signals. + +Recoverable failure paths are also intentionally left out: in-flight +buddy allocations and other transient races with the page allocator +can reach the same diagnostic, and panicking on them would risk +killing the box for a page destined for userspace where the standard +SIGBUS recovery path applies. Pages whose state could not be +classified at all are not covered either, since an unknown state is +not a sound basis for a panic decision. + +For many environments it is preferable to panic immediately with a clean +crash dump that captures the original error context, rather than to +continue and face a random crash later whose cause is difficult to +diagnose. + +Use cases +--------- + +This option is most useful in environments where unattributed crashes +are expensive to debug or where data integrity must take precedence +over availability: + +* Large fleets, where multi-bit ECC errors on kernel pages are observed + regularly and post-mortem analysis of an unrelated downstream crash + (often seconds to minutes after the original error) consumes + significant engineering effort. + +* Systems configured with kdump, where panicking at the moment of the + hardware error produces a vmcore that still contains the faulting + address, the affected page state, and the originating MCE/GHES + record — context that is typically lost by the time a delayed crash + occurs. + +* High-availability clusters that rely on fast, deterministic node + failure for failover, and prefer an immediate panic over silent data + corruption propagating to replicas or persistent storage. + +* Kernel and platform developers reproducing hwpoison issues with + tools such as ``mce-inject`` or error-injection debugfs interfaces, + where panicking on the unrecoverable path makes regressions + immediately visible instead of surfacing as later, unrelated + failures. + += ===================================================================== +0 Try to continue operation (default). +1 Panic immediately. If the ``panic`` sysctl is also non-zero then the + machine will be rebooted. += ===================================================================== + +Example:: + + echo 1 > /proc/sys/vm/panic_on_unrecoverable_memory_failure + + percpu_pagelist_high_fraction ============================= diff --git a/Documentation/admin-guide/thunderbolt.rst b/Documentation/admin-guide/thunderbolt.rst index 89df26553aa0..91a6cb109988 100644 --- a/Documentation/admin-guide/thunderbolt.rst +++ b/Documentation/admin-guide/thunderbolt.rst @@ -373,6 +373,67 @@ port which are named like ``thunderbolt0`` and so on. From this point you can either use standard userspace tools like ``ip`` to configure the interface or let your GUI handle it automatically. +Streaming data directly over Thunderbolt cable +---------------------------------------------- +In addition to Thunderbolt networking (aka. USB4NET) Linux supports +streaming data directly over a cable as well (aka. USB4STREAM). This is +possible through ``thunderbolt-stream`` driver. + +Similarly to ``thunderbolt-net`` you load the driver first on one end:: + + host1 # modprobe thunderbolt-stream + +Then you configure it via ``ConfigFS``:: + + host1 # cd /sys/kernel/config/thunderbolt/stream + host1 # mkdir -p 0-1.0/data + host1 # cd 0-1.0 + host1 # echo -1 > data/in_hopid + host1 # echo -1 > data/out_hopid + +This information is automatically announced to the other side via +XDomain properties so if you have cable connected the other side knows +that there is a stream named ``data`` available and can configure it for +you automatically:: + + host2 # cd /sys/kernel/config/thunderbolt/stream + host2 # mkdir -p 0-3.0/data + +Here we used auto-configuration but you can configure it manually too. +In that case you need to fill ``in_hopid`` and ``out_hopid`` accordingly. +If you set them to ``-1`` the next available HopID is used which is +typically what we want. + +Once they are configured you can use ``/dev/tbstreamX`` on both sides to +transfer data:: + + host2 # cat /dev/tbstream0 + host1 # dmesg > /dev/tbstream0 + +Once you are done with the stream you can remove them:: + + host2 # cd /sys/kernel/config/thunderbolt/stream + host2 # rmdir -p 0-1.0/data + host1 # cd /sys/kernel/config/thunderbolt/stream + host1 # rmdir -p 0-3.0/data + +Since streams are essentially files you can use any existing application +that supports ``read(2)`` and ``write(2)`` in some form. + +It is possible to have more than one stream and you can have both stream +and ``thunderbolt-net`` in use simultaneously. For example we can create +two streams with name ``control`` and ``data`` like this:: + + host1 # cd /sys/kernel/config/thunderbolt/stream + host1 # mkdir 0-1.0 + host1 # cd 0-1.0 + host1 # mkdir control + host1 # mkdir data + +Then you have ``/dev/tbstream0`` for ``control`` and ``/dev/tbstream1`` +for ``data``. Before you can use them you need to configure them as +shown above for the one stream case. + Forcing power ------------- Many OEMs include a method that can be used to force the power of a diff --git a/Documentation/arch/arm64/cpu-hotplug.rst b/Documentation/arch/arm64/cpu-hotplug.rst index 8fb438bf7781..7c3379b704aa 100644 --- a/Documentation/arch/arm64/cpu-hotplug.rst +++ b/Documentation/arch/arm64/cpu-hotplug.rst @@ -47,11 +47,12 @@ ever have can be described at boot. There are no power-domain considerations as such devices are emulated. CPU Hotplug on virtual systems is supported. It is distinct from physical -CPU Hotplug as all resources are described as ``present``, but CPUs may be -marked as disabled by firmware. Only the CPU's online/offline behaviour is -influenced by firmware. An example is where a virtual machine boots with a -single CPU, and additional CPUs are added once a cloud orchestrator deploys -the workload. +CPU Hotplug as all vCPU resources are statically described in the firmware +configuration tables (e.g. MADT), meaning their maximum possible count is +known at boot. However, vCPUs that are not enabled at boot are not marked +as ``present`` by the kernel until they are hotplugged. An example is where +a virtual machine boots with a single CPU, and additional CPUs are added +once a cloud orchestrator deploys the workload. For a virtual machine, the VMM (e.g. Qemu) plays the part of firmware. @@ -60,16 +61,19 @@ brought online. Firmware can enforce its policy via PSCI's return codes. e.g. ``DENIED``. The ACPI tables must describe all the resources of the virtual machine. CPUs -that firmware wishes to disable either from boot (or later) should not be -``enabled`` in the MADT GICC structures, but should have the ``online capable`` -bit set, to indicate they can be enabled later. The boot CPU must be marked as -``enabled``. The 'always on' GICR structure must be used to describe the -redistributors. +that are hot-pluggable must have the ``online capable`` bit set and the +``enabled`` bit cleared in the MADT GICC structures to indicate they can be +enabled later. The boot CPU must be marked as ``enabled`` with its +``online capable`` bit cleared. The 'always on' GICR structure must be used +to describe the redistributors. CPUs described as ``online capable`` but not ``enabled`` can be set to enabled by the DSDT's Processor object's _STA method. On virtual systems the _STA method -must always report the CPU as ``present``. Changes to the firmware policy can -be notified to the OS via device-check or eject-request. +must always set the ``ACPI_STA_DEVICE_PRESENT`` bit, while toggling the +``ACPI_STA_DEVICE_ENABLED`` bit to reflect its plug status. The kernel will +then dynamically mark the vCPU as ``present`` within the OS when the +``ACPI_STA_DEVICE_ENABLED`` bit becomes set during hot-add. Changes to the +firmware policy can be notified to the OS via device-check or eject-request. CPUs described as ``enabled`` in the static table, should not have their _STA modified dynamically by firmware. Soft-restart features such as kexec will diff --git a/Documentation/arch/x86/amd-memory-encryption.rst b/Documentation/arch/x86/amd-memory-encryption.rst index bd840df708ea..92edb26a50d2 100644 --- a/Documentation/arch/x86/amd-memory-encryption.rst +++ b/Documentation/arch/x86/amd-memory-encryption.rst @@ -53,7 +53,7 @@ CPUID function 0x8000001f reports information related to SME:: system physical addresses, not guest physical addresses) -If support for SME is present, MSR 0xc00100010 (MSR_AMD64_SYSCFG) can be used to +If support for SME is present, MSR 0xc0010010 (MSR_AMD64_SYSCFG) can be used to determine if SME is enabled and/or to enable memory encryption:: 0xc0010010: diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst index 11126ed6f40f..b9dc0c9dbee4 100644 --- a/Documentation/block/biovecs.rst +++ b/Documentation/block/biovecs.rst @@ -135,6 +135,7 @@ Usage of helpers: bio_first_bvec_all() bio_first_page_all() bio_first_folio_all() + bio_last_bvec_all() * The following helpers iterate over single-page segment. The passed 'struct bio_vec' will contain a single-page IO vector during the iteration:: diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 4c814ff6061e..c801a330aece 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -273,22 +273,29 @@ flags on a set of kfuncs as follows:: BTF_KFUNCS_END(bpf_task_set) This set encodes the BTF ID of each kfunc listed above, and encodes the flags -along with it. Ofcourse, it is also allowed to specify no flags. +along with it. It is also allowed to specify no flags. kfunc definitions should also always be annotated with the ``__bpf_kfunc`` -macro. This prevents issues such as the compiler inlining the kfunc if it's a -static kernel function, or the function being elided in an LTO build as it's -not used in the rest of the kernel. Developers should not manually add -annotations to their kfunc to prevent these issues. If an annotation is -required to prevent such an issue with your kfunc, it is a bug and should be -added to the definition of the macro so that other kfuncs are similarly -protected. An example is given below:: +macro. This prevents issues such as the compiler inlining the kfunc, or the +function being elided in an LTO build as it's not used in the rest of the +kernel. Developers should not manually add annotations to their kfunc to prevent +these issues. If an annotation is required to prevent such an issue with your +kfunc, it is a bug and should be added to the definition of the macro so that +other kfuncs are similarly protected. An example is given below:: __bpf_kfunc struct task_struct *bpf_get_task_pid(s32 pid) { ... } +Note that kfuncs must not be declared ``static``. A kfunc can be called from a +BPF program ``*.c`` file outside the compilation unit that defines it, so its +externally visible name must remain available for BTF ID lookup. ``static`` +linkage allows the compiler to rename the function, which can break this +BTF-based kfunc resolution. Further note that sparse may warn that an otherwise +unreferenced kfunc should be static. Such warnings should be ignored for kfunc +definitions. + 2.5.1 KF_ACQUIRE flag --------------------- diff --git a/Documentation/conf.py b/Documentation/conf.py index 9b822ab470d9..3f82cfbff778 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -61,12 +61,12 @@ def config_init(app, config): """ - Initialize path-dependent variabled + Initialize path-dependent variables On Sphinx, all directories are relative to what it is passed as SOURCEDIR parameter for sphinx-build. Due to that, all patterns that have directory names on it need to be dynamically set, after - converting them to a relative patch. + converting them to a relative path. As Sphinx doesn't include any patterns outside SOURCEDIR, we should exclude relative patterns that start with "../". diff --git a/Documentation/core-api/SMP.rst b/Documentation/core-api/SMP.rst new file mode 100644 index 000000000000..0265a9835f23 --- /dev/null +++ b/Documentation/core-api/SMP.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +============== +SMP primitives +============== + +.. kernel-doc:: include/linux/smp.h + :internal: + +.. kernel-doc:: kernel/smp.c + :export: diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index 13769d5c40bf..92f91c6a0d79 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -81,6 +81,7 @@ Documentation/locking/index.rst for more related documentation. padata ../RCU/index wrappers/memory-barriers.rst + SMP Low-level hardware management ============================= diff --git a/Documentation/core-api/kho/abi.rst b/Documentation/core-api/kho/abi.rst index 799d743105a6..edeb5b311963 100644 --- a/Documentation/core-api/kho/abi.rst +++ b/Documentation/core-api/kho/abi.rst @@ -28,6 +28,11 @@ KHO persistent memory tracker ABI .. kernel-doc:: include/linux/kho/abi/kexec_handover.h :doc: KHO persistent memory tracker +KHO serialization block ABI +=========================== + +.. kernel-doc:: include/linux/kho/abi/block.h + See Also ======== diff --git a/Documentation/core-api/kho/index.rst b/Documentation/core-api/kho/index.rst index 0a2dee4f8e7d..320914a42178 100644 --- a/Documentation/core-api/kho/index.rst +++ b/Documentation/core-api/kho/index.rst @@ -83,6 +83,17 @@ Public API .. kernel-doc:: kernel/liveupdate/kexec_handover.c :export: +KHO Serialization Blocks API +============================ + +.. kernel-doc:: kernel/liveupdate/kho_block.c + :doc: KHO Serialization Blocks + +.. kernel-doc:: include/linux/kho_block.h + +.. kernel-doc:: kernel/liveupdate/kho_block.c + :internal: + See Also ======== diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst index ccdd1615cf97..1eae24a9c18b 100644 --- a/Documentation/core-api/maple_tree.rst +++ b/Documentation/core-api/maple_tree.rst @@ -17,7 +17,8 @@ supports iterating over a range of entries and going to the previous or next entry in a cache-efficient manner. The tree can also be put into an RCU-safe mode of operation which allows reading and writing concurrently. Writers must synchronize on a lock, which can be the default spinlock, or the user can set -the lock to an external lock of a different type. +the lock to an external lock of a different type. Note that external locks may +interfere with allocations in a low memory situation. The Maple Tree maintains a small memory footprint and was designed to use modern processor cache efficiently. The majority of the users will be able to @@ -42,6 +43,15 @@ successful store operation within a given code segment when allocating cannot be done. Allocations of nodes are relatively small at around 256 bytes. +Since the maple tree uses internal nodes that are allocated and has rules on +data density, erasing an entry may cause allocations to occur. That is, +erasing an entry may consume memory. Users must take care to ensure that they +do not violate the larger system constraints on when and how memory is +allocated. Most situations are fine to allocate, but the pre-allocation +support is provided as a mechanism to avoid trickier situations. There is also +the possibility of using special entries and clean up the tree later, in +extreme circumstances. + .. _maple-tree-normal-api: Normal API @@ -63,7 +73,8 @@ success or an error code otherwise. mtree_store_range() works in the same way but takes a range. mtree_load() is used to retrieve the entry stored at a given index. You can use mtree_erase() to erase an entire range by only knowing one value within that range, or mtree_store() call with an entry of -NULL may be used to partially erase a range or many ranges at once. +NULL may be used to partially erase a range or many ranges at once. Note that +mtree_erase() may use GFP_KERNEL on allocations. If you want to only store a new entry to a range (or index) if that range is currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which @@ -163,7 +174,9 @@ You can use mas_erase() to erase an entire range by setting index and last of the maple state to the desired range to erase. This will erase the first range that is found in that range, set the maple state index and last as the range that was erased and return the entry that existed -at that location. +at that location. Note that mas_erase() may allocate with the GFP_KERNEL flag. +If this is not okay, consider using mas_store_gfp() and pass it a ``NULL``, +after setting up the correct range by walking to the entry. You can walk each entry within a range by using mas_for_each(). If you want to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as @@ -211,7 +224,7 @@ Advanced Locking The maple tree uses a spinlock by default, but external locks can be used for tree updates as well. To use an external lock, the tree must be initialized -with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the +with the ``MT_FLAGS_LOCK_EXTERN`` flag, this is usually done with the MTREE_INIT_EXT() #define, which takes an external lock as an argument. Functions and structures diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index aabdd3cba58e..4df7d5edbee5 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -73,6 +73,7 @@ Readahead Writeback --------- +.. kernel-doc:: include/linux/writeback.h .. kernel-doc:: mm/page-writeback.c :export: diff --git a/Documentation/crypto/async-tx-api.rst b/Documentation/crypto/async-tx-api.rst index f88a7809385e..49fcfc66314a 100644 --- a/Documentation/crypto/async-tx-api.rst +++ b/Documentation/crypto/async-tx-api.rst @@ -82,9 +82,9 @@ xor_val xor a series of source buffers and set a flag if the pq generate the p+q (raid6 syndrome) from a series of source buffers pq_val validate that a p and or q buffer are in sync with a given series of sources -datap (raid6_datap_recov) recover a raid6 data block and the p block +datap (raid6_recov_datap) recover a raid6 data block and the p block from the given sources -2data (raid6_2data_recov) recover 2 raid6 data blocks from the given +2data (raid6_recov_2data) recover 2 raid6 data blocks from the given sources ======== ==================================================================== diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst index ab93300c8e04..d6194346e366 100644 --- a/Documentation/crypto/userspace-if.rst +++ b/Documentation/crypto/userspace-if.rst @@ -1,3 +1,5 @@ +.. _crypto_userspace_interface: + User Space Interface ==================== @@ -12,9 +14,14 @@ AF_ALG is insecure and is deprecated. Originally added to the kernel in 2010, most kernel developers now consider it to be a mistake. Support for hardware accelerators, which was the original purpose of AF_ALG, has been removed. -AF_ALG continues to be supported only for backwards compatibility. On systems -where no programs using AF_ALG remain, the support for it should be disabled by -disabling ``CONFIG_CRYPTO_USER_API_*``. +AF_ALG continues to be supported only for backwards compatibility. + +Starting in Linux v7.3, the set of algorithms supported by AF_ALG is limited by +default. See :ref:`/proc/sys/crypto/af_alg_restrict `. + +On systems where no programs using AF_ALG remain, the support for it should be +disabled entirely by setting ``/proc/sys/crypto/af_alg_restrict`` to 2 or by +disabling ``CONFIG_CRYPTO_USER_API_*`` in the kernel configuration. Deprecation ----------- diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index dccede68698c..6139a08c34cd 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -184,6 +184,13 @@ Available options: Override checking of perl version. Runtime errors may be encountered after enabling this flag if the perl version does not meet the minimum specified. + - --spdx-cxx-comments + + Don't force C comments ``/* */`` for SPDX license (required by old + toolchains), allow also C++ comments ``//``. + + NOTE: it should *not* be used for Linux mainline. + - --codespell Use the codespell dictionary for checking spelling errors. @@ -210,6 +217,13 @@ Available options: Display the help text. +Configuration file +================== + +Default configuration options can be stored in ``.checkpatch.conf``, search +path: ``.:$HOME:.scripts`` or in a directory specified by ``$CHECKPATCH_CONFIG_DIR`` +environment variable (falling back to the default search path). + Message Levels ============== diff --git a/Documentation/dev-tools/container.rst b/Documentation/dev-tools/container.rst index 452415b64662..9e23f79d5ae1 100644 --- a/Documentation/dev-tools/container.rst +++ b/Documentation/dev-tools/container.rst @@ -40,7 +40,7 @@ Available options: ``-r, --runtime RUNTIME`` - Container runtime name. Supported runtimes: ``docker``, ``podman``. + Container runtime name. Supported runtimes: ``podman``, ``docker``. If not specified, the first one found on the system will be used i.e. Podman if present, otherwise Docker. @@ -75,8 +75,8 @@ working directory and adjust the user and group id as needed. The container image which would typically include a compiler toolchain is provided by the user and selected via the ``-i`` option. The container runtime -can be selected with the ``-r`` option, which can be either ``docker`` or -``podman``. If none is specified, the first one found on the system will be +can be selected with the ``-r`` option, which can be either ``podman`` or +``docker``. If none is specified, the first one found on the system will be used while giving priority to Podman. Support for other runtimes may be added later depending on their popularity among users. diff --git a/Documentation/dev-tools/kcov.rst b/Documentation/dev-tools/kcov.rst index 8127849d40f5..1a739290c8ec 100644 --- a/Documentation/dev-tools/kcov.rst +++ b/Documentation/dev-tools/kcov.rst @@ -237,6 +237,9 @@ Both ``kcov_remote_start`` and ``kcov_remote_stop`` annotations and the collection sections. The way a handle is used depends on the context where the matching code section executes. +A thread can use two separate KCOV instances to collect remote coverage and +normal coverage at the same time. + KCOV supports collecting remote coverage from the following contexts: 1. Global kernel background tasks. These are the tasks that are spawned during @@ -262,6 +265,9 @@ gets saved to the ``kcov_handle`` field in the current ``task_struct`` and needs to be passed to the newly spawned local tasks via custom kernel code modifications. Those tasks should in turn use the passed handle in their ``kcov_remote_start`` and ``kcov_remote_stop`` annotations. +In the kernel, common handles are wrapped in a ``kcov_common_handle_id``, which +consumes no space in builds without ``CONFIG_KCOV``; subsystems that integrate +with this mechanism should not need to use any ``#ifdef CONFIG_KCOV`` or such. KCOV follows a predefined format for both global and common handles. Each handle is a ``u64`` integer. Currently, only the one top and the lower 4 bytes diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst index d7bfe320338c..64c0ec7428a2 100644 --- a/Documentation/dev-tools/kselftest.rst +++ b/Documentation/dev-tools/kselftest.rst @@ -15,11 +15,6 @@ able to run that test on an older kernel. Hence, it is important to keep code that can still test an older kernel and make sure it skips the test gracefully on newer releases. -You can find additional information on Kselftest framework, how to -write new tests using the framework on Kselftest wiki: - -https://kselftest.wiki.kernel.org/ - On some systems, hot-plug tests could hang forever waiting for cpu and memory to be ready to be offlined. A special hot-plug target is created to run the full range of hot-plug tests. In default mode, hot-plug tests run diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 00149e824261..6b4b4f51c371 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -6,7 +6,7 @@ DT_MK_SCHEMA ?= dt-mk-schema DT_SCHEMA_LINT = $(shell which yamllint || \ echo "warning: python package 'yamllint' not installed, skipping" >&2) -DT_SCHEMA_MIN_VERSION = 2023.9 +DT_SCHEMA_MIN_VERSION = 2024.4 PHONY += check_dtschema_version check_dtschema_version: diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml index 0b418d9d60db..3030cf46fe74 100644 --- a/Documentation/devicetree/bindings/arm/altera.yaml +++ b/Documentation/devicetree/bindings/arm/altera.yaml @@ -116,6 +116,12 @@ properties: - intel,socfpga-agilex5-socdk-nand - const: intel,socfpga-agilex5 + - description: Agilex72 boards + items: + - enum: + - intel,socfpga-agilex72-socdk + - const: intel,socfpga-agilex72 + - description: Agilex7m boards items: - enum: diff --git a/Documentation/devicetree/bindings/arm/amlogic.yaml b/Documentation/devicetree/bindings/arm/amlogic.yaml index a885278bc4e2..9f73a0054fb2 100644 --- a/Documentation/devicetree/bindings/arm/amlogic.yaml +++ b/Documentation/devicetree/bindings/arm/amlogic.yaml @@ -234,6 +234,12 @@ properties: - amlogic,av400 - const: amlogic,a5 + - description: Boards with the Amlogic A9 A311Y3 SoC + items: + - enum: + - amlogic,by401 + - const: amlogic,a9 + - description: Boards with the Amlogic C3 C302X/C308L SoC items: - enum: diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml index 86876311ec59..6237fb2ae50e 100644 --- a/Documentation/devicetree/bindings/arm/fsl.yaml +++ b/Documentation/devicetree/bindings/arm/fsl.yaml @@ -1310,6 +1310,18 @@ properties: - const: tq,imx8mp-tqma8mpql # TQ-Systems GmbH i.MX8MP TQMa8MPQL SOM - const: fsl,imx8mp + - description: + TQMa8MPxS is a series of SOM featuring NXP i.MX8MP system-on-chip + variants. It has the SMARC-2.0 form factor and is designed to be + placed on different carrier boards. All CPU variants use the same + device tree hence only one compatible is needed. MB-SMARC-2 is a + carrier reference design. + items: + - enum: + - tq,imx8mp-tqma8mpqs-mb-smarc-2 # TQ-Systems GmbH i.MX8MP TQMa8MPQS SOM on MB-SMARC-2 + - const: tq,imx8mp-tqma8mpqs # TQ-Systems GmbH i.MX8MP TQMa8MPQS SOM + - const: fsl,imx8mp + - description: Variscite VAR-SOM-MX8M Plus based boards items: - const: variscite,var-som-mx8mp-symphony diff --git a/Documentation/devicetree/bindings/arm/qcom-soc.yaml b/Documentation/devicetree/bindings/arm/qcom-soc.yaml index 27261039d56f..37fdd5a080b7 100644 --- a/Documentation/devicetree/bindings/arm/qcom-soc.yaml +++ b/Documentation/devicetree/bindings/arm/qcom-soc.yaml @@ -11,10 +11,10 @@ maintainers: description: | Guidelines for new compatibles for SoC blocks/components. - When adding new compatibles in new bindings, use the format:: + When adding new compatibles in new bindings, use the format: qcom,SoC-IP - For example:: + For example: qcom,sdm845-llcc-bwmon When adding new compatibles to existing bindings, use the format in the diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml index 50cc18a6ec5e..3dada073a992 100644 --- a/Documentation/devicetree/bindings/arm/qcom.yaml +++ b/Documentation/devicetree/bindings/arm/qcom.yaml @@ -49,18 +49,31 @@ properties: - qcom,apq8074-dragonboard - const: qcom,apq8074 - - items: - - enum: - - qcom,apq8060-dragonboard - - qcom,msm8660-surf - - const: qcom,msm8660 - - items: - enum: - qcom,apq8084-mtp - qcom,apq8084-sbc - const: qcom,apq8084 + - items: + - enum: + - sony,karin_windy + - const: qcom,apq8094 + + - items: + - enum: + - arrow,apq8096-db820c + - inforce,ifc6640 + - const: qcom,apq8096-sbc + - const: qcom,apq8096 + + - items: + - const: arrow,apq8096sg-db820c + - const: arrow,apq8096-db820c + - const: qcom,apq8096-sbc + - const: qcom,apq8096sg + - const: qcom,apq8096 + - items: - enum: - qcom,eliza-mtp @@ -73,102 +86,126 @@ properties: - items: - enum: - - qcom,mahua-crd - - const: qcom,mahua + - 8dev,jalapeno + - alfa-network,ap120c-ac + - const: qcom,ipq4018 - items: - enum: - - fairphone,fp6 - - nothing,asteroids - - const: qcom,milos + - qcom,ipq4019-ap-dk01.1-c1 + - qcom,ipq4019-ap-dk04.1-c3 + - qcom,ipq4019-ap-dk07.1-c1 + - qcom,ipq4019-ap-dk07.1-c2 + - qcom,ipq4019-dk04.1-c1 + - const: qcom,ipq4019 - items: - enum: - - microsoft,dempsey - - microsoft,makepeace - - microsoft,moneypenny - - motorola,falcon - - samsung,ms013g - - samsung,s3ve3g - - const: qcom,msm8226 + - qcom,ipq5018-rdp432-c2 + - tplink,archer-ax55-v1 + - const: qcom,ipq5018 - items: - enum: - - htc,memul - - microsoft,superman-lte - - microsoft,tesla - - motorola,peregrine - - samsung,matisselte - - const: qcom,msm8926 - - const: qcom,msm8226 + - qcom,ipq5210-rdp504 + - const: qcom,ipq5210 - items: - enum: - - wingtech,wt82918hd - - const: qcom,msm8929 + - qcom,ipq5332-ap-mi01.2 + - qcom,ipq5332-ap-mi01.3 + - qcom,ipq5332-ap-mi01.6 + - qcom,ipq5332-ap-mi01.9 + - const: qcom,ipq5332 - items: - enum: - - asus,z00t - - huawei,kiwi - - longcheer,l9100 - - samsung,a7 - - sony,kanuti-tulip - - square,apq8039-t2 - - wingtech,wt82918 - - wingtech,wt82918hdhw39 - - const: qcom,msm8939 + - qcom,ipq5424-rdp466 + - const: qcom,ipq5424 - items: - enum: - - sony,kugo-row - - sony,suzu-row - - const: qcom,msm8956 + - qcom,ipq6018-cp01 + - qcom,ipq6018-cp01-c1 + - const: qcom,ipq6018 - items: - enum: - - qcom,msm8960-cdp - - samsung,expressatt - - const: qcom,msm8960 + - mikrotik,rb3011 + - qcom,ipq8064-ap148 + - const: qcom,ipq8064 - items: - enum: - - sony,huashan - - const: qcom,msm8960t - - const: qcom,msm8960 + - qcom,ipq8074-hk01 + - qcom,ipq8074-hk10-c1 + - qcom,ipq8074-hk10-c2 + - const: qcom,ipq8074 - items: - enum: - - lge,hammerhead - - samsung,hlte - - sony,xperia-amami - - sony,xperia-honami - - sony,xperia-togari - - const: qcom,msm8974 + - qcom,ipq9574-ap-al02-c2 + - qcom,ipq9574-ap-al02-c2-emmc + - qcom,ipq9574-ap-al02-c6 + - qcom,ipq9574-ap-al02-c7 + - qcom,ipq9574-ap-al02-c7-emmc + - qcom,ipq9574-ap-al02-c8 + - qcom,ipq9574-ap-al02-c9 + - const: qcom,ipq9574 - items: - enum: - - fairphone,fp2 - - htc,m8 - - oneplus,bacon - - samsung,klte - - sony,xperia-aries - - sony,xperia-castor - - sony,xperia-leo - - const: qcom,msm8974pro - - const: qcom,msm8974 + - qcom,ipq9650-rdp488 + - const: qcom,ipq9650 - items: - enum: - - samsung,kltechn - - const: samsung,klte - - const: qcom,msm8974pro - - const: qcom,msm8974 + - qcom,kaanapali-mtp + - qcom,kaanapali-qrd + - const: qcom,kaanapali - items: - enum: - - longcheer,l9360 - - const: qcom,msm8976 + - qcom,mahua-crd + - const: qcom,mahua + + - description: Sierra Wireless MangOH Green with WP8548 Module + items: + - const: swir,mangoh-green-wp8548 + - const: swir,wp8548 + - const: qcom,mdm9615 + + - items: + - enum: + - fairphone,fp6 + - nothing,asteroids + - const: qcom,milos + + - items: + - enum: + - microsoft,dempsey + - microsoft,makepeace + - microsoft,moneypenny + - motorola,falcon + - samsung,ms013g + - samsung,s3ve3g + - const: qcom,msm8226 + + - items: + - enum: + - htc,memul + - microsoft,superman-lte + - microsoft,tesla + - motorola,peregrine + - samsung,matisselte + - const: qcom,msm8926 + - const: qcom,msm8226 + + - items: + - enum: + - qcom,apq8060-dragonboard + - qcom,msm8660-surf + - const: qcom,msm8660 - items: - enum: @@ -217,11 +254,28 @@ properties: - xiaomi,tiare - const: qcom,msm8917 + - items: + - enum: + - wingtech,wt82918hd + - const: qcom,msm8929 + - items: - enum: - xiaomi,land - const: qcom,msm8937 + - items: + - enum: + - asus,z00t + - huawei,kiwi + - longcheer,l9100 + - samsung,a7 + - sony,kanuti-tulip + - square,apq8039-t2 + - wingtech,wt82918 + - wingtech,wt82918hdhw39 + - const: qcom,msm8939 + - items: - enum: - flipkart,rimob @@ -232,6 +286,57 @@ properties: - xiaomi,vince - const: qcom,msm8953 + - items: + - enum: + - sony,kugo-row + - sony,suzu-row + - const: qcom,msm8956 + + - items: + - enum: + - qcom,msm8960-cdp + - samsung,expressatt + - const: qcom,msm8960 + + - items: + - enum: + - sony,huashan + - const: qcom,msm8960t + - const: qcom,msm8960 + + - items: + - enum: + - lge,hammerhead + - samsung,hlte + - sony,xperia-amami + - sony,xperia-honami + - sony,xperia-togari + - const: qcom,msm8974 + + - items: + - enum: + - fairphone,fp2 + - htc,m8 + - oneplus,bacon + - samsung,klte + - sony,xperia-aries + - sony,xperia-castor + - sony,xperia-leo + - const: qcom,msm8974pro + - const: qcom,msm8974 + + - items: + - enum: + - samsung,kltechn + - const: samsung,klte + - const: qcom,msm8974pro + - const: qcom,msm8974 + + - items: + - enum: + - longcheer,l9360 + - const: qcom,msm8976 + - items: - enum: - lg,bullhead @@ -240,11 +345,6 @@ properties: - xiaomi,libra - const: qcom,msm8992 - - items: - - enum: - - sony,karin_windy - - const: qcom,apq8094 - - items: - enum: - huawei,angler @@ -256,20 +356,6 @@ properties: - sony,suzuran-row - const: qcom,msm8994 - - items: - - enum: - - arrow,apq8096-db820c - - inforce,ifc6640 - - const: qcom,apq8096-sbc - - const: qcom,apq8096 - - - items: - - const: arrow,apq8096sg-db820c - - const: arrow,apq8096-db820c - - const: qcom,apq8096-sbc - - const: qcom,apq8096sg - - const: qcom,apq8096 - - items: - enum: - oneplus,oneplus3 @@ -305,83 +391,9 @@ properties: - items: - enum: - - 8dev,jalapeno - - alfa-network,ap120c-ac - - const: qcom,ipq4018 - - - items: - - enum: - - qcom,ipq4019-ap-dk01.1-c1 - - qcom,ipq4019-ap-dk04.1-c3 - - qcom,ipq4019-ap-dk07.1-c1 - - qcom,ipq4019-ap-dk07.1-c2 - - qcom,ipq4019-dk04.1-c1 - - const: qcom,ipq4019 - - - items: - - enum: - - qcom,ipq5018-rdp432-c2 - - tplink,archer-ax55-v1 - - const: qcom,ipq5018 - - - items: - - enum: - - qcom,ipq5210-rdp504 - - const: qcom,ipq5210 - - - items: - - enum: - - qcom,ipq5332-ap-mi01.2 - - qcom,ipq5332-ap-mi01.3 - - qcom,ipq5332-ap-mi01.6 - - qcom,ipq5332-ap-mi01.9 - - const: qcom,ipq5332 - - - items: - - enum: - - qcom,ipq5424-rdp466 - - const: qcom,ipq5424 - - - items: - - enum: - - mikrotik,rb3011 - - qcom,ipq8064-ap148 - - const: qcom,ipq8064 - - - items: - - enum: - - qcom,ipq8074-hk01 - - qcom,ipq8074-hk10-c1 - - qcom,ipq8074-hk10-c2 - - const: qcom,ipq8074 - - - items: - - enum: - - qcom,ipq9574-ap-al02-c2 - - qcom,ipq9574-ap-al02-c2-emmc - - qcom,ipq9574-ap-al02-c6 - - qcom,ipq9574-ap-al02-c7 - - qcom,ipq9574-ap-al02-c7-emmc - - qcom,ipq9574-ap-al02-c8 - - qcom,ipq9574-ap-al02-c9 - - const: qcom,ipq9574 - - - items: - - enum: - - qcom,ipq9650-rdp488 - - const: qcom,ipq9650 - - - items: - - enum: - - qcom,kaanapali-mtp - - qcom,kaanapali-qrd - - const: qcom,kaanapali - - - description: Sierra Wireless MangOH Green with WP8548 Module - items: - - const: swir,mangoh-green-wp8548 - - const: swir,wp8548 - - const: qcom,mdm9615 + - qcom,iq10-rrd + - qcom,sa8797p-ride + - const: qcom,nord - description: Qualcomm Technologies, Inc. Robotics RB1 items: @@ -403,6 +415,20 @@ properties: - thundercomm,rubikpi3 - const: qcom,qcm6490 + - items: + - enum: + - qcom,qcs404-evb-1000 + - qcom,qcs404-evb-4000 + - const: qcom,qcs404-evb + - const: qcom,qcs404 + + - items: + - enum: + - arduino,monza + - qcom,monaco-evk + - qcom,qcs8300-ride + - const: qcom,qcs8300 + - description: Qualcomm Technologies, Inc. Distributed Unit 1000 platform items: - enum: @@ -416,6 +442,31 @@ properties: - qcom,qru1000-idp - const: qcom,qru1000 + - items: + - enum: + - qcom,sa8155p-adp + - const: qcom,sa8155p + + - items: + - enum: + - qcom,sa8295p-adp + - qcom,sa8540p-ride + - const: qcom,sa8540p + + - items: + - enum: + - qcom,sa8775p-ride + - qcom,sa8775p-ride-r3 + - const: qcom,sa8775p + + - items: + - enum: + - qcom,lemans-evk + - qcom,qcs9100-ride + - qcom,qcs9100-ride-r3 + - const: qcom,qcs9100 + - const: qcom,sa8775p + - description: Qualcomm AR2 Gen1 platform items: - enum: @@ -828,14 +879,6 @@ properties: - const: google,zombie-sku514 - const: qcom,sc7280 - - description: Xiaomi Poco F1 - items: - - enum: - - xiaomi,beryllium-ebbg - - xiaomi,beryllium-tianma - - const: xiaomi,beryllium - - const: qcom,sdm845 - - items: - enum: - lenovo,flex-5g @@ -853,6 +896,11 @@ properties: - qcom,sc8280xp-qrd - const: qcom,sc8280xp + - items: + - enum: + - inforce,ifc6560 + - const: qcom,sda660 + - items: - enum: - lenovo,tbx605f @@ -867,11 +915,6 @@ properties: - sony,voyager-row - const: qcom,sdm630 - - items: - - enum: - - inforce,ifc6560 - - const: qcom,sda660 - - items: - enum: - fairphone,fp3 @@ -899,74 +942,13 @@ properties: - const: google,bonito - const: qcom,sdm670 - - items: - - enum: - - qcom,sdx55-mtp - - qcom,sdx55-telit-fn980-tlb - - qcom,sdx55-t55 - - const: qcom,sdx55 - - - items: - - enum: - - qcom,sdx65-mtp - - const: qcom,sdx65 - - - items: - - enum: - - qcom,sdx75-idp - - const: qcom,sdx75 - - - items: - - enum: - - qcom,ipq6018-cp01 - - qcom,ipq6018-cp01-c1 - - const: qcom,ipq6018 - - - items: - - enum: - - qcom,qcs404-evb-1000 - - qcom,qcs404-evb-4000 - - const: qcom,qcs404-evb - - const: qcom,qcs404 - - - items: - - enum: - - arduino,monza - - qcom,monaco-evk - - qcom,qcs8300-ride - - const: qcom,qcs8300 - - - items: - - enum: - - qcom,qcs615-ride - - qcom,talos-evk - - const: qcom,qcs615 - - const: qcom,sm6150 - - - items: - - enum: - - qcom,sa8155p-adp - - const: qcom,sa8155p - - - items: - - enum: - - qcom,sa8295p-adp - - qcom,sa8540p-ride - - const: qcom,sa8540p - - - items: - - enum: - - qcom,sa8775p-ride - - qcom,sa8775p-ride-r3 - - const: qcom,sa8775p - - - items: + - description: Xiaomi Poco F1 + items: - enum: - - qcom,lemans-evk - - qcom,qcs9100-ride - - qcom,qcs9100-ride-r3 - - const: qcom,qcs9100 - - const: qcom,sa8775p + - xiaomi,beryllium-ebbg + - xiaomi,beryllium-tianma + - const: xiaomi,beryllium + - const: qcom,sdm845 - items: - enum: @@ -989,6 +971,23 @@ properties: - xiaomi,polaris - const: qcom,sdm845 + - items: + - enum: + - qcom,sdx55-mtp + - qcom,sdx55-telit-fn980-tlb + - qcom,sdx55-t55 + - const: qcom,sdx55 + + - items: + - enum: + - qcom,sdx65-mtp + - const: qcom,sdx65 + + - items: + - enum: + - qcom,sdx75-idp + - const: qcom,sdx75 + - items: - enum: - oneplus,billie2 @@ -1024,6 +1023,13 @@ properties: - xiaomi,willow - const: qcom,sm6125 + - items: + - enum: + - qcom,qcs615-ride + - qcom,talos-evk + - const: qcom,qcs615 + - const: qcom,sm6150 + - items: - enum: - sony,pdx213 @@ -1112,6 +1118,12 @@ properties: - const: qcom,qcs8550 - const: qcom,sm8550 + - items: + - enum: + - qcom,qcs8550-rb5gen2 + - const: qcom,qcs8550 + - const: qcom,sm8550 + - items: - enum: - ayaneo,pocket-s2 @@ -1215,7 +1227,7 @@ properties: items: items: - description: | - MSM chipset ID - an exact match value consisting of two bitfields:: + MSM chipset ID - an exact match value consisting of two bitfields: - bits 0-15 - The unique MSM chipset ID - bits 16-31 - Reserved; should be 0 - description: | @@ -1241,7 +1253,7 @@ properties: - items: - items: - description: | - Board ID consisting of three bitfields:: + Board ID consisting of three bitfields: - bits 31-24 - Unused - bits 23-16 - Platform Version Major - bits 15-8 - Platform Version Minor diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml index 1a9dde18626d..aa66a15be233 100644 --- a/Documentation/devicetree/bindings/arm/rockchip.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip.yaml @@ -724,6 +724,11 @@ properties: - const: hardkernel,odroid-m2 - const: rockchip,rk3588s + - description: HINLINK H28K + items: + - const: hinlink,h28k + - const: rockchip,rk3528 + - description: HINLINK H66K / H68K items: - enum: @@ -1306,6 +1311,16 @@ properties: - const: turing,rk1 - const: rockchip,rk3588 + - description: Vicharak Axon + items: + - const: vicharak,axon + - const: rockchip,rk3588 + + - description: Vicharak Vaaman2 + items: + - const: vicharak,vaaman2 + - const: rockchip,rk3588 + - description: WolfVision PF5 mainboard items: - const: wolfvision,rk3568-pf5 @@ -1347,6 +1362,11 @@ properties: - const: xunlong,orangepi-cm5 - const: rockchip,rk3588s + - description: Youyeetoo YY3588 + items: + - const: youyeetoo,yy3588 + - const: rockchip,rk3588 + - description: Zkmagic A95X Z2 items: - const: zkmagic,a95x-z2 diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml index 653f85997643..ab000befe76d 100644 --- a/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml +++ b/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml @@ -11,10 +11,10 @@ maintainers: description: | Guidelines for new compatibles for SoC blocks/components. - When adding new compatibles in new bindings, use the format:: + When adding new compatibles in new bindings, use the format: samsung,SoC-IP - For example:: + For example: samsung,exynos5433-cmu-isp select: diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml index e6443c266fa1..82dd58b95f8a 100644 --- a/Documentation/devicetree/bindings/arm/sunxi.yaml +++ b/Documentation/devicetree/bindings/arm/sunxi.yaml @@ -96,6 +96,12 @@ properties: - const: allwinner,ba10-tvbox - const: allwinner,sun4i-a10 + - description: Baijie A133 HelperBoard + items: + - const: baijie,helperboard-a133 + - const: baijie,helperboard-a133-core + - const: allwinner,sun50i-a100 + - description: BananaPi items: - const: lemaker,bananapi diff --git a/Documentation/devicetree/bindings/arm/xen.txt b/Documentation/devicetree/bindings/arm/xen.txt deleted file mode 100644 index f925290d4641..000000000000 --- a/Documentation/devicetree/bindings/arm/xen.txt +++ /dev/null @@ -1,62 +0,0 @@ -* Xen hypervisor device tree bindings - -Xen ARM virtual platforms shall have a top-level "hypervisor" node with -the following properties: - -- compatible: - compatible = "xen,xen-", "xen,xen"; - where is the version of the Xen ABI of the platform. - -- reg: specifies the base physical address and size of the regions in memory - where the special resources should be mapped to, using an HYPERVISOR_memory_op - hypercall. - Region 0 is reserved for mapping grant table, it must be always present. - The memory region is large enough to map the whole grant table (it is larger - or equal to gnttab_max_grant_frames()). - Regions 1...N are extended regions (unused address space) for mapping foreign - GFNs and grants, they might be absent if there is nothing to expose. - -- interrupts: the interrupt used by Xen to inject event notifications. - A GIC node is also required. - -To support UEFI on Xen ARM virtual platforms, Xen populates the FDT "uefi" node -under /hypervisor with following parameters: - -________________________________________________________________________________ -Name | Size | Description -================================================================================ -xen,uefi-system-table | 64-bit | Guest physical address of the UEFI System - | | Table. --------------------------------------------------------------------------------- -xen,uefi-mmap-start | 64-bit | Guest physical address of the UEFI memory - | | map. --------------------------------------------------------------------------------- -xen,uefi-mmap-size | 32-bit | Size in bytes of the UEFI memory map - | | pointed to in previous entry. --------------------------------------------------------------------------------- -xen,uefi-mmap-desc-size | 32-bit | Size in bytes of each entry in the UEFI - | | memory map. --------------------------------------------------------------------------------- -xen,uefi-mmap-desc-ver | 32-bit | Version of the mmap descriptor format. --------------------------------------------------------------------------------- - -Example (assuming #address-cells = <2> and #size-cells = <2>): - -hypervisor { - compatible = "xen,xen-4.3", "xen,xen"; - reg = <0 0xb0000000 0 0x20000>; - interrupts = <1 15 0xf08>; - uefi { - xen,uefi-system-table = <0xXXXXXXXX>; - xen,uefi-mmap-start = <0xXXXXXXXX>; - xen,uefi-mmap-size = <0xXXXXXXXX>; - xen,uefi-mmap-desc-size = <0xXXXXXXXX>; - xen,uefi-mmap-desc-ver = <0xXXXXXXXX>; - }; -}; - -The format and meaning of the "xen,uefi-*" parameters are similar to those in -Documentation/arch/arm/uefi.rst, which are provided by the regular UEFI stub. However -they differ because they are provided by the Xen hypervisor, together with a set -of UEFI runtime services implemented via hypercalls, see -http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,platform.h.html. diff --git a/Documentation/devicetree/bindings/arm/xen.yaml b/Documentation/devicetree/bindings/arm/xen.yaml new file mode 100644 index 000000000000..bad077a112c9 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/xen.yaml @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/xen.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xen hypervisor + +maintainers: + - Stefano Stabellini + +description: + Xen ARM virtual platforms shall have a top-level "hypervisor" node with + the properties defined below. + +properties: + $nodename: + pattern: '^hypervisor(@[0-9a-f]+)?$' + + compatible: + description: + Specifies the Xen hypervisor. The version of the Xen ABI is encoded + in the first item as "xen,xen-", followed by the generic + "xen,xen" string. + items: + - pattern: '^xen,xen-[0-9]+\.[0-9]+$' + - const: xen,xen + + reg: + description: | + Base physical address and size of the regions in memory where special + resources should be mapped to, using a HYPERVISOR_memory_op hypercall. + + Region 0 is reserved for mapping the grant table and must always be + present. The memory region must be large enough to map the whole grant + table (it is larger or equal to gnttab_max_grant_frames()). + + Regions 1...N are extended regions (unused address space) for mapping + foreign GFNs and grants. They might be absent if there is nothing to + expose. + minItems: 1 + + interrupts: + description: + The interrupt used by Xen to inject event notifications. + A GIC node is also required. + maxItems: 1 + + uefi: + type: object + description: + Node populated by Xen to support UEFI on Xen ARM virtual platforms. + The format and meaning of the "xen,uefi-*" parameters are similar to + those in Documentation/arch/arm/uefi.rst, but are provided by the Xen + hypervisor together with a set of UEFI runtime services implemented via + hypercalls. + + properties: + xen,uefi-system-table: + description: Guest physical address of the UEFI System Table. + $ref: /schemas/types.yaml#/definitions/uint64 + + xen,uefi-mmap-start: + description: Guest physical address of the UEFI memory map. + $ref: /schemas/types.yaml#/definitions/uint64 + + xen,uefi-mmap-size: + description: Size in bytes of the UEFI memory map pointed to by xen,uefi-mmap-start. + $ref: /schemas/types.yaml#/definitions/uint32 + + xen,uefi-mmap-desc-size: + description: Size in bytes of each entry in the UEFI memory map. + $ref: /schemas/types.yaml#/definitions/uint32 + + xen,uefi-mmap-desc-ver: + description: Version of the mmap descriptor format. + $ref: /schemas/types.yaml#/definitions/uint32 + + required: + - xen,uefi-system-table + - xen,uefi-mmap-start + - xen,uefi-mmap-size + - xen,uefi-mmap-desc-size + - xen,uefi-mmap-desc-ver + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + hypervisor@b0000000 { + compatible = "xen,xen-4.3", "xen,xen"; + reg = <0xb0000000 0x20000>; + interrupts = <1 15 0xf08>; + + uefi { + xen,uefi-system-table = /bits/ 64 <0x1301415>; + xen,uefi-mmap-start = /bits/ 64 <0x7591400>; + xen,uefi-mmap-size = <0x1800>; + xen,uefi-mmap-desc-size = <0x30>; + xen,uefi-mmap-desc-ver = <1>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/clock/amlogic,t7-peripherals-clkc.yaml b/Documentation/devicetree/bindings/clock/amlogic,t7-peripherals-clkc.yaml index 55bb73707d58..a4b214a941ea 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,t7-peripherals-clkc.yaml +++ b/Documentation/devicetree/bindings/clock/amlogic,t7-peripherals-clkc.yaml @@ -24,7 +24,7 @@ properties: const: 1 clocks: - minItems: 14 + minItems: 15 items: - description: input oscillator - description: input sys clk @@ -40,12 +40,13 @@ properties: - description: input gp1 pll - description: input mpll1 - description: input mpll2 + - description: input mpll3 - description: external input rmii oscillator (optional) - description: input video pll0 (optional) - description: external pad input for rtc (optional) clock-names: - minItems: 14 + minItems: 15 items: - const: xtal - const: sys @@ -61,6 +62,7 @@ properties: - const: gp1 - const: mpll1 - const: mpll2 + - const: mpll3 - const: ext_rmii - const: vid_pll0 - const: ext_rtc @@ -97,7 +99,8 @@ examples: <&gp0 1>, <&gp1 1>, <&mpll 4>, - <&mpll 6>; + <&mpll 6>, + <&mpll 8>; clock-names = "xtal", "sys", "fix", @@ -111,6 +114,7 @@ examples: "gp0", "gp1", "mpll1", - "mpll2"; + "mpll2", + "mpll3"; }; }; diff --git a/Documentation/devicetree/bindings/clock/amlogic,t7-pll-clkc.yaml b/Documentation/devicetree/bindings/clock/amlogic,t7-pll-clkc.yaml index 49c61f65deff..b488d92b7984 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,t7-pll-clkc.yaml +++ b/Documentation/devicetree/bindings/clock/amlogic,t7-pll-clkc.yaml @@ -72,7 +72,7 @@ allOf: contains: enum: - amlogic,t7-gp0-pll - - amlogic,t7-gp1--pll + - amlogic,t7-gp1-pll - amlogic,t7-hifi-pll - amlogic,t7-pcie-pll - amlogic,t7-mpll diff --git a/Documentation/devicetree/bindings/clock/canaan,k230-clk.yaml b/Documentation/devicetree/bindings/clock/canaan,k230-clk.yaml new file mode 100644 index 000000000000..34c93cb5db40 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/canaan,k230-clk.yaml @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/canaan,k230-clk.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K230 Clock + +maintainers: + - Xukai Wang + +description: + The Canaan K230 clock controller generates various clocks for SoC + peripherals. See include/dt-bindings/clock/canaan,k230-clk.h for + valid clock IDs. + +properties: + compatible: + const: canaan,k230-clk + + reg: + items: + - description: PLL control registers + - description: Sysclk control registers + + clocks: + items: + - description: Main external reference clock + - description: + External clock which used as the pulse input + for the timer to provide timing signals. + + clock-names: + items: + - const: osc24m + - const: timer-pulse-in + + '#clock-cells': + const: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + +additionalProperties: false + +examples: + - | + clock-controller@91102000 { + compatible = "canaan,k230-clk"; + reg = <0x91102000 0x40>, + <0x91100000 0x108>; + clocks = <&osc24m>, <&timerx_pulse_in>; + clock-names = "osc24m", "timer-pulse-in"; + #clock-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml b/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml index 6f3a8578fe2a..0db5504013d5 100644 --- a/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml +++ b/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml @@ -37,6 +37,9 @@ properties: '#power-domain-cells': const: 1 + '#reset-cells': + const: 1 + required: - compatible - reg @@ -44,16 +47,27 @@ required: additionalProperties: false -if: - not: - properties: - compatible: - contains: - const: marvell,pxa1908-apmu - -then: - properties: - '#power-domain-cells': false +allOf: + - if: + not: + properties: + compatible: + contains: + const: marvell,pxa1908-apmu + then: + properties: + '#power-domain-cells': false + - if: + not: + properties: + compatible: + contains: + enum: + - marvell,pxa1908-apbc + - marvell,pxa1908-apbcp + then: + properties: + '#reset-cells': false examples: # APMU block: diff --git a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml index ef2b1e204430..a177a1934b19 100644 --- a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml @@ -42,12 +42,6 @@ properties: - const: cfg_ahb_clk - const: gcc_disp_gpll0_div_clk_src - '#clock-cells': - const: 1 - - '#power-domain-cells': - const: 1 - power-domains: description: A phandle and PM domain specifier for the CX power domain. @@ -58,18 +52,16 @@ properties: A phandle to an OPP node describing the power domain's performance point. maxItems: 1 - reg: - maxItems: 1 - required: - compatible - - reg - clocks - clock-names - - '#clock-cells' - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | @@ -101,6 +93,7 @@ examples: power-domains = <&rpmpd SM6125_VDDCX>; #clock-cells = <1>; + #reset-cells = <1>; #power-domain-cells = <1>; }; ... diff --git a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml index 53a5ab319159..6863db9bd092 100644 --- a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml @@ -13,7 +13,7 @@ description: | Qualcomm display clock control module provides the clocks, resets and power domains on SM8150/SM8250/SM8350. - See also:: + See also: include/dt-bindings/clock/qcom,dispcc-sm8150.h include/dt-bindings/clock/qcom,dispcc-sm8250.h include/dt-bindings/clock/qcom,dispcc-sm8350.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml index 27df7e3e5bf3..68532244901e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on APQ8064. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8960.h include/dt-bindings/reset/qcom,gcc-msm8960.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml index 0a0a26d9beab..1c022e75fd71 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on APQ8084. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-apq8084.h include/dt-bindings/reset/qcom,gcc-apq8084.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml index 4d2614d4f368..c7fb84438db7 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml @@ -15,7 +15,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on IPQ6018. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-ipq6018.h include/dt-bindings/reset/qcom,gcc-ipq6018.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml index a71557395c01..b4d3175780bc 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml @@ -13,7 +13,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on IPQ8064. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-ipq806x.h (qcom,gcc-ipq8064) include/dt-bindings/reset/qcom,gcc-ipq806x.h (qcom,gcc-ipq8064) diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml index d7da30b0e7ee..0a7be7583bdd 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-mdm9607.h allOf: diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml index 418dea31eb62..0656d5ee448d 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-mdm9615.h allOf: diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml index e03b6d0acdb6..70c9da1f35c2 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks and resets on MSM8660 - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8660.h include/dt-bindings/reset/qcom,gcc-msm8660.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml index ce1f5a60bd8c..2edb6c251d99 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml @@ -13,7 +13,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on MSM8909, MSM8917 or QM215. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8909.h include/dt-bindings/clock/qcom,gcc-msm8917.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml index 258b6b93deca..af4b639ea8c3 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on MSM8916 or MSM8939. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8916.h include/dt-bindings/clock/qcom,gcc-msm8939.h include/dt-bindings/reset/qcom,gcc-msm8916.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml index ced3118c8580..fc0360554f68 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml @@ -15,7 +15,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on MSM8937, MSM8940, MSM8953 or SDM439. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8917.h include/dt-bindings/clock/qcom,gcc-msm8953.h diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml index 929fafc84c19..378dfe7854ac 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml @@ -15,7 +15,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on MSM8974 (all variants) and MSM8226. - See also:: + See also: include/dt-bindings/clock/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974) include/dt-bindings/reset/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974) diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml index 724ce0491118..72aaf699cf70 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on SDM630, SDM636 and SDM660 - See also:: + See also: include/dt-bindings/clock/qcom,gcc-sdm660.h (qcom,gcc-sdm630 and qcom,gcc-sdm660) $ref: qcom,gcc.yaml# diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml index 4cdff6161bf0..3ac4419009a9 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml @@ -14,7 +14,7 @@ description: | Qualcomm graphics clock control module provides the clocks, resets and power domains on Qualcomm SoCs. - See also:: + See also: include/dt-bindings/clock/qcom,gpucc-sdm845.h include/dt-bindings/clock/qcom,gpucc-sa8775p.h include/dt-bindings/clock/qcom,gpucc-sc7180.h diff --git a/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml new file mode 100644 index 000000000000..4f428c0f7286 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,hawi-gcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller on Hawi + +maintainers: + - Vivek Aknurwar + +description: | + Qualcomm global clock control module provides the clocks, resets and power + domains on Hawi. + + See also: include/dt-bindings/clock/qcom,hawi-gcc.h + +properties: + compatible: + const: qcom,hawi-gcc + + clocks: + items: + - description: Board XO source + - description: Board Always On XO source + - description: Sleep clock source + - description: PCIE 0 Pipe clock source + - description: PCIE 1 Pipe clock source + - description: UFS PHY RX symbol 0 clock + - description: UFS PHY RX symbol 1 clock + - description: UFS PHY TX symbol 0 clock + - description: USB3 PHY wrapper pipe clock + +required: + - compatible + - clocks + - '#power-domain-cells' + +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,hawi-gcc"; + reg = <0x00100000 0x1f4200>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>, + <&pcie0_phy>, + <&pcie1_phy>, + <&ufs_mem_phy 0>, + <&ufs_mem_phy 1>, + <&ufs_mem_phy 2>, + <&usb_1_qmpphy>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml index 489d0fc5607c..9925b931ecad 100644 --- a/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on IPQ5018 - See also:: + See also: include/dt-bindings/clock/qcom,ipq5018-gcc.h include/dt-bindings/reset/qcom,ipq5018-gcc.h diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml index de338c05190f..a4f9af8fa187 100644 --- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml @@ -8,7 +8,7 @@ title: Qualcomm CMN PLL Clock Controller on IPQ SoC maintainers: - Bjorn Andersson - - Luo Jie + - Luo Jie description: The CMN (or common) PLL clock controller expects a reference @@ -25,6 +25,7 @@ properties: compatible: enum: - qcom,ipq5018-cmn-pll + - qcom,ipq5332-cmn-pll - qcom,ipq5424-cmn-pll - qcom,ipq6018-cmn-pll - qcom,ipq8074-cmn-pll diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml index 27ae9938febc..5b128fa841aa 100644 --- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml @@ -14,7 +14,7 @@ description: | Qualcomm global clock control module provides the clocks, resets and power domains on IPQ9574 - See also:: + See also: include/dt-bindings/clock/qcom,ipq9574-gcc.h include/dt-bindings/reset/qcom,ipq9574-gcc.h diff --git a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml index 466c884aa2ba..e868963f659b 100644 --- a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml @@ -44,7 +44,7 @@ required: - power-domains - '#power-domain-cells' -unevaluatedProperties: false +additionalProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml index f63149ecf3e1..707b25d2c11e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml @@ -25,6 +25,10 @@ properties: - description: Sleep clock source - description: Camera AHB clock from GCC + interconnects: + items: + - description: Interconnect path to enable the MultiMedia NoC + required: - compatible - clocks @@ -37,12 +41,16 @@ unevaluatedProperties: false examples: - | #include + #include + #include clock-controller@adb0000 { compatible = "qcom,milos-camcc"; reg = <0x0adb0000 0x40000>; clocks = <&bi_tcxo_div2>, <&sleep_clk>, <&gcc GCC_CAMERA_AHB_CLK>; + interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS + &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>; #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml new file mode 100644 index 000000000000..fbcb5d3f3e3d --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,milos-gxclkctl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Graphics Power Domain Controller on Milos + +maintainers: + - Luca Weiss + +description: | + Qualcomm GX(graphics) is a clock controller which has PLLs, clocks and + Power domains (GDSC). This module provides the power domains control + of gxclkctl on Qualcomm SoCs which helps the recovery of Graphics subsystem. + + See also: + include/dt-bindings/clock/qcom,kaanapali-gxclkctl.h + +properties: + compatible: + enum: + - qcom,milos-gxclkctl + + reg: + maxItems: 1 + + power-domains: + description: + Power domains required for the clock controller to operate + items: + - description: GFX power domain + - description: GPUCC(CX) power domain + + '#power-domain-cells': + const: 1 + +required: + - compatible + - reg + - power-domains + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + soc { + #address-cells = <2>; + #size-cells = <2>; + + clock-controller@3d64000 { + compatible = "qcom,milos-gxclkctl"; + reg = <0x0 0x03d64000 0x0 0x6000>; + power-domains = <&rpmhpd RPMHPD_GFX>, + <&gpucc 0>; + #power-domain-cells = <1>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml b/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml index 61473385da2d..8d7a58609372 100644 --- a/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml @@ -8,13 +8,13 @@ title: Qualcomm NSS Clock & Reset Controller on QCA8386/QCA8084 maintainers: - Bjorn Andersson - - Luo Jie + - Luo Jie description: | Qualcomm NSS clock control module provides the clocks and resets on QCA8386(switch mode)/QCA8084(PHY mode) - See also:: + See also: include/dt-bindings/clock/qcom,qca8k-nsscc.h include/dt-bindings/reset/qcom,qca8k-nsscc.h diff --git a/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml index 734880805c1b..bedbdabef672 100644 --- a/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm graphics clock control module provides the clocks, resets and power domains on Qualcomm SoCs. - See also:: + See also: include/dt-bindings/clock/qcom,qcm2290-gpucc.h properties: diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml index ab97d4b7dba8..b6c835bfd0d9 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml @@ -12,7 +12,7 @@ maintainers: description: | The clock enumerators are defined in and - come in pairs:: FOO_CLK followed by FOO_A_CLK. The latter clock is + come in pairs: FOO_CLK followed by FOO_A_CLK. The latter clock is an "active" clock, which means that the consumer only care that the clock is available when the apps CPU subsystem is active, i.e. not suspended or in deep idle. If it is important that the clock keeps running during system diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index a2c404a57981..d344b3386042 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -19,6 +19,7 @@ properties: enum: - qcom,eliza-rpmh-clk - qcom,glymur-rpmh-clk + - qcom,hawi-rpmh-clk - qcom,kaanapali-rpmh-clk - qcom,milos-rpmh-clk - qcom,nord-rpmh-clk diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml index 99ab9106009f..fd06ac9bceb9 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm LPASS core and audio clock control module provides the clocks and power domains on SC7280. - See also:: + See also: include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h diff --git a/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml index 273d66e245c5..f235b4e24cc7 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm LPASS core and audio clock control module provides the clocks, and reset on SC8280XP. - See also:: + See also: include/dt-bindings/clock/qcom,lpasscc-sc8280xp.h properties: diff --git a/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml index 8cbab3fbb660..d7e1938b5e1b 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml @@ -14,7 +14,7 @@ description: | Qualcomm LPASS core and audio clock controllers provide audio-related resets on SM6115 and its derivatives. - See also:: + See also: include/dt-bindings/clock/qcom,sm6115-lpasscc.h properties: diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml index 5c2ecec0624e..a986ab4ce7c7 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm video clock control module provides the clocks, resets and power domains on Qualcomm SoCs. - See also:: + See also: include/dt-bindings/clock/qcom,videocc-sm8350.h include/dt-bindings/reset/qcom,videocc-sm8350.h diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml index 8492a7ef7324..8e460df9f744 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml @@ -18,6 +18,7 @@ description: | See also: include/dt-bindings/clock/qcom,kaanapali-camcc.h include/dt-bindings/clock/qcom,kaanapali-cambistmclkcc.h + include/dt-bindings/clock/qcom,nord-camcc.h include/dt-bindings/clock/qcom,sm8450-camcc.h include/dt-bindings/clock/qcom,sm8550-camcc.h include/dt-bindings/clock/qcom,sm8650-camcc.h @@ -29,6 +30,7 @@ properties: enum: - qcom,kaanapali-cambistmclkcc - qcom,kaanapali-camcc + - qcom,nord-camcc - qcom,sm8450-camcc - qcom,sm8475-camcc - qcom,sm8550-camcc diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml index fdbdf605ee69..ba85692240e0 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml @@ -18,6 +18,7 @@ description: | include/dt-bindings/clock/qcom,glymur-gpucc.h include/dt-bindings/clock/qcom,kaanapali-gpucc.h include/dt-bindings/clock/qcom,milos-gpucc.h + include/dt-bindings/clock/qcom,nord-gpucc.h include/dt-bindings/clock/qcom,sar2130p-gpucc.h include/dt-bindings/clock/qcom,sm4450-gpucc.h include/dt-bindings/clock/qcom,sm8450-gpucc.h @@ -33,6 +34,8 @@ properties: - qcom,glymur-gpucc - qcom,kaanapali-gpucc - qcom,milos-gpucc + - qcom,nord-gpu2cc + - qcom,nord-gpucc - qcom,sar2130p-gpucc - qcom,sm4450-gpucc - qcom,sm8450-gpucc diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml index 7bbf120d928c..9b9878e9b9cf 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml @@ -17,21 +17,25 @@ description: | See also: include/dt-bindings/clock/qcom,glymur-videocc.h include/dt-bindings/clock/qcom,kaanapali-videocc.h + include/dt-bindings/clock/qcom,nord-videocc.h include/dt-bindings/clock/qcom,sm8450-videocc.h include/dt-bindings/clock/qcom,sm8650-videocc.h include/dt-bindings/clock/qcom,sm8750-videocc.h + include/dt-bindings/clock/qcom,x1p42100-videocc.h properties: compatible: enum: - qcom,glymur-videocc - qcom,kaanapali-videocc + - qcom,nord-videocc - qcom,sm8450-videocc - qcom,sm8475-videocc - qcom,sm8550-videocc - qcom,sm8650-videocc - qcom,sm8750-videocc - qcom,x1e80100-videocc + - qcom,x1p42100-videocc clocks: items: @@ -70,6 +74,7 @@ allOf: - qcom,sm8450-videocc - qcom,sm8550-videocc - qcom,sm8750-videocc + - qcom,x1p42100-videocc then: required: - required-opps diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-dispcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-dispcc.yaml index 591ce91b8d54..61f58fbd5bd2 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8550-dispcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-dispcc.yaml @@ -16,6 +16,7 @@ description: | See also: - include/dt-bindings/clock/qcom,kaanapali-dispcc.h + - include/dt-bindings/clock/qcom,nord-dispcc.h - include/dt-bindings/clock/qcom,sm8550-dispcc.h - include/dt-bindings/clock/qcom,sm8650-dispcc.h - include/dt-bindings/clock/qcom,sm8750-dispcc.h @@ -25,6 +26,8 @@ properties: compatible: enum: - qcom,kaanapali-dispcc + - qcom,nord-dispcc0 + - qcom,nord-dispcc1 - qcom,sar2130p-dispcc - qcom,sm8550-dispcc - qcom,sm8650-dispcc diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml index 1ccdf4b0f5dd..08824f848973 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml @@ -17,6 +17,7 @@ description: | See also: - include/dt-bindings/clock/qcom,eliza-tcsr.h - include/dt-bindings/clock/qcom,glymur-tcsr.h + - include/dt-bindings/clock/qcom,hawi-tcsrcc.h - include/dt-bindings/clock/qcom,nord-tcsrcc.h - include/dt-bindings/clock/qcom,sm8550-tcsr.h - include/dt-bindings/clock/qcom,sm8650-tcsr.h @@ -28,6 +29,7 @@ properties: - enum: - qcom,eliza-tcsr - qcom,glymur-tcsr + - qcom,hawi-tcsrcc - qcom,kaanapali-tcsr - qcom,milos-tcsr - qcom,nord-tcsrcc diff --git a/Documentation/devicetree/bindings/clock/qcom,videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,videocc.yaml index f4ff9acef9d5..124d259fc85e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,videocc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,videocc.yaml @@ -13,7 +13,7 @@ description: | Qualcomm video clock control module provides the clocks, resets and power domains on Qualcomm SoCs. - See also:: + See also: include/dt-bindings/clock/qcom,sm6350-videocc.h include/dt-bindings/clock/qcom,videocc-sc7180.h include/dt-bindings/clock/qcom,videocc-sc7280.h diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml index 938a2f1ff3fc..b28614186cc0 100644 --- a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml @@ -23,6 +23,7 @@ properties: compatible: enum: - qcom,x1e80100-camcc + - qcom,x1p42100-camcc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/renesas,cpg-div6-clock.yaml b/Documentation/devicetree/bindings/clock/renesas,cpg-div6-clock.yaml index 2197c952e21d..b6ee8c8efd46 100644 --- a/Documentation/devicetree/bindings/clock/renesas,cpg-div6-clock.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,cpg-div6-clock.yaml @@ -60,7 +60,7 @@ examples: clock-output-names = "main", "pll0", "pll1", "pll2", "pll2s", "pll2h", "z", "z2", "i", "m3", "b", "m1", "m2", - "zx", "zs", "hp"; + "zx", "zs", "hp", "ztr", "zt"; }; sdhi2_clk: sdhi2_clk@e615007c { diff --git a/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml b/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml index f261445bf341..dd3e66a4559b 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml @@ -26,16 +26,22 @@ properties: maxItems: 1 clocks: + minItems: 3 items: - description: AUDIO_EXTAL clock input - description: RTXIN clock input - description: QEXTAL clock input + - description: AUDIO_CLKB clock input + - description: AUDIO_CLKC clock input clock-names: + minItems: 3 items: - const: audio_extal - const: rtxin - const: qextal + - const: audio_clkb + - const: audio_clkc '#clock-cells': description: | diff --git a/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml index b339f1f9f072..990d287d0a90 100644 --- a/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Renesas Versaclock7 Programmable Clock maintainers: - - Alex Helms + - Biju Das description: | Renesas Versaclock7 is a family of configurable clock generator and diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml index b05f83533e3d..56ab972c3da5 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml @@ -14,17 +14,17 @@ maintainers: description: | Expected external clocks, defined in DTS as fixed-rate clocks with a matching - name:: + name: - "fin_pll" - PLL input clock from XXTI - "xrtcxti" - input clock from XRTCXTI - "ioclk_pcm_extclk" - pcm external operation clock - "ioclk_spdif_extclk" - spdif external operation clock - "ioclk_i2s_cdclk" - i2s0 codec clock - Phy clocks:: + Phy clocks: There are several clocks which are generated by specific PHYs. These clocks are fed into the clock controller and then routed to the hardware blocks. - These clocks are defined as fixed clocks in the driver with following names:: + These clocks are defined as fixed clocks in the driver with following names: - "phyclk_dptx_phy_ch3_txd_clk" - dp phy clock for channel 3 - "phyclk_dptx_phy_ch2_txd_clk" - dp phy clock for channel 2 - "phyclk_dptx_phy_ch1_txd_clk" - dp phy clock for channel 1 diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml index b737c9d35a1c..1d907dd8fbf1 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml @@ -14,7 +14,7 @@ maintainers: description: | Expected external clocks, defined in DTS as fixed-rate clocks with a matching - name:: + name: - "fin_pll" - PLL input clock from XXTI All available clocks are defined as preprocessor macros in diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml index 3f9326e09f79..8a289f1e2ace 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml @@ -14,7 +14,7 @@ maintainers: description: | Expected external clocks, defined in DTS as fixed-rate clocks with a matching - name:: + name: - "oscclk" - PLL input clock from XXTI All available clocks are defined as preprocessor macros in diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml index c137c6744ef9..a51cd4fafb41 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml @@ -14,7 +14,7 @@ maintainers: description: | Expected external clocks, defined in DTS as fixed-rate clocks with a matching - name:: + name: - "fin_pll" - PLL input clock from XXTI All available clocks are defined as preprocessor macros in diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml index cdc5ded59fe5..68c2fd318765 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml @@ -17,7 +17,7 @@ description: | Exynos850 clock controller is comprised of several CMU units, generating clocks for different domains. Those CMU units are modeled as separate device tree nodes, and might depend on each other. Root clocks in that clock tree are - two external clocks:: OSCCLK (26 MHz) and RTCCLK (32768 Hz). Those external + two external clocks: OSCCLK (26 MHz) and RTCCLK (32768 Hz). Those external clocks must be defined as fixed-rate clocks in dts. CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml index 32f39e543b36..e9d17d48b4f3 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml @@ -17,7 +17,7 @@ description: | Exynos Auto v9 clock controller is comprised of several CMU units, generating clocks for different domains. Those CMU units are modeled as separate device tree nodes, and might depend on each other. Root clocks in that clock tree are - two external clocks:: OSCCLK/XTCXO (26 MHz) and RTCCLK/XrtcXTI (32768 Hz). + two external clocks: OSCCLK/XTCXO (26 MHz) and RTCCLK/XrtcXTI (32768 Hz). The external OSCCLK must be defined as fixed-rate clock in dts. CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml index 6b1fc61a2ff9..475db824d4d3 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml @@ -17,7 +17,7 @@ description: | ExynosAuto v920 clock controller is comprised of several CMU units, generating clocks for different domains. Those CMU units are modeled as separate device tree nodes, and might depend on each other. Root clocks in that clock tree are - two external clocks:: OSCCLK/XTCXO (38.4 MHz) and RTCCLK/XrtcXTI (32768 Hz). + two external clocks: OSCCLK/XTCXO (38.4 MHz) and RTCCLK/XrtcXTI (32768 Hz). The external OSCCLK must be defined as fixed-rate clock in dts. CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and diff --git a/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml index 67a33665cf00..b1617d96d3fb 100644 --- a/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml @@ -14,7 +14,7 @@ maintainers: description: | Expected external clocks, defined in DTS as fixed-rate clocks with a matching - name:: + name: - "xxti" - external crystal oscillator connected to XXTI and XXTO pins of the SoC, - "xusbxti" - external crystal oscillator connected to XUSBXTI and XUSBXTO diff --git a/Documentation/devicetree/bindings/clock/xlnx,versal-clk.yaml b/Documentation/devicetree/bindings/clock/xlnx,versal-clk.yaml index bef109d163a8..b533ffd082fd 100644 --- a/Documentation/devicetree/bindings/clock/xlnx,versal-clk.yaml +++ b/Documentation/devicetree/bindings/clock/xlnx,versal-clk.yaml @@ -17,9 +17,7 @@ description: | properties: compatible: oneOf: - - enum: - - xlnx,versal-clk - - xlnx,zynqmp-clk + - const: xlnx,versal-clk - items: - enum: - xlnx,versal-net-clk @@ -32,11 +30,17 @@ properties: description: List of clock specifiers which are external input clocks to the given clock controller. minItems: 2 - maxItems: 8 + items: + - description: reference clock + - description: alternate reference clock for programmable logic + - description: alternate reference clock clock-names: minItems: 2 - maxItems: 8 + items: + - const: ref + - const: pl_alt_ref + - const: alt_ref required: - compatible @@ -47,78 +51,24 @@ required: additionalProperties: false allOf: - - if: - properties: - compatible: - contains: - enum: - - xlnx,versal-clk - - then: - properties: - clocks: - items: - - description: reference clock - - description: alternate reference clock for programmable logic - - clock-names: - items: - - const: ref - - const: pl_alt_ref - - if: properties: compatible: contains: enum: - xlnx,versal-net-clk - then: properties: clocks: - items: - - description: reference clock - - description: alternate reference clock for programmable logic - - description: alternate reference clock - + minItems: 3 clock-names: - items: - - const: ref - - const: pl_alt_ref - - const: alt_ref - - - if: - properties: - compatible: - contains: - enum: - - xlnx,zynqmp-clk - - then: + minItems: 3 + else: properties: clocks: - minItems: 5 - items: - - description: PS reference clock - - description: reference clock for video system - - description: alternative PS reference clock - - description: auxiliary reference clock - - description: transceiver reference clock - - description: (E)MIO clock source (Optional clock) - - description: GEM emio clock (Optional clock) - - description: Watchdog external clock (Optional clock) - + maxItems: 2 clock-names: - minItems: 5 - items: - - const: pss_ref_clk - - const: video_clk - - const: pss_alt_ref_clk - - const: aux_ref_clk - - const: gt_crx_ref_clk - - pattern: "^mio_clk[00-77]+.*$" - - pattern: "gem[0-3]+_emio_clk.*$" - - pattern: "swdt[0-1]+_ext_clk.*$" + maxItems: 2 examples: - | @@ -135,12 +85,11 @@ examples: }; }; + - | clock-controller { - #clock-cells = <1>; - compatible = "xlnx,zynqmp-clk"; - clocks = <&pss_ref_clk>, <&video_clk>, <&pss_alt_ref_clk>, - <&aux_ref_clk>, <>_crx_ref_clk>; - clock-names = "pss_ref_clk", "video_clk", "pss_alt_ref_clk", - "aux_ref_clk", "gt_crx_ref_clk"; + compatible = "xlnx,versal-net-clk", "xlnx,versal-clk"; + clocks = <&ref>, <&pl_alt_ref>, <&alt_ref>; + clock-names = "ref", "pl_alt_ref", "alt_ref"; + #clock-cells = <1>; }; ... diff --git a/Documentation/devicetree/bindings/clock/xlnx,zynqmp-clk.yaml b/Documentation/devicetree/bindings/clock/xlnx,zynqmp-clk.yaml new file mode 100644 index 000000000000..ccea3184b041 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/xlnx,zynqmp-clk.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/xlnx,zynqmp-clk.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xilinx ZynqMP clock controller + +maintainers: + - Michal Simek + +description: + The clock controller is a hardware block of Xilinx ZynqMP clock tree. It + reads required input clock frequencies from the devicetree and acts as clock + provider for all clock consumers of PS clocks. + +properties: + compatible: + const: xlnx,zynqmp-clk + + "#clock-cells": + const: 1 + + clocks: + description: List of clock specifiers which are external input + clocks to the given clock controller. + minItems: 5 + items: + - description: PS reference clock + - description: reference clock for video system + - description: alternative PS reference clock + - description: auxiliary reference clock + - description: transceiver reference clock + - description: (E)MIO clock source + - description: GEM emio clock + - description: Watchdog external clock + + clock-names: + minItems: 5 + items: + - const: pss_ref_clk + - const: video_clk + - const: pss_alt_ref_clk + - const: aux_ref_clk + - const: gt_crx_ref_clk + - pattern: "^mio_clk(0[0-9]|[1-6][0-9]|7[0-7])+.*$" + - pattern: "gem[0-3]+_emio_clk.*$" + - pattern: "swdt[0-1]+_ext_clk.*$" + +required: + - compatible + - "#clock-cells" + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + clock-controller { + compatible = "xlnx,zynqmp-clk"; + clocks = <&pss_ref_clk>, <&video_clk>, <&pss_alt_ref_clk>, + <&aux_ref_clk>, <>_crx_ref_clk>; + clock-names = "pss_ref_clk", "video_clk", "pss_alt_ref_clk", + "aux_ref_clk", "gt_crx_ref_clk"; + #clock-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml index db895c50e2d2..d690eff2e86d 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml @@ -17,6 +17,7 @@ properties: - qcom,hawi-inline-crypto-engine - qcom,kaanapali-inline-crypto-engine - qcom,milos-inline-crypto-engine + - qcom,nord-inline-crypto-engine - qcom,qcs8300-inline-crypto-engine - qcom,sa8775p-inline-crypto-engine - qcom,sc7180-inline-crypto-engine @@ -63,6 +64,7 @@ allOf: enum: - qcom,eliza-inline-crypto-engine - qcom,milos-inline-crypto-engine + - qcom,nord-inline-crypto-engine then: required: diff --git a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml index dbc0613e427e..395425a70db8 100644 --- a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml +++ b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml @@ -73,16 +73,16 @@ properties: clocks: description: | - Several clocks are used, depending on the variant. Typical ones are:: - - bus:: Display AHB clock. - - byte:: Display byte clock. - - byte_intf:: Display byte interface clock. - - core:: Display core clock. - - core_mss:: Core MultiMedia SubSystem clock. - - iface:: Display AXI clock. - - mdp_core:: MDP Core clock. - - mnoc:: MNOC clock - - pixel:: Display pixel clock. + Several clocks are used, depending on the variant. Typical ones are: + - bus: Display AHB clock. + - byte: Display byte clock. + - byte_intf: Display byte interface clock. + - core: Display core clock. + - core_mss: Core MultiMedia SubSystem clock. + - iface: Display AXI clock. + - mdp_core: MDP Core clock. + - mnoc: MNOC clock + - pixel: Display pixel clock. minItems: 3 maxItems: 12 diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml index e2730a2f25cf..6c827cf9692b 100644 --- a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml +++ b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml @@ -200,9 +200,11 @@ examples: <0x0aec2000 0x1c8>; clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>, - <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>; + <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>, + <&gcc GCC_EDP_REF_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; #clock-cells = <1>; #phy-cells = <0>; diff --git a/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml b/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml new file mode 100644 index 000000000000..f820dc2f732b --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/anbernic,td4310.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Anbernic TD4310 Based Panels + +maintainers: + - Chris Morgan + +description: + Anbernic TD4310 Based Panels, such as the RG-Vita-Pro panel + (a 1080x1920 5.5 inch panel). + +allOf: + - $ref: panel-common.yaml# + +properties: + compatible: + items: + - enum: + - anbernic,panel-vita-pro + - const: anbernic,td4310 + + reg: + maxItems: 1 + + vdd-supply: + description: Panel power supply + +required: + - compatible + - port + - reg + - reset-gpios + - vdd-supply + +unevaluatedProperties: false + +examples: + - | + #include + + dsi { + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "anbernic,panel-vita-pro", "anbernic,td4310"; + reg = <0>; + backlight = <&backlight>; + enable-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + rotation = <270>; + vdd-supply = <&vdd_lcd>; + + port { + endpoint { + remote-endpoint = <&dsi_out>; + }; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml b/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml new file mode 100644 index 000000000000..90e69f30cd91 --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/chipone,icna3512.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Chipone ICNA3512 and ICNA3520 display drivers + +maintainers: + - Neil Armstrong + +description: + The Chipone ICNA3512 and ICNA3520 are DDICs connected + using a MIPI-DSI video interface. + +allOf: + - $ref: panel-common.yaml# + +properties: + compatible: + oneOf: + - items: + - enum: + - ayaneo,pocketds-panel-top + - ayntec,odin2portal-panel + - const: chipone,icna3512 + + - items: + - enum: + - ayntec,odin3-panel + - ayntec,thor-panel-top + - const: chipone,icna3520 + + reg: + maxItems: 1 + description: DSI virtual channel + + vdd-supply: true + vddio-supply: true + vci-supply: true + disp-supply: true + blvdd-supply: true + + port: true + reset-gpios: true + rotation: true + +required: + - compatible + - reg + - vdd-supply + - vddio-supply + - vci-supply + - disp-supply + - blvdd-supply + - reset-gpios + +additionalProperties: false + +examples: + - | + #include + + dsi { + #address-cells = <1>; + #size-cells = <0>; + panel@0 { + compatible = "ayntec,odin2portal-panel", "chipone,icna3512"; + reg = <0>; + vdd-supply = <&vreg_l11b_1p2>; + vddio-supply = <&vreg_l12b_1p8>; + vci-supply = <&vreg_l13b_3p0>; + disp-supply = <&vdd_disp_2v8>; + blvdd-supply = <&vdd_bl_5v0>; + reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml index e067a2f6d0b2..aeca3c9a599c 100644 --- a/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml +++ b/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml @@ -40,6 +40,9 @@ properties: vddi-supply: description: power supply for IC + bl-supply: + description: power supply for backlight, in case it's managed via DSC + backlight: true ports: true diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml new file mode 100644 index 000000000000..ba8c5bbf8ffc --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/ilitek,ili7807s.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ilitek ILI7807S-based DSI panels + +maintainers: + - Arpit Saini + +allOf: + - $ref: panel-common.yaml# + +properties: + compatible: + items: + - enum: + - dlc,dlc0697 + - const: ilitek,ili7807s + + reg: + maxItems: 1 + description: DSI virtual channel + + vddi-supply: + description: I/O voltage supply (1.8V) + + avdd-supply: + description: Positive LCD bias supply (AVDD), typically +5.5V + (range 4.5V to 6.3V) + + avee-supply: + description: Negative LCD bias supply (AVEE), typically -5.5V + (range -6.3V to -4.5V) + +required: + - compatible + - reg + - reset-gpios + - vddi-supply + - avdd-supply + - avee-supply + - port + +unevaluatedProperties: false + +examples: + - | + #include + + dsi { + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "dlc,dlc0697", "ilitek,ili7807s"; + reg = <0>; + + reset-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>; + vddi-supply = <&pm4125_l15>; + avdd-supply = <&avdd>; + avee-supply = <&avee>; + + port { + panel_in: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml new file mode 100644 index 000000000000..ea7449273022 --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/ilitek,ili9488.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ilitek ILI9488 based MIPI-DSI panels + +maintainers: + - Igor Reznichenko + +allOf: + - $ref: panel-common.yaml# + +properties: + compatible: + items: + - enum: + - focuslcds,e35gh-i-mw800cb + - const: ilitek,ili9488 + + reg: + maxItems: 1 + + vci-supply: true + iovcc-supply: true + +required: + - compatible + - reg + - vci-supply + - iovcc-supply + - reset-gpios + - backlight + - port + +unevaluatedProperties: false + +examples: + - | + #include + + dsi { + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "focuslcds,e35gh-i-mw800cb", "ilitek,ili9488"; + reg = <0>; + vci-supply = <®_vci_panel>; + iovcc-supply = <®_iovcc_panel>; + reset-gpios = <&gpio3 6 GPIO_ACTIVE_LOW>; + backlight = <&pwm_bl>; + + port { + panel_in: endpoint { + remote-endpoint = <&dsi_out>; + }; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml b/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml new file mode 100644 index 000000000000..adfdd2c300a3 --- /dev/null +++ b/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/panel/renesas,r63419.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas R63419 based dual-DSI video mode Display Panel + +maintainers: + - Neil Armstrong + +description: + The Renesas R63419 is a generic DDIC used to control dual-DSI LCD panels. + +allOf: + - $ref: panel-common-dual.yaml# + +properties: + compatible: + items: + - enum: + - ayaneo,wt0600-2k + - ayaneo,wt0630-2k + - const: renesas,r63419 + + reg: + maxItems: 1 + + vdd-supply: true + vddio-supply: true + vsp-supply: true + vsn-supply: true + vci-supply: true + + backlight: true + reset-gpios: true + rotation: true + ports: true + +required: + - compatible + - reg + - vdd-supply + - vddio-supply + - vsp-supply + - vsn-supply + - vci-supply + - backlight + - reset-gpios + - ports + +additionalProperties: false + +examples: + - | + #include + + dsi { + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "ayaneo,wt0600-2k", "renesas,r63419"; + reg = <0>; + + reset-gpios = <&gpio 176 GPIO_ACTIVE_LOW>; + + vdd-supply = <&vdd_3v0_lcd>; + vddio-supply = <&vdd_1v8_io>; + vsn-supply = <&vdd_5v0_neg>; + vsp-supply = <&vdd_5v0_pos>; + vci-supply = <&vdd_3v0_vci>; + + backlight = <&backlight>; + + rotation = <90>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in0: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + + port@1 { + reg = <1>; + panel_in1: endpoint { + remote-endpoint = <&dsi1_out>; + }; + }; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml index 1bbe0da3997c..27536eeaed85 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml @@ -25,6 +25,8 @@ properties: - samsung,atna40ct06 # Samsung 14" WQXGA+ (2880x1800 pixels) eDP AMOLED panel - samsung,atna40cu11 + # Samsung 14" WQXGA+ (2880x1800 pixels) eDP AMOLED panel + - samsung,atna40hq08 # Samsung 14" WQXGA+ (2880×1800 pixels) eDP AMOLED panel - samsung,atna40yk20 # Samsung 14.5" WQXGA+ (2880x1800 pixels) eDP AMOLED panel diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml index d99b23b88cc5..6548f157fd96 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml @@ -15,18 +15,24 @@ properties: enum: - rockchip,rk3288-dp - rockchip,rk3399-edp + - rockchip,rk3576-edp - rockchip,rk3588-edp clocks: minItems: 2 - maxItems: 3 + items: + - description: Reference clock + - description: APB bus clock + - description: GRF or AHB bus clock clock-names: minItems: 2 items: - const: dp - const: pclk - - const: grf + - enum: + - grf + - hclk power-domains: maxItems: 1 @@ -65,9 +71,46 @@ allOf: compatible: contains: enum: + - rockchip,rk3288-dp + then: + properties: + clocks: + maxItems: 2 + clock-names: + maxItems: 2 + + - if: + properties: + compatible: + contains: + enum: + - rockchip,rk3399-edp + then: + properties: + clocks: + minItems: 3 + clock-names: + items: + - const: dp + - const: pclk + - const: grf + + - if: + properties: + compatible: + contains: + enum: + - rockchip,rk3576-edp - rockchip,rk3588-edp then: properties: + clocks: + minItems: 3 + clock-names: + items: + - const: dp + - const: pclk + - const: hclk resets: minItems: 2 reset-names: diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml index ff685031bb2c..729705f419bb 100644 --- a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml +++ b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml @@ -41,7 +41,7 @@ properties: additionalProperties: false description: | Timing configuration for lcd i80 interface support. - The parameters are defined as:: + The parameters are defined as: VCLK(internal) __|??????|_____|??????|_____|??????|_____|??????|_____|?? : : : : : Address Output --: @@ -95,6 +98,12 @@ properties: eDMA are implemented in big endian mode, otherwise in little mode. type: boolean + dma-channel-mask: + description: | + Bitmask of available DMA channels (inverted definition). + Bit semantics: 0 means channel available, 1 means channel unavailable + default: 0 + required: - "#dma-cells" - compatible diff --git a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml index 3708518fe7fc..bba20e88a6dc 100644 --- a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml +++ b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml @@ -40,6 +40,7 @@ properties: - items: - enum: - mediatek,mt6991-uart-dma + - mediatek,mt8189-uart-dma - mediatek,mt8196-uart-dma - const: mediatek,mt6985-uart-dma - enum: diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml index 0dabe9bbb219..bc093c783d98 100644 --- a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml +++ b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml @@ -14,17 +14,16 @@ description: | maintainers: - Jon Hunter - Rajesh Gumasta - -allOf: - - $ref: dma-controller.yaml# + - Akhil R properties: compatible: oneOf: - - const: nvidia,tegra186-gpcdma + - enum: + - nvidia,tegra264-gpcdma + - nvidia,tegra186-gpcdma - items: - enum: - - nvidia,tegra264-gpcdma - nvidia,tegra234-gpcdma - nvidia,tegra194-gpcdma - const: nvidia,tegra186-gpcdma @@ -51,6 +50,14 @@ properties: iommus: maxItems: 1 + iommu-map: + description: + Maps DMA channel numbers to IOMMU stream IDs. A single entry can map all + channels when stream IDs are contiguous. In systems where the channels or + stream IDs are not contiguous, multiple entries may be needed. + minItems: 1 + maxItems: 32 + dma-coherent: true dma-channel-mask: @@ -60,12 +67,23 @@ required: - compatible - reg - interrupts - - resets - - reset-names - "#dma-cells" - iommus - dma-channel-mask +allOf: + - $ref: dma-controller.yaml# + - if: + properties: + compatible: + contains: + enum: + - nvidia,tegra186-gpcdma + then: + required: + - resets + - reset-names + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml index 6493a6968bb4..0923fb189ada 100644 --- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml @@ -23,6 +23,8 @@ properties: - qcom,bam-v1.4.0 # MSM8916, SDM630 - qcom,bam-v1.7.0 + # Kaanapali + - qcom,bam-v2.0.0 - items: - enum: # SDM845, SM6115, SM8150, SM8250 and QCM2290 @@ -118,4 +120,23 @@ examples: #dma-cells = <1>; qcom,ee = <0>; }; + - | + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + dma-controller@1dc4000 { + compatible = "qcom,bam-v2.0.0"; + reg = <0x0 0x01dc4000 0x0 0x22000>; + interrupts = ; + #dma-cells = <1>; + iommus = <&apps_smmu 0xc0 0>, <&apps_smmu 0xc1 0>; + qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <20>; + qcom,controlled-remotely; + }; + }; ... diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index fde1df035ad1..54dca623223d 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -24,7 +24,9 @@ properties: - qcom,sm6350-gpi-dma - items: - enum: + - qcom,eliza-gpi-dma - qcom,glymur-gpi-dma + - qcom,hawi-gpi-dma - qcom,kaanapali-gpi-dma - qcom,milos-gpi-dma - qcom,qcm2290-gpi-dma @@ -35,6 +37,7 @@ properties: - qcom,sc7280-gpi-dma - qcom,sc8280xp-gpi-dma - qcom,sdx75-gpi-dma + - qcom,shikra-gpi-dma - qcom,sm6115-gpi-dma - qcom,sm6375-gpi-dma - qcom,sm8350-gpi-dma diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml index 804514732dbe..0a30a455b0ee 100644 --- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml @@ -21,11 +21,12 @@ properties: - enum: - snps,axi-dma-1.01a - intel,kmb-axi-dma - - sophgo,cv1800b-axi-dma - starfive,jh7110-axi-dma - starfive,jh8100-axi-dma - items: - - const: altr,agilex5-axi-dma + - enum: + - altr,agilex5-axi-dma + - sophgo,cv1800b-axi-dma - const: snps,axi-dma-1.01a reg: diff --git a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml index ec06235baf5c..62ce6d81526b 100644 --- a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml +++ b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml @@ -14,7 +14,9 @@ allOf: properties: compatible: - const: spacemit,k1-pdma + enum: + - spacemit,k1-pdma + - spacemit,k3-pdma reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml index 340ae9e91cb0..2c88e927d9d3 100644 --- a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml +++ b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml @@ -81,8 +81,13 @@ properties: description: Should be the number of framebuffers as configured in h/w. xlnx,flush-fsync: - type: boolean - description: Tells which channel to Flush on Frame sync. + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2, 3] + description: + Tells which channel to flush on frame sync. + 1 - flush both channels + 2 - flush mm2s channel + 3 - flush s2mm channel xlnx,sg-length-width: $ref: /schemas/types.yaml#/definitions/uint32 @@ -93,7 +98,7 @@ properties: Width in bits of the length register as configured in hardware. xlnx,irq-delay: - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint8 minimum: 0 maximum: 255 description: @@ -251,7 +256,7 @@ examples: "m_axi_s2mm_aclk", "m_axis_mm2s_aclk", "s_axis_s2mm_aclk"; xlnx,num-fstores = <8>; - xlnx,flush-fsync; + xlnx,flush-fsync = <1>; xlnx,addrwidth = <32>; dma-channel-mm2s { diff --git a/Documentation/devicetree/bindings/embedded-controller/microsoft,surface-rt-ec.yaml b/Documentation/devicetree/bindings/embedded-controller/microsoft,surface-rt-ec.yaml new file mode 100644 index 000000000000..0fee574a3015 --- /dev/null +++ b/Documentation/devicetree/bindings/embedded-controller/microsoft,surface-rt-ec.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/embedded-controller/microsoft,surface-rt-ec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microsoft Surface RT fuel gauge and charger EC + +maintainers: + - Jonas Schwöbel + - Svyatoslav Ryhel + +description: + An Embedded Controller used in Microsoft Surface RT for monitoring + battery properties and charger status. + +allOf: + - $ref: /schemas/power/supply/power-supply.yaml# + +properties: + compatible: + const: microsoft,surface-rt-ec + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + enable-gpios: + maxItems: 1 + + monitored-battery: true + +required: + - compatible + - reg + - interrupts + - enable-gpios + +additionalProperties: false + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + embedded-controller@a { + compatible = "microsoft,surface-rt-ec"; + reg = <0x0a>; + + interrupt-parent = <&gpio>; + interrupts = <74 IRQ_TYPE_EDGE_RISING>; + + enable-gpios = <&gpio 88 GPIO_ACTIVE_HIGH>; + monitored-battery = <&battery>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml b/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml index d50438b0fca8..72af37cdb103 100644 --- a/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml +++ b/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml @@ -48,8 +48,7 @@ properties: const: 1 clock-controller: - $ref: /schemas/clock/xlnx,versal-clk.yaml# - description: The clock controller is a hardware block of Xilinx versal + description: The clock controller is a hardware block of Xilinx SoC clock tree. It reads required input clock frequencies from the devicetree and acts as clock provider for all clock consumers of PS clocks.list of clock specifiers which are external input clocks to the given clock @@ -113,10 +112,14 @@ allOf: const: xlnx,zynqmp-firmware then: properties: + clock-controller: + $ref: /schemas/clock/xlnx,zynqmp-clk.yaml# pinctrl: $ref: /schemas/pinctrl/xlnx,zynqmp-pinctrl.yaml# else: properties: + clock-controller: + $ref: /schemas/clock/xlnx,versal-clk.yaml# pinctrl: $ref: /schemas/pinctrl/xlnx,versal-pinctrl.yaml# @@ -132,6 +135,14 @@ examples: zynqmp_firmware: zynqmp-firmware { compatible = "xlnx,zynqmp-firmware"; #power-domain-cells = <1>; + clock-controller { + compatible = "xlnx,zynqmp-clk"; + clocks = <&pss_ref_clk>, <&video_clk>, <&pss_alt_ref_clk>, + <&aux_ref_clk>, <>_crx_ref_clk>; + clock-names = "pss_ref_clk", "video_clk", "pss_alt_ref_clk", + "aux_ref_clk", "gt_crx_ref_clk"; + #clock-cells = <1>; + }; soc-nvmem { compatible = "xlnx,zynqmp-nvmem-fw"; nvmem-layout { diff --git a/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt b/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt deleted file mode 100644 index 52a294cf2730..000000000000 --- a/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt +++ /dev/null @@ -1,12 +0,0 @@ -Altera Arria10 Partial Reconfiguration IP - -Required properties: -- compatible : should contain "altr,a10-pr-ip" -- reg : base address and size for memory mapped io. - -Example: - - fpga_mgr: fpga-mgr@ff20c000 { - compatible = "altr,a10-pr-ip"; - reg = <0xff20c000 0x10>; - }; diff --git a/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt b/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt deleted file mode 100644 index d52f3340414d..000000000000 --- a/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt +++ /dev/null @@ -1,17 +0,0 @@ -Altera SOCFPGA FPGA Manager - -Required properties: -- compatible : should contain "altr,socfpga-fpga-mgr" -- reg : base address and size for memory mapped io. - - The first index is for FPGA manager register access. - - The second index is for writing FPGA configuration data. -- interrupts : interrupt for the FPGA Manager device. - -Example: - - hps_0_fpgamgr: fpgamgr@ff706000 { - compatible = "altr,socfpga-fpga-mgr"; - reg = <0xFF706000 0x1000 - 0xFFB90000 0x1000>; - interrupts = <0 175 4>; - }; diff --git a/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml b/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml new file mode 100644 index 000000000000..1f4df40308bd --- /dev/null +++ b/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/fpga/altr,a10-pr-ip.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera Arria10 Partial Reconfiguration IP + +maintainers: + - Matthew Gerlach + +description: + The Altera Arria 10 Partial Reconfiguration IP core allows the host + processor to perform partial reconfiguration of the FPGA fabric. + +properties: + compatible: + const: altr,a10-pr-ip + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + fpga-mgr@ff20c000 { + compatible = "altr,a10-pr-ip"; + reg = <0xff20c000 0x10>; + }; diff --git a/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml b/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml new file mode 100644 index 000000000000..9bcc1200d61d --- /dev/null +++ b/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/fpga/altr,socfpga-fpga-mgr.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera SOCFPGA FPGA Manager + +maintainers: + - Steffen Trumtrar + +properties: + compatible: + const: altr,socfpga-fpga-mgr + + reg: + items: + - description: FPGA manager register access + - description: Writing FPGA configuration data + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + fpgamgr@ff706000 { + compatible = "altr,socfpga-fpga-mgr"; + reg = <0xff706000 0x1000>, + <0xffb90000 0x1000>; + interrupts = <0 175 4>; + }; diff --git a/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml b/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml new file mode 100644 index 000000000000..7c7444ff9c3a --- /dev/null +++ b/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/fpga/efinix,trion-config.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Efinix SPI FPGA Manager + +maintainers: + - Ian Dannapel + +description: | + Efinix FPGAs (Trion, Topaz, and Titanium families) support loading bitstreams + through "SPI Passive Mode". + Additional pin hogs for bus width configuration should be set + elsewhere, if necessary. + + References: + - https://www.efinixinc.com/docs/an006-configuring-trion-fpgas-v6.3.pdf + - https://www.efinixinc.com/docs/an033-configuring-titanium-fpgas-v2.8.pdf + - https://www.efinixinc.com/docs/an061-configuring-topaz-fpgas-v1.1.pdf + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + oneOf: + - items: + - enum: + - efinix,titanium-config + - efinix,topaz-config + - const: efinix,trion-config + - const: efinix,trion-config + + spi-cpha: true + + spi-cpol: true + + spi-max-frequency: + maximum: 25000000 + + reg: + maxItems: 1 + + reset-gpios: + description: + reset and re-configuration trigger pin (low active) + maxItems: 1 + + cdone-gpios: + description: + optional configuration done status pin (high active) + maxItems: 1 + +required: + - compatible + - reg + - reset-gpios + - spi-cpha + - spi-cpol + +unevaluatedProperties: false + +examples: + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; + fpga-mgr@0 { + compatible = "efinix,trion-config"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-cpha; + spi-cpol; + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + cdone-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>; + }; + }; + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; + fpga-mgr@0 { + compatible = "efinix,titanium-config", "efinix,trion-config"; + reg = <0>; + spi-max-frequency = <25000000>; + spi-cpha; + spi-cpol; + reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>; + cdone-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml b/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml new file mode 100644 index 000000000000..c93e3a1a135b --- /dev/null +++ b/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/fpga/technologic,ts7300-fpga.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Technologic Systems TS-7300 FPGA Manager + +maintainers: + - Florian Fainelli + +description: + FPGA manager for the Altera Cyclone II FPGA on Technologic Systems + TS-7300 board. The FPGA is programmed via the memory-mapped interface + implemented in the CPLD. + +properties: + compatible: + const: technologic,ts7300-fpga + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + fpga-mgr@13c00000 { + compatible = "technologic,ts7300-fpga"; + reg = <0x13c00000 0x2>; + }; +... diff --git a/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml new file mode 100644 index 000000000000..f13c910b73c6 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright 2023 Realtek Semiconductor Corporation +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/realtek,rtd1625-gpio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Realtek DHC RTD1625 GPIO controller + +maintainers: + - Tzuyi Chang + +description: | + GPIO controller for the Realtek RTD1625 SoC, featuring a per-pin register + architecture that differs significantly from earlier RTD series controllers. + Each GPIO has dedicated registers for configuration (direction, input/output + values, debounce), and interrupt control supporting edge and level detection + modes. + +properties: + compatible: + enum: + - realtek,rtd1625-iso-gpio + - realtek,rtd1625-isom-gpio + + reg: + maxItems: 1 + + interrupts: + items: + - description: Interrupt number of the assert GPIO interrupt, which is + triggered when there is a rising edge. + - description: Interrupt number of the deassert GPIO interrupt, which is + triggered when there is a falling edge. + - description: Interrupt number of the level-sensitive GPIO interrupt, + triggered by a configured logic level. + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + gpio-ranges: true + + gpio-controller: true + + "#gpio-cells": + const: 2 + +required: + - compatible + - reg + - gpio-ranges + - gpio-controller + - "#gpio-cells" + +additionalProperties: false + +examples: + - | + gpio@89100 { + compatible = "realtek,rtd1625-isom-gpio"; + reg = <0x89100 0x30>; + interrupt-parent = <&iso_m_irq_mux>; + interrupts = <0>, <1>, <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&isom_pinctrl 0 0 4>; + gpio-controller; + #gpio-cells = <2>; + }; diff --git a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml index a1f54dbae3f3..a6bccb600a57 100644 --- a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml +++ b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml @@ -8,7 +8,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Imagination Technologies PowerVR and IMG Rogue GPUs maintainers: - - Frank Binns + - Alessio Belle + - Luigi Santivetti properties: compatible: diff --git a/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml b/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml index f5898b04381c..58b799b8e96e 100644 --- a/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml +++ b/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml @@ -9,7 +9,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Imagination Technologies PowerVR SGX GPUs maintainers: - - Frank Binns + - Alessio Belle + - Luigi Santivetti properties: compatible: diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml index d6a7517f2a50..283cd1662689 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml @@ -25,19 +25,38 @@ description: | https://www.silergy.com/ download/downloadFile?id=5669&type=product&ftype=note + The BD12780 and BD12780A are hot-swap controllers from ROHM. They are + functionally compatible with the ADM1278. The main difference between + the BD12780A and the BD12780 is amount of configurable I2C addresses. + + Datasheets: + https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780muv-lb-e.pdf + https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780amuv-lb-e.pdf + + The BD12790 is a ROHM hot-swap controller, functionally similar to the + ADM1272. + properties: compatible: - enum: - - adi,adm1075 - - adi,adm1272 - - adi,adm1273 - - adi,adm1275 - - adi,adm1276 - - adi,adm1278 - - adi,adm1281 - - adi,adm1293 - - adi,adm1294 - - silergy,mc09c + oneOf: + - enum: + - adi,adm1075 + - adi,adm1272 + - adi,adm1273 + - adi,adm1275 + - adi,adm1276 + - adi,adm1278 + - adi,adm1281 + - adi,adm1293 + - adi,adm1294 + - rohm,bd12780 + - rohm,bd12790 + - silergy,mc09c + + # Require BD12780 as a fall-back for BD12780A. + - items: + - const: rohm,bd12780a + - const: rohm,bd12780 reg: maxItems: 1 @@ -88,6 +107,7 @@ allOf: enum: - adi,adm1272 - adi,adm1273 + - rohm,bd12790 then: properties: adi,volt-curr-sample-average: @@ -104,6 +124,7 @@ allOf: - adi,adm1281 - adi,adm1293 - adi,adm1294 + - rohm,bd12780 - silergy,mc09c then: properties: diff --git a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml index 17351fdbefce..56b0cecfca5f 100644 --- a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml +++ b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml @@ -45,6 +45,8 @@ properties: - const: low - const: high + label: true + vdd-supply: description: Dedicated, controllable supply-regulator to reset the device and @@ -55,6 +57,9 @@ required: - reg - vdd-supply +allOf: + - $ref: hwmon-common.yaml# + additionalProperties: false examples: @@ -72,6 +77,7 @@ examples: <5 IRQ_TYPE_EDGE_RISING>, <6 IRQ_TYPE_EDGE_RISING>; interrupt-names = "ready", "low", "high"; + label = "Room"; vdd-supply = <®_vdd>; }; }; diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml new file mode 100644 index 000000000000..03ef82c11e1a --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- + +$id: http://devicetree.org/schemas/hwmon/pmbus/silergy,sq24860.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Silergy SQ24860 eFuse + +maintainers: + - Ziming Zhu + +description: + The Silergy SQ24860 is an integrated, high-current circuit protection and + power management device with PMBus interface. + +properties: + compatible: + const: silergy,sq24860 + + reg: + maxItems: 1 + + silergy,rimon-micro-ohms: + description: + Micro-ohms value of the resistance installed between the IMON pin and + the ground reference. + + interrupts: + description: PMBus SMBAlert interrupt. + maxItems: 1 + + regulators: + type: object + description: + List of regulators provided by this controller. + + properties: + vout: + $ref: /schemas/regulator/regulator.yaml# + type: object + unevaluatedProperties: false + + additionalProperties: false + +required: + - compatible + - reg + - silergy,rimon-micro-ohms + +additionalProperties: false + +examples: + - | + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + hw-monitor@40 { + compatible = "silergy,sq24860"; + reg = <0x40>; + + interrupt-parent = <&gpio>; + interrupts = <42 8>; + silergy,rimon-micro-ohms = <1600000000>; + + regulators { + cpu0_vout: vout { + regulator-name = "main_cpu0"; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml index a20f140dc79a..fe42daabaaa8 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml @@ -46,12 +46,32 @@ properties: additionalProperties: false + ti,current-range: + description: | + Configure the current limit setting. When present, this property + overrides the hardware setting of the physical CL pin by configuring + the DEVICE_SETUP register. + - "low": maps to 25 mV (LM25066) or 26 mV (LM5064, LM5066, LM5066i) + - "high": maps to 46 mV (LM25066) or 50 mV (LM5064, LM5066, LM5066i) + $ref: /schemas/types.yaml#/definitions/string + enum: + - low + - high + required: - compatible - reg allOf: - $ref: /schemas/hwmon/hwmon-common.yaml# + - if: + properties: + compatible: + contains: + const: ti,lm25056 + then: + properties: + ti,current-range: false unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index 009d78b30859..a79ade3142fa 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -28,6 +28,7 @@ properties: - ti,ina228 - ti,ina230 - ti,ina231 + - ti,ina232 - ti,ina233 - ti,ina234 - ti,ina237 @@ -114,6 +115,7 @@ allOf: - ti,ina228 - ti,ina230 - ti,ina231 + - ti,ina232 - ti,ina234 - ti,ina237 - ti,ina238 @@ -136,6 +138,7 @@ allOf: - ti,ina226 - ti,ina230 - ti,ina231 + - ti,ina232 - ti,ina234 - ti,ina260 - ti,ina700 diff --git a/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml b/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml new file mode 100644 index 000000000000..d04570680c05 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/altr,softip-i2c-v1.0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera I2C Controller + +maintainers: + - Thor Thayer + - Chaitanya Sabnis + +description: + Altera's synthesizable logic block I2C Controller for use in Altera's FPGAs. + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +properties: + compatible: + const: altr,softip-i2c-v1.0 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + fifo-size: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Size of the RX and TX FIFOs in bytes. + +required: + - compatible + - reg + - interrupts + - clocks + +unevaluatedProperties: false + +examples: + - | + i2c@80000 { + compatible = "altr,softip-i2c-v1.0"; + reg = <0x80000 0x40>; + interrupt-parent = <&intc>; + interrupts = <0 43 4>; + clocks = <&clk_0>; + clock-frequency = <100000>; + #address-cells = <1>; + #size-cells = <0>; + fifo-size = <4>; + + eeprom@51 { + compatible = "atmel,24c32"; + reg = <0x51>; + pagesize = <32>; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-altera.txt b/Documentation/devicetree/bindings/i2c/i2c-altera.txt deleted file mode 100644 index 767664f448ec..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-altera.txt +++ /dev/null @@ -1,39 +0,0 @@ -* Altera I2C Controller -* This is Altera's synthesizable logic block I2C Controller for use -* in Altera's FPGAs. - -Required properties : - - compatible : should be "altr,softip-i2c-v1.0" - - reg : Offset and length of the register set for the device - - interrupts : where IRQ is the interrupt number. - - clocks : phandle to input clock. - - #address-cells = <1>; - - #size-cells = <0>; - -Recommended properties : - - clock-frequency : desired I2C bus clock frequency in Hz. - -Optional properties : - - fifo-size : Size of the RX and TX FIFOs in bytes. - - Child nodes conforming to i2c bus binding - -Example : - - i2c@100080000 { - compatible = "altr,softip-i2c-v1.0"; - reg = <0x00000001 0x00080000 0x00000040>; - interrupt-parent = <&intc>; - interrupts = <0 43 4>; - clocks = <&clk_0>; - clock-frequency = <100000>; - #address-cells = <1>; - #size-cells = <0>; - fifo-size = <4>; - - eeprom@51 { - compatible = "atmel,24c32"; - reg = <0x51>; - pagesize = <32>; - }; - }; - diff --git a/Documentation/devicetree/bindings/i2c/i2c-axxia.txt b/Documentation/devicetree/bindings/i2c/i2c-axxia.txt deleted file mode 100644 index 7d53a2b79553..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-axxia.txt +++ /dev/null @@ -1,30 +0,0 @@ -LSI Axxia I2C - -Required properties : -- compatible : Must be "lsi,api2c" -- reg : Offset and length of the register set for the device -- interrupts : the interrupt specifier -- #address-cells : Must be <1>; -- #size-cells : Must be <0>; -- clock-names : Must contain "i2c". -- clocks: Must contain an entry for each name in clock-names. See the common - clock bindings. - -Optional properties : -- clock-frequency : Desired I2C bus clock frequency in Hz. If not specified, - the default 100 kHz frequency will be used. As only Normal and Fast modes - are supported, possible values are 100000 and 400000. - -Example : - -i2c@2010084000 { - compatible = "lsi,api2c"; - device_type = "i2c"; - #address-cells = <1>; - #size-cells = <0>; - reg = <0x20 0x10084000 0x00 0x1000>; - interrupts = <0 19 4>; - clocks = <&clk_per>; - clock-names = "i2c"; - clock-frequency = <400000>; -}; diff --git a/Documentation/devicetree/bindings/i2c/i2c-mux-pinctrl.yaml b/Documentation/devicetree/bindings/i2c/i2c-mux-pinctrl.yaml index 2e3d555eb96c..99812a893476 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-mux-pinctrl.yaml +++ b/Documentation/devicetree/bindings/i2c/i2c-mux-pinctrl.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Pinctrl-based I2C Bus Mux maintainers: - - Wolfram Sang + - Thierry Reding description: | This binding describes an I2C bus multiplexer that uses pin multiplexing to route the I2C diff --git a/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt b/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt deleted file mode 100644 index b9d9755e4172..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt +++ /dev/null @@ -1,74 +0,0 @@ -Register-based I2C Bus Mux - -This binding describes an I2C bus multiplexer that uses a single register -to route the I2C signals. - -Required properties: -- compatible: i2c-mux-reg -- i2c-parent: The phandle of the I2C bus that this multiplexer's master-side - port is connected to. -* Standard I2C mux properties. See i2c-mux.yaml in this directory. -* I2C child bus nodes. See i2c-mux.yaml in this directory. - -Optional properties: -- reg: this pair of specifies the register to control the mux. - The depends on its parent node. It can be any memory-mapped - address. The size must be either 1, 2, or 4 bytes. If reg is omitted, the - resource of this device will be used. -- little-endian: The existence indicates the register is in little endian. -- big-endian: The existence indicates the register is in big endian. - If both little-endian and big-endian are omitted, the endianness of the - CPU will be used. -- write-only: The existence indicates the register is write-only. -- idle-state: value to set the muxer to when idle. When no value is - given, it defaults to the last value used. - -Whenever an access is made to a device on a child bus, the value set -in the relevant node's reg property will be output to the register. - -If an idle state is defined, using the idle-state (optional) property, -whenever an access is not being made to a device on a child bus, the -register will be set according to the idle value. - -If an idle state is not defined, the most recently used value will be -left programmed into the register. - -Example of a mux on PCIe card, the host is a powerpc SoC (big endian): - - i2c-mux { - /* the depends on the address translation - * of the parent device. If omitted, device resource - * will be used instead. The size is to determine - * whether iowrite32, iowrite16, or iowrite8 will be used. - */ - reg = <0x6028 0x4>; - little-endian; /* little endian register on PCIe */ - compatible = "i2c-mux-reg"; - #address-cells = <1>; - #size-cells = <0>; - i2c-parent = <&i2c1>; - i2c@0 { - reg = <0>; - #address-cells = <1>; - #size-cells = <0>; - - si5338: clock-generator@70 { - compatible = "silabs,si5338"; - reg = <0x70>; - /* other stuff */ - }; - }; - - i2c@1 { - /* data is written using iowrite32 */ - reg = <1>; - #address-cells = <1>; - #size-cells = <0>; - - si5338: clock-generator@70 { - compatible = "silabs,si5338"; - reg = <0x70>; - /* other stuff */ - }; - }; - }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-mux-reg.yaml b/Documentation/devicetree/bindings/i2c/i2c-mux-reg.yaml new file mode 100644 index 000000000000..01ade0771c60 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c-mux-reg.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/i2c-mux-reg.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Register-based I2C Bus Mux + +maintainers: + - Peter Rosin + +description: | + This binding describes an I2C bus multiplexer that uses a single + memory-mapped register to route the I2C signals. + + Whenever an access is made to a device on a child bus, the value + set in the relevant node's reg property is output to the register. + + If an idle state is defined via the idle-state property, the + register is set to that value whenever no access is being made. + Otherwise the most recently used value is left programmed. + +allOf: + - $ref: /schemas/i2c/i2c-mux.yaml# + +properties: + compatible: + const: i2c-mux-reg + + reg: + maxItems: 1 + description: | + Offset and size of the register that selects the active child + bus, relative to the parent node's address space. The size + determines the access width and must be 1, 2, or 4 bytes. If + omitted, the platform device's own memory resource is used + instead. + + i2c-parent: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle of the I2C bus that this multiplexer's master-side port + is connected to. + + little-endian: + type: boolean + description: Register is accessed in little-endian byte order. + + big-endian: + type: boolean + description: Register is accessed in big-endian byte order. + + write-only: + type: boolean + description: + Register is write-only; the driver must not read back the + current selection. + + idle-state: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Value to write to the register when no child bus is selected. + +required: + - compatible + - i2c-parent + +unevaluatedProperties: false + +examples: + - | + i2c-mux@6028 { + compatible = "i2c-mux-reg"; + reg = <0x6028 0x4>; + little-endian; + #address-cells = <1>; + #size-cells = <0>; + i2c-parent = <&i2c1>; + + i2c@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml b/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml new file mode 100644 index 000000000000..2d1c3069c3a3 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/lsi,api2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: LSI Axxia I2C Controller + +maintainers: + - Anders Berg + - Chaitanya Sabnis + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +properties: + compatible: + const: lsi,api2c + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: i2c + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +unevaluatedProperties: false + +examples: + - | + i2c@10084000 { + compatible = "lsi,api2c"; + reg = <0x10084000 0x1000>; + interrupts = <0 19 4>; + clocks = <&clk_per>; + clock-names = "i2c"; + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml b/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml index 6ff58b64d496..bd63c70aac6b 100644 --- a/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml +++ b/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml @@ -37,6 +37,9 @@ properties: modes are supported, possible values are 100000 and 400000. enum: [100000, 400000] + resets: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml b/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml new file mode 100644 index 000000000000..04caf622b3ba --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/qcom,sa8255p-geni-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm SA8255p QUP GENI I2C Controller + +maintainers: + - Praveen Talari + +properties: + compatible: + oneOf: + - const: qcom,sa8255p-geni-i2c + - items: + - const: qcom,sa8797p-geni-i2c + - const: qcom,sa8255p-geni-i2c + + reg: + maxItems: 1 + + dmas: + maxItems: 2 + + dma-names: + items: + - const: tx + - const: rx + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 2 + + power-domain-names: + items: + - const: power + - const: perf + +required: + - compatible + - reg + - interrupts + - power-domains + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + + i2c@a90000 { + compatible = "qcom,sa8255p-geni-i2c"; + reg = <0xa90000 0x4000>; + interrupts = ; + dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>, + <&gpi_dma0 1 0 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; + power-domains = <&scmi0_pd 0>, <&scmi0_dvfs 0>; + power-domain-names = "power", "perf"; + }; +... diff --git a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml index a2ddc6803617..07600b49f2f9 100644 --- a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml @@ -35,7 +35,7 @@ properties: gpios: description: | - The order of the GPIOs should be the following:: . The GPIO + The order of the GPIOs should be the following: . The GPIO specifier depends on the gpio controller. Required in all cases except for "samsung,s3c2440-hdmiphy-i2c" whose input/output lines are permanently wired to the respective client. diff --git a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml index 39bb1a1784c9..d488fb420945 100644 --- a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml +++ b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml @@ -9,9 +9,6 @@ title: MIPI I3C HCI maintainers: - Nicolas Pitre -allOf: - - $ref: /schemas/i3c/i3c.yaml# - description: | MIPI I3C Host Controller Interface @@ -28,9 +25,17 @@ description: | properties: compatible: - const: mipi-i3c-hci + enum: + - mipi-i3c-hci + - microchip,sama7d65-i3c-hci reg: maxItems: 1 + + clocks: + items: + - description: Peripheral bus clock + - description: System Generic clock + interrupts: maxItems: 1 @@ -39,6 +44,20 @@ required: - reg - interrupts +allOf: + - $ref: /schemas/i3c/i3c.yaml# + - if: + properties: + compatible: + contains: + const: microchip,sama7d65-i3c-hci + then: + required: + - clocks + else: + properties: + clocks: false + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml index 79df2696ef24..4a3f7d3e05c3 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml @@ -18,7 +18,11 @@ description: | service a wide variety of precision, wide bandwidth data acquisition applications. + The AD4880 is a dual-channel variant with two independent ADC channels, + each with its own SPI configuration interface. + https://www.analog.com/media/en/technical-documentation/data-sheets/ad4080.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/ad4880.pdf $ref: /schemas/spi/spi-peripheral-props.yaml# @@ -34,9 +38,16 @@ properties: - adi,ad4086 - adi,ad4087 - adi,ad4088 + - adi,ad4880 + - adi,ad4884 reg: - maxItems: 1 + minItems: 1 + maxItems: 2 + description: + SPI chip select(s). For single-channel devices, one chip select. + For multi-channel devices like AD4880, two chip selects are required + as each channel has its own SPI configuration interface. spi-max-frequency: description: Configuration of the SPI bus. @@ -60,7 +71,10 @@ properties: vrefin-supply: true io-backends: - maxItems: 1 + minItems: 1 + items: + - description: Backend for channel A (primary) + - description: Backend for channel B (secondary) adi,lvds-cnv-enable: description: Enable the LVDS signal type on the CNV pin. Default is CMOS. @@ -81,6 +95,27 @@ required: - vdd33-supply - vrefin-supply +allOf: + - if: + properties: + compatible: + contains: + enum: + - adi,ad4880 + - adi,ad4884 + then: + properties: + reg: + minItems: 2 + io-backends: + minItems: 2 + else: + properties: + reg: + maxItems: 1 + io-backends: + maxItems: 1 + additionalProperties: false examples: @@ -101,4 +136,21 @@ examples: io-backends = <&iio_backend>; }; }; + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + + adc@0 { + compatible = "adi,ad4880"; + reg = <0>, <1>; + spi-max-frequency = <10000000>; + vdd33-supply = <&vdd33>; + vddldo-supply = <&vddldo>; + vrefin-supply = <&vrefin>; + clocks = <&cnv>; + clock-names = "cnv"; + io-backends = <&iio_backend_cha>, <&iio_backend_chb>; + }; + }; ... diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml index d00690a8d3fb..cc38617bb829 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml @@ -5,19 +5,30 @@ $id: http://devicetree.org/schemas/iio/adc/adi,ad4130.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices AD4130 ADC device driver +title: Analog Devices AD4130 family ADCs maintainers: - Cosmin Tanislav description: | - Bindings for the Analog Devices AD4130 ADC. Datasheet can be found here: + Bindings for the Analog Devices AD4130 family ADCs. + Datasheets can be found here: + https://www.analog.com/media/en/technical-documentation/data-sheets/AD4129-4.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/AD4129-8.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/AD4130-4.pdf https://www.analog.com/media/en/technical-documentation/data-sheets/AD4130-8.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/AD4131-4.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/AD4131-8.pdf properties: compatible: enum: + - adi,ad4129-4 + - adi,ad4129-8 + - adi,ad4130-4 - adi,ad4130 + - adi,ad4131-4 + - adi,ad4131-8 reg: maxItems: 1 @@ -32,6 +43,10 @@ properties: interrupts: maxItems: 1 + description: | + Data Ready / FIFO interrupt. For devices with FIFO support, the + interrupt polarity specified here is inverted when the device enters + FIFO mode, and normal for data ready. interrupt-names: description: | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml new file mode 100644 index 000000000000..af28a0c1cfa9 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/adc/adi,ad4691.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices AD4691 Family Multichannel SAR ADCs + +maintainers: + - Radu Sabau + +description: | + The AD4691 family are high-speed, low-power, multichannel successive + approximation register (SAR) analog-to-digital converters (ADCs) with + an SPI-compatible serial interface. The ADC supports CNV Burst Mode, + where an external PWM drives the CNV pin, and Manual Mode, where CNV + is directly tied to the SPI chip-select. + + Datasheets: + * https://www.analog.com/en/products/ad4691.html + * https://www.analog.com/en/products/ad4692.html + * https://www.analog.com/en/products/ad4693.html + * https://www.analog.com/en/products/ad4694.html + +$ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + enum: + - adi,ad4691 + - adi,ad4692 + - adi,ad4693 + - adi,ad4694 + + reg: + maxItems: 1 + + spi-max-frequency: + maximum: 40000000 + + spi-cpol: true + spi-cpha: true + + avdd-supply: + description: Analog power supply (4.5V to 5.5V). + + vdd-supply: + description: + External 1.8V digital core supply. When present, the internal LDO is + disabled (LDO_EN = 0). Mutually exclusive with ldo-in-supply. + + ldo-in-supply: + description: + LDO input supply (2.4V to 5.5V). When present and vdd-supply is absent, + the internal LDO generates 1.8V VDD from this input (LDO_EN = 1). + Mutually exclusive with vdd-supply. + + vio-supply: + description: I/O voltage supply (1.71V to 1.89V or VDD). + + ref-supply: + description: External reference voltage supply (2.4V to 5.25V). + + refin-supply: + description: Internal reference buffer input supply. + + reset-gpios: + description: + GPIO line controlling the hardware reset pin (active-low). + maxItems: 1 + + pwms: + description: + PWM connected to the CNV pin. When present, selects CNV Burst Mode where + the PWM drives the conversion rate. When absent, Manual Mode is used + (CNV tied to SPI CS). + maxItems: 1 + + interrupts: + description: + Interrupt lines connected to the ADC GP pins. Each GP pin can be + physically wired to an interrupt-capable input on the SoC. + maxItems: 4 + + interrupt-names: + description: Names of the interrupt lines, matching the GP pin names. + minItems: 1 + maxItems: 4 + items: + enum: + - gp0 + - gp1 + - gp2 + - gp3 + + gpio-controller: true + + '#gpio-cells': + const: 2 + + '#trigger-source-cells': + description: + This node can act as a trigger source. The single cell in a consumer + reference specifies the GP pin number (0-3) used as the trigger output. + const: 1 + +required: + - compatible + - reg + - avdd-supply + - vio-supply + +allOf: + # vdd-supply and ldo-in-supply are mutually exclusive, one is required: + # either an external 1.8V VDD is provided or the internal LDO is fed from + # ldo-in-supply to generate VDD. + - oneOf: + - required: + - vdd-supply + - required: + - ldo-in-supply + # ref-supply and refin-supply are mutually exclusive, one is required + - oneOf: + - required: + - ref-supply + - required: + - refin-supply + +unevaluatedProperties: false + +examples: + - | + #include + /* AD4692 in CNV Burst Mode with SPI offload */ + spi { + #address-cells = <1>; + #size-cells = <0>; + + adc@0 { + compatible = "adi,ad4692"; + reg = <0>; + spi-cpol; + spi-cpha; + spi-max-frequency = <40000000>; + + avdd-supply = <&avdd_supply>; + ldo-in-supply = <&avdd_supply>; + vio-supply = <&vio_supply>; + ref-supply = <&ref_5v>; + + reset-gpios = <&gpio0 15 GPIO_ACTIVE_LOW>; + + pwms = <&pwm_gen 0 0>; + + #trigger-source-cells = <1>; + }; + }; + + - | + #include + /* AD4692 in Manual Mode (CNV tied to SPI CS) */ + spi { + #address-cells = <1>; + #size-cells = <0>; + + adc@0 { + compatible = "adi,ad4692"; + reg = <0>; + spi-cpol; + spi-cpha; + spi-max-frequency = <31250000>; + + avdd-supply = <&avdd_supply>; + ldo-in-supply = <&avdd_supply>; + vio-supply = <&vio_supply>; + refin-supply = <&refin_supply>; + + reset-gpios = <&gpio0 15 GPIO_ACTIVE_LOW>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml b/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml index da605a051b94..6467800d30e2 100644 --- a/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml @@ -14,6 +14,7 @@ properties: oneOf: - enum: - allwinner,sun20i-d1-gpadc + - allwinner,sun55i-a523-gpadc - items: - enum: - allwinner,sun50i-h616-gpadc @@ -29,7 +30,12 @@ properties: const: 0 clocks: - maxItems: 1 + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + maxItems: 2 interrupts: maxItems: 1 @@ -40,6 +46,30 @@ properties: resets: maxItems: 1 +allOf: + - if: + properties: + compatible: + enum: + - allwinner,sun55i-a523-gpadc + then: + properties: + clocks: + items: + - description: Bus clock + - description: Module clock + clock-names: + items: + - const: bus + - const: mod + required: + - clock-names + else: + properties: + clocks: + maxItems: 1 + clock-names: false + patternProperties: "^channel@[0-9a-f]+$": $ref: adc.yaml diff --git a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml index 9c57eb13f892..099235b5fd54 100644 --- a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml +++ b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml @@ -10,14 +10,9 @@ maintainers: - Andreas Klinger description: | - Bit-banging driver using two GPIOs: - - sck-gpio gives a clock to the sensor with 24 cycles for data retrieval - and up to 3 cycles for selection of the input channel and gain for the - next measurement - - dout-gpio is the sensor data the sensor responds to the clock - - Specifications about the driver can be found at: - http://www.aviaic.com/ENProducts.aspx + The HX711 is a 24-bit ADC with selectable gain (32/64/128) and two + differential input channels. Channel A supports gain 64 and 128; + channel B supports gain 32. properties: compatible: @@ -26,23 +21,43 @@ properties: sck-gpios: description: - Definition of the GPIO for the clock (output). In the datasheet it is - named PD_SCK + GPIO for the clock output (PD_SCK in the datasheet). maxItems: 1 dout-gpios: description: - Definition of the GPIO for the data-out sent by the sensor in - response to the clock (input). - See Documentation/devicetree/bindings/gpio/gpio.txt for information - on how to specify a consumer gpio. + GPIO for the data output from the sensor (DOUT in the datasheet). maxItems: 1 avdd-supply: description: - Definition of the regulator used as analog supply + Analog supply voltage (AVDD). + + vsup-supply: + description: + Supply voltage for the on-chip regulator (VSUP). + + dvdd-supply: + description: + Digital supply voltage (DVDD). On the HX711, DVDD is a + board-supplied digital rail; it was missing from the original + binding. On the HX710B, DVDD must be greater than or equal to + AVDD; when DVDD is a battery rail and AVDD is a regulated supply, + one channel monitors the DVDD-AVDD difference for battery level + detection. + + rate-gpios: + description: + GPIO connected to the RATE pin. When driven low the output data + rate is 10 SPS; when driven high it is 80 SPS. If omitted the + RATE pin state is determined by the board wiring. + maxItems: 1 clock-frequency: + description: + Controls the SCK bit-bang timing. The value is used to derive the + delay between SCK edges; keep the SCK high time below 60 us to + avoid triggering chip power-down mode. minimum: 20000 maximum: 2500000 default: 400000 @@ -62,6 +77,7 @@ examples: compatible = "avia,hx711"; sck-gpios = <&gpio3 10 GPIO_ACTIVE_HIGH>; dout-gpios = <&gpio0 7 GPIO_ACTIVE_HIGH>; + rate-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>; avdd-supply = <&avdd>; clock-frequency = <100000>; }; diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml index c978c3a3e31a..63aac8de22ad 100644 --- a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml @@ -78,6 +78,10 @@ patternProperties: reg: maxItems: 1 + label: + description: | + Unique name to identify which channel this is. + qcom,decimation: $ref: /schemas/types.yaml#/definitions/uint32 description: | @@ -130,36 +134,47 @@ examples: vcoin: adc-channel@0 { reg = <0x00 0x00>; + label = "vcoin"; }; vbat: adc-channel@1 { reg = <0x00 0x01>; + label = "vbat"; }; dcin: adc-channel@2 { reg = <0x00 0x02>; + label = "dcin"; }; ichg: adc-channel@3 { reg = <0x00 0x03>; + label = "ichg"; }; vph_pwr: adc-channel@4 { reg = <0x00 0x04>; + label = "vph_pwr"; }; usb_vbus: adc-channel@a { reg = <0x00 0x0a>; + label = "usb_vbus"; }; die_temp: adc-channel@b { reg = <0x00 0x0b>; + label = "die_temp"; }; ref_625mv: adc-channel@c { reg = <0x00 0x0c>; + label = "ref_625mv"; }; ref_1250mv: adc-channel@d { reg = <0x00 0x0d>; + label = "ref_1250mv"; }; ref_325mv: adc-channel@e { reg = <0x00 0x0e>; + label = "ref_325mv"; }; ref_muxoff: adc-channel@f { reg = <0x00 0x0f>; + label = "ref_muxoff"; }; }; }; diff --git a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml index 40d87346ff4c..a5b0debe85b1 100644 --- a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml +++ b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Sensirion SCD30 carbon dioxide sensor maintainers: - - Tomasz Duszynski + - Maxwell Doose description: | Air quality sensor capable of measuring co2 concentration, temperature diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml index 713f535bb33a..02e8c78e36d3 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml @@ -32,27 +32,95 @@ properties: reg: maxItems: 1 + vdd-supply: + description: Input power supply. + + vlogic-supply: + description: + Digital power supply. On some tiny package variants for single-channel + devices, this supply is internally connected to vdd; in that case, specify + this property with the same regulator as vdd. + + vref-supply: + description: + Reference voltage supply. If not supplied the internal reference is used. + vcc-supply: - description: If not supplied the internal reference is used. + deprecated: true + description: Use vref-supply instead. + + reset-gpios: + description: Active-low RESET pin to reset the device. + maxItems: 1 + + ldac-gpios: + description: + Active-low LDAC pin used to asynchronously update the DAC channels. + maxItems: 1 + + gain-gpios: + description: + GAIN pin that sets a multiplier for the DAC output voltage. When high, + the DAC output voltage is multiplied by 2, otherwise it is unchanged. + maxItems: 1 + + adi,range-double: + description: + Sets the initial voltage output range from 0 to 2xVREF. On devices that + have a GAIN pin and no gain-gpios property is set, this indicates the pin + is hardwired high. + type: boolean required: - compatible - reg + - vdd-supply + - vlogic-supply allOf: - $ref: /schemas/spi/spi-peripheral-props.yaml# + - if: + properties: + compatible: + contains: + enum: + - adi,ad5676 + - adi,ad5683 + - adi,ad5684 + - adi,ad5686 + then: + required: + - vref-supply + - if: + properties: + compatible: + contains: + enum: + - adi,ad5310r + - adi,ad5681r + - adi,ad5682r + - adi,ad5683 + - adi,ad5683r + then: + properties: + gain-gpios: false unevaluatedProperties: false examples: - | + #include spi { #address-cells = <1>; #size-cells = <0>; dac@0 { reg = <0>; compatible = "adi,ad5310r"; - vcc-supply = <&dac_vref0>; + vdd-supply = <&dac_vdd>; + vlogic-supply = <&dac_vlogic>; + vref-supply = <&dac_vref>; + reset-gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + ldac-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>; }; }; ... diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml index b5a88b03dc2f..e10f8596f9d3 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml @@ -33,18 +33,82 @@ properties: reg: maxItems: 1 + vdd-supply: + description: Input power supply. + + vlogic-supply: + description: + Digital power supply. On some tiny package variants for single-channel + devices, this supply is internally connected to vdd; in that case, specify + this property with the same regulator as vdd. + + vref-supply: + description: + Reference voltage supply. If not supplied the internal reference is used. + vcc-supply: - description: | - The regulator supply for DAC reference voltage. + deprecated: true + description: Use vref-supply instead. + + reset-gpios: + description: Active-low RESET pin to reset the device. + maxItems: 1 + + ldac-gpios: + description: + Active-low LDAC pin used to asynchronously update the DAC channels. + maxItems: 1 + + gain-gpios: + description: + GAIN pin that sets a multiplier for the DAC output voltage. When high, + the DAC output voltage is multiplied by 2, otherwise it is unchanged. + maxItems: 1 + + adi,range-double: + description: + Sets the initial voltage output range from 0 to 2xVREF. On devices that + have a GAIN pin and no gain-gpios property is set, this indicates the pin + is hardwired high. + type: boolean required: - compatible - reg + - vdd-supply + - vlogic-supply + +allOf: + - if: + properties: + compatible: + contains: + enum: + - adi,ad5693 + - adi,ad5694 + - adi,ad5696 + then: + required: + - vref-supply + - if: + properties: + compatible: + contains: + enum: + - adi,ad5311r + - adi,ad5691r + - adi,ad5692r + - adi,ad5693 + - adi,ad5693r + then: + properties: + gain-gpios: false additionalProperties: false examples: - | + #include i2c { #address-cells = <1>; #size-cells = <0>; @@ -52,7 +116,10 @@ examples: ad5696: dac@0 { compatible = "adi,ad5696"; reg = <0>; - vcc-supply = <&dac_vref>; + vdd-supply = <&dac_vdd>; + vlogic-supply = <&dac_vlogic>; + vref-supply = <&dac_vref>; + ldac-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>; }; }; ... diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml new file mode 100644 index 000000000000..19cc744a9f0f --- /dev/null +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/dac/adi,ad5706r.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices AD5706R 4-Channel Current Output DAC + +maintainers: + - Alexis Czezar Torreno + +description: | + The AD5706R is a 4-channel, 16-bit resolution, current output + digital-to-analog converter (DAC) with programmable output current + ranges (50mA, 150mA, 200mA, 300mA), an integrated 2.5V voltage + reference, and load DAC, A/B toggle, and dither functions. + + Datasheet: + https://www.analog.com/en/products/ad5706r.html + +properties: + compatible: + enum: + - adi,ad5706r + + reg: + maxItems: 1 + + avdd-supply: + description: Analog power supply (2.9V to 3.6V). + + iovdd-supply: + description: Logic power supply (1.14V to 1.89V). + + pvdd0-supply: + description: Power supply for IDAC0 channel (1.65V to AVDD). + + pvdd1-supply: + description: Power supply for IDAC1 channel (1.65V to AVDD). + + pvdd2-supply: + description: Power supply for IDAC2 channel (1.65V to AVDD). + + pvdd3-supply: + description: Power supply for IDAC3 channel (1.65V to AVDD). + + vref-supply: + description: + Optional external 2.5V voltage reference. If not provided, the + internal 2.5V reference is used. + + pwms: + maxItems: 1 + description: + Optional PWM connected to the LDAC/TGP/DCK pin for hardware + triggered DAC updates, toggle, or dither clock generation. + + reset-gpios: + maxItems: 1 + description: + GPIO connected to the active low RESET pin. If not provided, + software reset is used. + + enable-gpios: + maxItems: 1 + description: + GPIO connected to the active low OUT_EN pin. Controls whether + the current outputs are enabled or in high-Z/ground state. + +required: + - compatible + - reg + - avdd-supply + - iovdd-supply + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + + spi { + #address-cells = <1>; + #size-cells = <0>; + + dac@0 { + compatible = "adi,ad5706r"; + reg = <0>; + avdd-supply = <&avdd>; + iovdd-supply = <&iovdd>; + pvdd0-supply = <&pvdd>; + pvdd1-supply = <&pvdd>; + pvdd2-supply = <&pvdd>; + pvdd3-supply = <&pvdd>; + vref-supply = <&vref>; + spi-max-frequency = <50000000>; + pwms = <&pwm0 0 1000000 0>; + reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + enable-gpios = <&gpio0 12 GPIO_ACTIVE_LOW>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml index d2466aa6bda2..d131f136bd15 100644 --- a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml +++ b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml @@ -161,8 +161,8 @@ patternProperties: properties: reg: description: The channel number. - minItems: 1 - maxItems: 8 + minimum: 0 + maximum: 7 label: description: Unique name to identify which channel this is. @@ -280,23 +280,23 @@ examples: #address-cells = <1>; #size-cells = <0>; - dac@0 { - compatible = "microchip,mcp47feb02"; - reg = <0>; - vdd-supply = <&vdac_vdd>; - vref-supply = <&vref_reg>; + dac@60 { + compatible = "microchip,mcp47feb02"; + reg = <0x60>; + vdd-supply = <&vdac_vdd>; + vref-supply = <&vref_reg>; - #address-cells = <1>; - #size-cells = <0>; - channel@0 { - reg = <0>; - label = "Adjustable_voltage_ch0"; - }; + #address-cells = <1>; + #size-cells = <0>; + channel@0 { + reg = <0>; + label = "Adjustable_voltage_ch0"; + }; - channel@1 { - reg = <0x1>; - label = "Adjustable_voltage_ch1"; - }; - }; + channel@1 { + reg = <0x1>; + label = "Adjustable_voltage_ch1"; + }; + }; }; ... diff --git a/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml b/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml new file mode 100644 index 000000000000..f4fae9210382 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/frequency/adi,adf41513.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices ADF41513 PLL Frequency Synthesizer + +maintainers: + - Rodrigo Alencar + +description: + The ADF41513 is an ultralow noise frequency synthesizer that can be used to + implement local oscillators (LOs) as high as 26.5 GHz in the upconversion and + downconversion sections of wireless receivers and transmitters. The ADF41510 + supports frequencies up to 10 GHz. + + https://www.analog.com/en/products/adf41510.html + https://www.analog.com/en/products/adf41513.html + +properties: + compatible: + enum: + - adi,adf41510 + - adi,adf41513 + + reg: + maxItems: 1 + + spi-max-frequency: + maximum: 25000000 + + clocks: + maxItems: 1 + description: Clock that provides the reference input frequency. + + avdd1-supply: + description: PFD and Up and Down Digital Driver Power Supply (3.3 V) + + avdd2-supply: + description: RF Buffer and Prescaler Power Supply (3.3 V) + + avdd3-supply: + description: N Divider Power Supply (3.3 V) + + avdd4-supply: + description: R Divider and Lock Detector Power Supply (3.3 V) + + avdd5-supply: + description: Sigma-Delta Modulator and SPI Power Supply (3.3 V) + + vp-supply: + description: Charge Pump Power Supply (3.3 V) + + enable-gpios: + description: + GPIO that controls the chip enable pin. A logic low on this pin + powers down the device and puts the charge pump output into + three-state mode. + maxItems: 1 + + lock-detect-gpios: + description: + GPIO for lock detect functionality. When configured for digital lock + detect, this pin will output a logic high when the PLL is locked. + maxItems: 1 + + adi,power-up-frequency-mhz: + minimum: 1000 + maximum: 26500 + default: 10000 + description: + The PLL tunes to this frequency during the initialization sequence. + This property should be set to a frequency supported by the loop filter + and VCO used in the design. Range is 1 GHz to 26.5 GHz for ADF41513, + and 1 GHz to 10 GHz for ADF41510. + + adi,reference-div-factor: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 32 + default: 1 + description: + Value for the reference division factor (R Counter). The driver will + increment R Counter as needed to achieve a PFD frequency within the + allowed range. High R counter values will reduce the PFD frequency, which + lowers the frequency resolution, and affects phase noise performance. + As it affects the PFD frequency, this value depends on the loop filter + design. + + adi,reference-doubler-enable: + description: + Enables the reference doubler when deriving the PFD frequency. + The maximum reference frequency when the doubler is enabled is 225 MHz. + As it affects the PFD frequency, this value depends on the loop filter + design. + type: boolean + + adi,reference-div2-enable: + description: + Enables the reference divide-by-2 function when deriving the PFD + frequency. As it affects the PFD frequency, this value depends on the + loop filter design. + type: boolean + + adi,charge-pump-resistor-ohms: + minimum: 1800 + maximum: 10000 + default: 2700 + description: + External charge pump resistor (R_SET) value in ohms. This sets the maximum + charge pump current along with the charge pump current setting. + + adi,charge-pump-current-microamp: + minimum: 81 + maximum: 7200 + description: + Charge pump current (I_CP) in microamps. The value will be rounded to the + nearest supported value. Range of acceptable values depends on the + charge pump resistor value, such that 810 mV <= I_CP * R_SET <= 12960 mV. + This value depends on the loop filter and the VCO design. + + adi,logic-level-1v8-enable: + description: + Set MUXOUT and DLD logic levels to 1.8V. Default is 3.3V. + type: boolean + + adi,phase-detector-polarity-positive-enable: + description: + Set phase detector polarity to positive. Default is negative. + Use positive polarity with non-inverting loop filter and VCO with + positive tuning slope, or with inverting loop filter and VCO with + negative tuning slope. + type: boolean + + adi,lock-detector-count: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 64 + description: + Sets the value for Lock Detector count of the PLL, which determines the + number of consecutive phase detector cycles that must be within the lock + detector window before lock is declared. Lower values increase the lock + detection sensitivity, while higher values provides a more stable lock + detection. Applications that consume the lock detect signal may require + different settings based on system requirements. + enum: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + + adi,phase-resync-period-ns: + default: 0 + description: + When this value is non-zero, enable phase resync functionality, which + produces a consistent output phase offset with respect to the input + reference. The value specifies the resync period in nanoseconds, used + to configure clock dividers with respect to the PFD frequency. This value + should be set to a value that is at least as long as the worst case lock + time, i.e., it depends mostly on the loop filter design. + + adi,le-sync-enable: + description: + Synchronizes Load Enable (LE) transitions with the reference signal to + avoid asynchronous glitches in the output. This is recommended when using + the PLL as a frequency synthesizer, where the reference signal will always + be present while the device is being configured. When using the PLL as a + frequency tracker, where the reference signal may be absent, LE sync + should be left disabled. + type: boolean + +dependencies: + adi,charge-pump-resistor-ohms: [ 'adi,charge-pump-current-microamp' ] + +required: + - compatible + - reg + - clocks + - avdd1-supply + - avdd2-supply + - avdd3-supply + - avdd4-supply + - avdd5-supply + - vp-supply + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + - if: + properties: + compatible: + contains: + const: adi,adf41510 + then: + properties: + adi,power-up-frequency-mhz: + maximum: 10000 + +unevaluatedProperties: false + +examples: + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + + pll@0 { + compatible = "adi,adf41513"; + reg = <0>; + spi-max-frequency = <25000000>; + clocks = <&ref_clk>; + avdd1-supply = <&avdd1_3v3>; + avdd2-supply = <&avdd2_3v3>; + avdd3-supply = <&avdd3_3v3>; + avdd4-supply = <&avdd4_3v3>; + avdd5-supply = <&avdd5_3v3>; + vp-supply = <&vp_3v3>; + enable-gpios = <&gpio0 10 GPIO_ACTIVE_HIGH>; + lock-detect-gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>; + + adi,power-up-frequency-mhz = <15500>; + adi,charge-pump-current-microamp = <3600>; + adi,charge-pump-resistor-ohms = <2700>; + adi,reference-doubler-enable; + adi,lock-detector-count = <64>; + adi,phase-resync-period-ns = <0>; + adi,phase-detector-polarity-positive-enable; + adi,le-sync-enable; + }; + }; +... diff --git a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml index fcbd4b430e48..8e094651381c 100644 --- a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml +++ b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml @@ -26,6 +26,9 @@ properties: vdd-supply: true vddio-supply: true + mount-matrix: + description: an optional 3x3 mounting rotation matrix. + spi-max-frequency: maximum: 10000000 @@ -56,6 +59,9 @@ examples: reg = <0x69>; interrupt-parent = <&gpio6>; interrupts = <18 IRQ_TYPE_EDGE_RISING>; + mount-matrix = "0", "1", "0", + "1", "0", "0", + "0", "0", "1"; }; }; ... diff --git a/Documentation/devicetree/bindings/iio/light/brcm,apds9999.yaml b/Documentation/devicetree/bindings/iio/light/brcm,apds9999.yaml new file mode 100644 index 000000000000..9f5b3b294c2c --- /dev/null +++ b/Documentation/devicetree/bindings/iio/light/brcm,apds9999.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/light/brcm,apds9999.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# +title: Broadcom APDS-9999 Digital Proximity and RGB Sensor + +maintainers: + - Jose A. Perez de Azpillaga + +description: | + Broadcom APDS-9999 is a digital proximity and RGB sensor with + ambient light sensing (ALS) capability. The device uses individual + R, G, B, and IR channels plus a Vertical Cavity Surface Emitting + Laser (VCSEL) for proximity detection. + + Datasheet: https://docs.broadcom.com/docs/APDS-9999-DS + +properties: + compatible: + enum: + - brcm,apds9999 + + reg: + maxItems: 1 + + vdd-supply: true + + vcsel-supply: + description: VCSEL power supply (VVCSEL pin) + + interrupts: + maxItems: 1 + +additionalProperties: false + +required: + - compatible + - reg + - vdd-supply + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + light-sensor@52 { + compatible = "brcm,apds9999"; + reg = <0x52>; + vdd-supply = <&vdd_reg>; + vcsel-supply = <&vcsel_reg>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml index d81229857944..9921ccaa64a0 100644 --- a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml +++ b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml @@ -26,6 +26,8 @@ properties: - amstaos,tmd2672 - amstaos,tsl2772 - amstaos,tmd2772 + - avago,apds9900 + - avago,apds9901 - avago,apds9930 reg: diff --git a/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml b/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml index 4ea69f1fdd63..0041e1db6838 100644 --- a/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml +++ b/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/iio/light/vishay,veml6030.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: VEML3235, VEML6030, VEML6035 and VEML7700 Ambient Light Sensors (ALS) +title: VEML3235, VEML3328, VEML6030, VEML6035 and VEML7700 Ambient Light Sensors (ALS) maintainers: - Rishi Gupta @@ -21,6 +21,7 @@ description: | Specifications about the sensors can be found at: https://www.vishay.com/docs/80131/veml3235.pdf + https://www.vishay.com/docs/84968/veml3328.pdf https://www.vishay.com/docs/84366/veml6030.pdf https://www.vishay.com/docs/84889/veml6035.pdf https://www.vishay.com/docs/84286/veml7700.pdf @@ -29,6 +30,7 @@ properties: compatible: enum: - vishay,veml3235 + - vishay,veml3328 - vishay,veml6030 - vishay,veml6035 - vishay,veml7700 @@ -79,6 +81,7 @@ allOf: compatible: enum: - vishay,veml3235 + - vishay,veml3328 - vishay,veml7700 then: properties: diff --git a/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml b/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml new file mode 100644 index 000000000000..4a4df6bb70fe --- /dev/null +++ b/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/magnetometer/memsic,mmc5983.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MEMSIC MMC5983MA 3-axis magnetic sensor + +maintainers: + - Vladislav Kulikov + +properties: + compatible: + const: memsic,mmc5983 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: + description: Regulator that provides power to the sensor + + vddio-supply: + description: Regulator that provides power to the digital interface and INT pin + +required: + - compatible + - reg + - vdd-supply + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + magnetometer@30 { + compatible = "memsic,mmc5983"; + reg = <0x30>; + vdd-supply = <&vdd_3v3_reg>; + vddio-supply = <&vdd_3v3_reg>; + }; + }; + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + + magnetometer@0 { + compatible = "memsic,mmc5983"; + reg = <0>; + spi-max-frequency = <10000000>; + vdd-supply = <&vdd_3v3_reg>; + vddio-supply = <&vdd_3v3_reg>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml b/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml new file mode 100644 index 000000000000..38abd083a4fb --- /dev/null +++ b/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/magnetometer/qstcorp,qmc5883l.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: QST QMC5883L 3-Axis Magnetic Sensor + +maintainers: + - Siratul Islam + +description: | + QST QMC5883L 3-Axis Magnetic Sensor on I2C bus. + https://www.qstcorp.com/upload/pdf/202512/13-52-04%20QMC5883L%20Datasheet%20Rev.%20B.pdf + +properties: + compatible: + enum: + - qstcorp,qmc5883l + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: true + + vddio-supply: true + +additionalProperties: false + +required: + - compatible + - reg + - vdd-supply + - vddio-supply + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + magnetometer@d { + compatible = "qstcorp,qmc5883l"; + reg = <0x0d>; + vdd-supply = <&vdd_3v3_reg>; + vddio-supply = <&vdd_3v3_reg>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml b/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml index a22725f7619b..13e5f29f0588 100644 --- a/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml @@ -4,14 +4,18 @@ $id: http://devicetree.org/schemas/iio/temperature/adi,ltc2983.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices LTC2983, LTC2986, LTM2985 Multi-sensor Temperature system +title: Analog Devices LTC2983 and similar Multi-sensor Temperature systems maintainers: - Nuno Sá description: | - Analog Devices LTC2983, LTC2984, LTC2986, LTM2985 Multi-Sensor Digital - Temperature Measurement Systems + Analog Devices Multi-Sensor Digital Temperature Measurement Systems: + - ADT7604 + - LTC2983 + - LTC2984 + - LTC2986 + - LTM2985 https://www.analog.com/media/en/technical-documentation/data-sheets/2983fc.pdf https://www.analog.com/media/en/technical-documentation/data-sheets/2984fb.pdf @@ -43,6 +47,7 @@ properties: compatible: oneOf: - enum: + - adi,adt7604 - adi,ltc2983 - adi,ltc2986 - adi,ltm2985 @@ -436,6 +441,121 @@ patternProperties: required: - adi,custom-temp + '^copper-trace@': + $ref: '#/$defs/sensor-node' + unevaluatedProperties: false + description: | + Copper trace resistance sensor (some parts only). Two variants exist: + sub-ohm (< 1 ohm, no custom table allowed) and standard (> 1 ohm, + required custom table). + + properties: + reg: + minimum: 2 + maximum: 20 + + adi,sensor-type: + description: Sensor type for copper trace sensors. + $ref: /schemas/types.yaml#/definitions/uint32 + const: 32 + + adi,rsense-handle: + description: Associated sense resistor sensor. + $ref: /schemas/types.yaml#/definitions/phandle + + adi,copper-trace-sub-ohm: + description: + Select the sub-ohm (< 1 ohm) copper trace variant. Custom table + and excitation current are not allowed in this mode. + type: boolean + + adi,excitation-current-microamp: + description: + Excitation current applied to the copper trace. Not used in + sub-ohm mode. The datasheet recommends 1mA for copper trace + sensors due to their typically small resistance. + enum: [5, 10, 25, 50, 100, 250, 500, 1000] + default: 1000 + + adi,custom-copper-trace: + description: + Resistance-to-temperature table for copper trace sensors with + resistance > 1 ohm. Required when adi,copper-trace-sub-ohm is not + set. See Page 36 of the datasheet. + $ref: /schemas/types.yaml#/definitions/uint64-matrix + minItems: 3 + maxItems: 64 + items: + items: + - description: Resistance point in uOhms. + - description: Temperature point in uK. + + required: + - adi,rsense-handle + + allOf: + - if: + required: + - adi,copper-trace-sub-ohm + then: + properties: + adi,custom-copper-trace: false + adi,excitation-current-microamp: false + - if: + not: + required: + - adi,copper-trace-sub-ohm + then: + required: + - adi,custom-copper-trace + + '^leak-detector@': + $ref: '#/$defs/sensor-node' + unevaluatedProperties: false + description: | + Leak detector sensor (some parts only). Outputs resistance in ohms and + a coverage percentage via IIO_COVERAGE (raw/1024 = coverage %). + + properties: + reg: + minimum: 2 + maximum: 20 + + adi,sensor-type: + description: Sensor type for leak detector sensors. + $ref: /schemas/types.yaml#/definitions/uint32 + const: 33 + + adi,rsense-handle: + description: Associated sense resistor sensor. + $ref: /schemas/types.yaml#/definitions/phandle + + adi,excitation-current-nanoamp: + description: + Excitation current applied to the leak detector. The correct value + depends on the electrical characteristics of the liquid being sensed. + For example, 10000 (10µA) is recommended for PG25 (see datasheet + Table 39). + enum: [250, 500, 1000, 5000, 10000, 25000, 50000, 100000, 250000, + 500000, 1000000] + + adi,custom-leak-detector: + description: | + Lookup table mapping resistance to coverage percentage. Entries must + be in ascending resistance order. + $ref: /schemas/types.yaml#/definitions/uint64-matrix + minItems: 3 + maxItems: 64 + items: + items: + - description: Resistance point in uOhms. + - description: Coverage data percentage (0 to 100). + + required: + - adi,rsense-handle + - adi,excitation-current-nanoamp + - adi,custom-leak-detector + '^rsense@': $ref: '#/$defs/sensor-node' unevaluatedProperties: false @@ -477,6 +597,32 @@ allOf: patternProperties: '^temp@': false + - if: + properties: + compatible: + contains: + const: adi,adt7604 + then: + patternProperties: + '^thermocouple@': false + '^diode@': false + '^adc@': false + '^temp@': false + '^rtd@': + properties: + adi,sensor-type: + not: + const: 18 + '^thermistor@': + properties: + adi,sensor-type: + not: + const: 27 + else: + patternProperties: + '^copper-trace@': false + '^leak-detector@': false + examples: - | #include @@ -556,4 +702,69 @@ examples: }; }; }; + + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + + temperature-sensor@0 { + compatible = "adi,adt7604"; + reg = <0>; + interrupt-parent = <&gpio>; + interrupts = <25 IRQ_TYPE_EDGE_RISING>; + + #address-cells = <1>; + #size-cells = <0>; + vdd-supply = <&supply>; + + trace_rsense: rsense@2 { + reg = <2>; + adi,sensor-type = <29>; + adi,rsense-val-milli-ohms = <100000>; // 100 ohm + }; + + copper-trace@4 { + reg = <4>; + adi,sensor-type = <32>; + adi,rsense-handle = <&trace_rsense>; + adi,copper-trace-sub-ohm; + }; + + r_sense: rsense@12 { + reg = <12>; + adi,sensor-type = <29>; + adi,rsense-val-milli-ohms = <1000000>; // 1 kohm + }; + + leak-detector@14 { + reg = <14>; + adi,sensor-type = <33>; + adi,rsense-handle = <&r_sense>; + adi,excitation-current-nanoamp = <10000>; + adi,custom-leak-detector = + /bits/ 64 < 0 100>, + /bits/ 64 < 202020000 99>, + /bits/ 64 < 285710000 70>, + /bits/ 64 < 333330000 60>, + /bits/ 64 < 400000000 50>, + /bits/ 64 < 500000000 40>, + /bits/ 64 < 666670000 30>, + /bits/ 64 < 1000000000 20>, + /bits/ 64 < 2000000000 10>, + /bits/ 64 <1000000000000 0>; + }; + + rtd@18 { + reg = <18>; + adi,sensor-type = <12>; // PT100 + adi,rsense-handle = <&r_sense>; + adi,number-of-wires = <2>; + adi,rsense-share; + adi,excitation-current-microamp = <500>; + adi,rtd-curve = <0>; + }; + }; + }; ... diff --git a/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml b/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml index fbba5e934861..7fb3e6820791 100644 --- a/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml @@ -7,19 +7,25 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: TI TMP117 - Digital temperature sensor with integrated NV memory description: | - TI TMP116/117 - Digital temperature sensor with integrated NV memory that - supports I2C interface. + TI TMP116/117/119 - Digital temperature sensor with integrated NV memory + that supports I2C interface. https://www.ti.com/lit/gpn/tmp116 https://www.ti.com/lit/gpn/tmp117 + https://www.ti.com/lit/gpn/tmp119 maintainers: - Puranjay Mohan properties: compatible: - enum: - - ti,tmp116 - - ti,tmp117 + oneOf: + - enum: + - ti,tmp116 + - ti,tmp117 + - items: + - enum: + - ti,tmp119 + - const: ti,tmp117 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml index 6bdb8040be65..524c8b51f53f 100644 --- a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml +++ b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml @@ -23,6 +23,7 @@ properties: - const: allwinner,sun50i-r329-lradc - items: - enum: + - allwinner,sun50i-a100-lradc - allwinner,sun50i-h616-lradc - allwinner,sun20i-d1-lradc - const: allwinner,sun50i-r329-lradc diff --git a/Documentation/devicetree/bindings/input/imagis,isa1200.yaml b/Documentation/devicetree/bindings/input/imagis,isa1200.yaml new file mode 100644 index 000000000000..4bc8630edcdd --- /dev/null +++ b/Documentation/devicetree/bindings/input/imagis,isa1200.yaml @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/imagis,isa1200.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Imagis ISA1200 haptic motor driver + +maintainers: + - Svyatoslav Ryhel + - Linus Walleij + +description: + The ISA1200 is a high-performance enhanced haptic motor driver designed + for mobile hand-held devices. It supports various voltages for both ERM + (Eccentric Rotating Mass) and LRA (Linear Resonant Actuator) type + actuators. Thanks to an embedded LDO, battery power can be used directly + in handheld applications. + +properties: + compatible: + const: imagis,isa1200 + + reg: + maxItems: 1 + + control-gpios: + description: + One or two GPIOs flagged as active high linked to HEN and LEN pins + minItems: 1 + maxItems: 2 + + clocks: + maxItems: 1 + + pwms: + maxItems: 1 + + vdd-supply: + description: + Regulator for 2.4V - 5.5V power supply + + vddp-supply: + description: + Regulator for 2.4V - 3.6V IO power supply + + imagis,clk-div: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Divider for the external input clock/PWM + enum: [128, 256, 512, 1024] + default: 128 + + imagis,pll-div: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Divider for the internal PLL clock + minimum: 1 + maximum: 15 + default: 1 + + imagis,mode: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + Defines the motor type isa1200 drives + 0 - LRA (Linear Resonant Actuator) + 1 - ERM (Eccentric Rotating Mass) + enum: [0, 1] + default: 0 + + imagis,period-ns: + description: + Period of the internal PWM channel in nanoseconds. + minimum: 10000 + maximum: 30000 + + imagis,duty-cycle-ns: + description: + Duty cycle of the external/internal PWM channel in nanoseconds, + defaults to 50% of the channel's period + + ldo: + $ref: /schemas/regulator/regulator.yaml# + type: object + description: + Embedded LDO regulator with voltage range 2.3V - 3.8V + unevaluatedProperties: false + + required: + - regulator-min-microvolt + - regulator-max-microvolt + +required: + - compatible + - reg + - ldo + +oneOf: + - required: + - clocks + - imagis,period-ns + - required: + - pwms + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + haptic-engine@49 { + compatible = "imagis,isa1200"; + reg = <0x49>; + + clocks = <&isa1200_refclk>; + + control-gpios = <&gpio 22 GPIO_ACTIVE_HIGH>, + <&gpio 23 GPIO_ACTIVE_HIGH>; + + vdd-supply = <&vdd_3v3_vbat>; + vddp-supply = <&vdd_2v8_vvib>; + + imagis,clk-div = <256>; + imagis,pll-div = <2>; + + imagis,mode = <0>; /* LRA_MODE */ + + imagis,period-ns = <13400>; + imagis,duty-cycle-ns = <100>; + + ldo { + regulator-name = "vdd_vib"; + regulator-min-microvolt = <2300000>; + regulator-max-microvolt = <2300000>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml index 2d762193f1c0..983f4701b51e 100644 --- a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml +++ b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml @@ -10,6 +10,16 @@ description: | The Microchip CAP1xxx Family of RightTouchTM multiple-channel capacitive touch controllers and LED drivers. The device communication via I2C only. + For more product information please see the links below: + CAP1106: https://ww1.microchip.com/downloads/en/DeviceDoc/00001624B.pdf + CAP1114: https://ww1.microchip.com/downloads/en/DeviceDoc/00002444A.pdf + CAP1126: https://ww1.microchip.com/downloads/en/DeviceDoc/00001623B.pdf + CAP1188: https://ww1.microchip.com/downloads/en/DeviceDoc/00001620C.pdf + CAP1203: https://ww1.microchip.com/downloads/en/DeviceDoc/00001572B.pdf + CAP1206: https://ww1.microchip.com/downloads/en/DeviceDoc/00001567B.pdf + CAP1293: https://ww1.microchip.com/downloads/en/DeviceDoc/00001566B.pdf + CAP1298: https://ww1.microchip.com/downloads/en/DeviceDoc/00001571B.pdf + maintainers: - Rob Herring @@ -17,6 +27,7 @@ properties: compatible: enum: - microchip,cap1106 + - microchip,cap1114 - microchip,cap1126 - microchip,cap1188 - microchip,cap1203 @@ -40,13 +51,20 @@ properties: device's ALERT#/CM_IRQ# pin is connected to. The device only has one interrupt source. + reset-gpios: + description: | + GPIO connected to the active-high RESET pin of the chip; + driving it high asserts reset and deep sleep, while driving + it low releases reset for normal operation. + maxItems: 1 + autorepeat: description: | Enables the Linux input system's autorepeat feature on the input device. linux,keycodes: minItems: 3 - maxItems: 8 + maxItems: 14 description: | Specifies an array of numeric keycode values to be used for the channels. If this property is @@ -106,6 +124,8 @@ properties: is required for a touch to be registered, making the touch sensor less sensitive. The number of entries must correspond to the number of channels. + CAP1114 is an exception where channels 8~14 reuse the eighth entry's + threshold, so counts differ. microchip,calib-sensitivity: $ref: /schemas/types.yaml#/definitions/uint32-array @@ -124,14 +144,16 @@ properties: The number of entries must correspond to the number of channels. patternProperties: - "^led@[0-7]$": + "^led@[0-9a]$": type: object description: CAP11xx LEDs $ref: /schemas/leds/common.yaml# properties: reg: - enum: [0, 1, 2, 3, 4, 5, 6, 7] + description: LED channel number + minimum: 0 + maximum: 10 label: true @@ -156,9 +178,64 @@ allOf: - microchip,cap1206 - microchip,cap1293 - microchip,cap1298 + then: + properties: + reset-gpios: false + + - if: + properties: + compatible: + contains: + enum: + - microchip,cap1114 + then: + properties: + linux,keycodes: + minItems: 14 + else: + properties: + linux,keycodes: + maxItems: 8 + + - if: + properties: + compatible: + contains: + enum: + - microchip,cap1106 + - microchip,cap1203 + - microchip,cap1206 + - microchip,cap1293 + - microchip,cap1298 + then: + patternProperties: + "^led@": false + + - if: + properties: + compatible: + contains: + enum: + - microchip,cap1126 then: patternProperties: - "^led@[0-7]$": false + "^led@": + properties: + reg: + maximum: 1 + + - if: + properties: + compatible: + contains: + enum: + - microchip,cap1188 + then: + patternProperties: + "^led@": + properties: + reg: + maximum: 7 - if: properties: @@ -166,6 +243,7 @@ allOf: contains: enum: - microchip,cap1106 + - microchip,cap1114 - microchip,cap1126 - microchip,cap1188 - microchip,cap1203 @@ -183,6 +261,8 @@ additionalProperties: false examples: - | + #include + i2c { #address-cells = <1>; #size-cells = <0>; @@ -204,6 +284,8 @@ examples: <109>, /* KEY_PAGEDOWN */ <104>; /* KEY_PAGEUP */ + reset-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>; + #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/input/syna,rmi4.yaml b/Documentation/devicetree/bindings/input/syna,rmi4.yaml index 8685ef4481f4..fb4804ac3544 100644 --- a/Documentation/devicetree/bindings/input/syna,rmi4.yaml +++ b/Documentation/devicetree/bindings/input/syna,rmi4.yaml @@ -18,9 +18,14 @@ description: | properties: compatible: - enum: - - syna,rmi4-i2c - - syna,rmi4-spi + oneOf: + - enum: + - syna,rmi4-i2c + - syna,rmi4-spi + - items: + - enum: + - syna,rmi4-s3706b # OnePlus 6/6T + - const: syna,rmi4-i2c reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml b/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml index 12256ae7df90..64c4f24ea3dd 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml @@ -40,6 +40,10 @@ properties: vdd-supply: description: Power supply + reset-gpios: + description: Reset GPIO (active-low) + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/input/touchscreen/wacom,w9007a-lt03.yaml b/Documentation/devicetree/bindings/input/touchscreen/wacom,w9007a-lt03.yaml new file mode 100644 index 000000000000..6d1da6a435d3 --- /dev/null +++ b/Documentation/devicetree/bindings/input/touchscreen/wacom,w9007a-lt03.yaml @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/touchscreen/wacom,w9007a-lt03.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Wacom W9000-series penabled I2C touchscreen + +maintainers: + - Hendrik Noack + +description: | + The W9000-series are penabled touchscreen controllers by Wacom. + + The firmware of controllers in different devices may differ. This can also + affect the controller's behavior. + +allOf: + - $ref: touchscreen.yaml# + +properties: + compatible: + enum: + - wacom,w9002 + - wacom,w9007a-lt03 + - wacom,w9007a-v1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: true + + flash-mode-gpios: + maxItems: 1 + + reset-gpios: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + digitizer@56 { + compatible = "wacom,w9007a-lt03"; + reg = <0x56>; + interrupt-parent = <&gpd1>; + interrupts = <1 IRQ_TYPE_EDGE_RISING>; + + vdd-supply = <&stylus_reg>; + + flash-mode-gpios = <&gpd1 3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpx0 1 GPIO_ACTIVE_LOW>; + + touchscreen-x-mm = <216>; + touchscreen-y-mm = <135>; + touchscreen-inverted-x; + }; + }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml new file mode 100644 index 000000000000..49a2dca5db62 --- /dev/null +++ b/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interconnect/qcom,hawi-rpmh.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm RPMh Network-On-Chip Interconnect on Hawi + +maintainers: + - Vivek Aknurwar + +description: | + RPMh interconnect providers support system bandwidth requirements through + RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is + able to communicate with the BCM through the Resource State Coordinator (RSC) + associated with each execution environment. Provider nodes must point to at + least one RPMh device child node pertaining to their RSC and each provider + can map to multiple RPMh resources. + + See also: include/dt-bindings/interconnect/qcom,hawi-rpmh.h + +properties: + compatible: + enum: + - qcom,hawi-aggre1-noc + - qcom,hawi-clk-virt + - qcom,hawi-cnoc-main + - qcom,hawi-gem-noc + - qcom,hawi-llclpi-noc + - qcom,hawi-lpass-ag-noc + - qcom,hawi-lpass-lpiaon-noc + - qcom,hawi-lpass-lpicx-noc + - qcom,hawi-mc-virt + - qcom,hawi-mmss-noc + - qcom,hawi-nsp-noc + - qcom,hawi-pcie-anoc + - qcom,hawi-stdst-cfg + - qcom,hawi-stdst-main + - qcom,hawi-system-noc + + reg: + maxItems: 1 + + clocks: + minItems: 2 + maxItems: 3 + +required: + - compatible + +allOf: + - $ref: qcom,rpmh-common.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,hawi-clk-virt + - qcom,hawi-mc-virt + then: + properties: + reg: false + else: + required: + - reg + + - if: + properties: + compatible: + contains: + enum: + - qcom,hawi-pcie-anoc + then: + properties: + clocks: + items: + - description: aggre-NOC PCIe AXI clock + - description: cfg-NOC PCIe a-NOC AHB clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,hawi-aggre1-noc + then: + properties: + clocks: + items: + - description: aggre UFS PHY AXI clock + - description: aggre USB3 PRIM AXI clock + - description: RPMH CC IPA clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,hawi-aggre1-noc + - qcom,hawi-pcie-anoc + then: + required: + - clocks + else: + properties: + clocks: false + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + clk_virt: interconnect-0 { + compatible = "qcom,hawi-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + aggre_noc: interconnect@f00000 { + compatible = "qcom,hawi-aggre1-noc"; + reg = <0x0 0xf00000 0x0 0x54400>; + #interconnect-cells = <2>; + clocks = <&gcc_aggre_ufs_phy_axi_clk>, + <&gcc_aggre_usb3_prim_axi_clk>, + <&rpmhcc_ipa_clk>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml index ce79521bb1ef..e002e70580f9 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml @@ -13,7 +13,7 @@ description: | Bandwidth Monitor measures current throughput on buses between various NoC fabrics and provides information when it crosses configured thresholds. - Certain SoCs might have more than one Bandwidth Monitors, for example on SDM845:: + Certain SoCs might have more than one Bandwidth Monitors, for example on SDM845: - Measuring the bandwidth between CPUs and Last Level Cache Controller - called just BWMON, - Measuring the bandwidth between Last Level Cache Controller and memory @@ -26,6 +26,7 @@ properties: - items: - enum: - qcom,glymur-cpu-bwmon + - qcom,hawi-cpu-bwmon - qcom,kaanapali-cpu-bwmon - qcom,qcm2290-cpu-bwmon - qcom,qcs615-cpu-bwmon @@ -45,6 +46,7 @@ properties: - const: qcom,sdm845-bwmon # BWMON v4, unified register space - items: - enum: + - qcom,hawi-llcc-bwmon - qcom,qcs615-llcc-bwmon - qcom,qcs8300-llcc-bwmon - qcom,sa8775p-llcc-bwmon diff --git a/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml new file mode 100644 index 000000000000..3650d3d5b918 --- /dev/null +++ b/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interconnect/qcom,nord-rpmh.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm RPMh Network-On-Chip Interconnect on Nord + +maintainers: + - Odelu Kukatla + +description: | + RPMh interconnect providers support system bandwidth requirements through + RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is + able to communicate with the BCM through the Resource State Coordinator (RSC) + associated with each execution environment. Provider nodes must point to at + least one RPMh device child node pertaining to their RSC and each provider + can map to multiple RPMh resources. + + See also: include/dt-bindings/interconnect/qcom,nord-rpmh.h + +properties: + compatible: + enum: + - qcom,nord-aggre1-noc + - qcom,nord-aggre1-noc-tile + - qcom,nord-aggre2-noc + - qcom,nord-aggre2-noc-tile + - qcom,nord-clk-virt + - qcom,nord-cnoc-cfg + - qcom,nord-cnoc-main + - qcom,nord-hpass-ag-noc + - qcom,nord-hscnoc + - qcom,nord-mc-virt + - qcom,nord-mmss-noc + - qcom,nord-nsp-data-noc-0 + - qcom,nord-nsp-data-noc-1 + - qcom,nord-nsp-data-noc-2 + - qcom,nord-nsp-data-noc-3 + - qcom,nord-pcie-cfg + - qcom,nord-pcie-data-inbound + - qcom,nord-pcie-data-outbound + - qcom,nord-system-noc + + reg: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 4 + +required: + - compatible + +allOf: + - $ref: qcom,rpmh-common.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,nord-clk-virt + - qcom,nord-mc-virt + then: + properties: + reg: false + else: + required: + - reg + + - if: + properties: + compatible: + contains: + enum: + - qcom,nord-aggre1-noc-tile + then: + properties: + clocks: + items: + - description: aggre UFS PHY AXI clock + - description: aggre USB2 AXI clock + - description: aggre USB3 PRIM AXI clock + - description: aggre USB3 SEC AXI clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,nord-aggre2-noc + then: + properties: + clocks: + items: + - description: RPMH CC IPA clock + + - if: + properties: + compatible: + contains: + enum: + - qcom,nord-aggre1-noc-tile + - qcom,nord-aggre2-noc + then: + required: + - clocks + else: + properties: + clocks: false + +unevaluatedProperties: false + +examples: + - | + clk_virt: interconnect-clk-virt { + compatible = "qcom,nord-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + aggre1_noc_tile: interconnect@1720000 { + compatible = "qcom,nord-aggre1-noc-tile"; + reg = <0x01720000 0x23400>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + clocks = <&ne_gcc_aggre_noc_ufs_phy_axi_clk>, + <&ne_gcc_aggre_noc_usb2_axi_clk>, + <&ne_gcc_aggre_noc_usb3_prim_axi_clk>, + <&ne_gcc_aggre_noc_usb3_sec_axi_clk>; + }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml b/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml index 8f6bc6399626..51428a2b0ce0 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml @@ -79,6 +79,19 @@ allOf: - const: aggre2_usb3_axi - const: cfg_noc_usb2_axi + - if: + properties: + compatible: + enum: + - qcom,sdm660-bimc + - qcom,sdm660-cnoc + - qcom,sdm660-gnoc + - qcom,sdm660-snoc + then: + properties: + clocks: false + clock-names: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml b/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml new file mode 100644 index 000000000000..a0c26de94ccf --- /dev/null +++ b/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interconnect/qcom,shikra.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Shikra Network-On-Chip interconnect + +maintainers: + - Raviteja Laggyshetty + +description: + The Qualcomm Shikra interconnect providers support adjusting the + bandwidth requirements between the various NoC fabrics. + +properties: + compatible: + enum: + - qcom,shikra-config-noc + - qcom,shikra-mem-noc-core + - qcom,shikra-sys-noc + + reg: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 4 + + clock-names: + minItems: 1 + maxItems: 4 + +# Child node's properties +patternProperties: + '^interconnect-[a-z0-9]+$': + type: object + description: + The interconnect providers do not have a separate QoS register space, + but share parent's space. + + $ref: qcom,rpm-common.yaml# + + properties: + compatible: + enum: + - qcom,shikra-clk-virt + - qcom,shikra-mc-virt + - qcom,shikra-mmrt-virt + - qcom,shikra-mmnrt-virt + + required: + - compatible + + unevaluatedProperties: false + +required: + - compatible + - reg + +allOf: + - $ref: qcom,rpm-common.yaml# + - if: + properties: + compatible: + const: qcom,shikra-mem-noc-core + + then: + properties: + clocks: + items: + - description: GPU-NoC AXI clock + + clock-names: + items: + - const: gpu_axi + patternProperties: + '^interconnect-[a-z0-9]+$': false + + - if: + properties: + compatible: + const: qcom,shikra-sys-noc + + then: + properties: + clocks: + items: + - description: EMAC0-NoC AXI clock. + - description: EMAC1-NoC AXI clock. + - description: USB2-NoC AXI clock. + - description: USB3-NoC AXI clock. + + clock-names: + items: + - const: emac0_axi + - const: emac1_axi + - const: usb2_axi + - const: usb3_axi + + - if: + properties: + compatible: + const: qcom,shikra-config-noc + + then: + properties: + clocks: false + clock-names: false + patternProperties: + '^interconnect-[a-z0-9]+$': false + +unevaluatedProperties: false + +examples: + - | + interconnect@1880000 { + compatible = "qcom,shikra-sys-noc"; + reg = <0x01880000 0x6a080>; + #interconnect-cells = <2>; + clocks = <&gcc_emac0_axi_sys_noc_clk>, + <&gcc_emac1_axi_sys_noc_clk>, + <&gcc_sys_noc_usb2_prim_axi_clk>, + <&gcc_sys_noc_usb3_prim_axi_clk>; + clock-names = "emac0_axi", + "emac1_axi", + "usb2_axi", + "usb3_axi"; + + interconnect-clk { + compatible = "qcom,shikra-clk-virt"; + #interconnect-cells = <2>; + }; + }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml b/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml index 14b1a0b08e73..cdae0acf3a1d 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml @@ -62,23 +62,33 @@ allOf: - if: properties: compatible: - const: qcom,sm6115-cnoc + const: qcom,sm6115-bimc + then: + properties: + clocks: false + clock-names: false + patternProperties: + '^interconnect-[a-z0-9]+$': false + - if: + properties: + compatible: + const: qcom,sm6115-cnoc then: properties: clocks: items: - description: USB-NoC AXI clock - clock-names: items: - const: usb_axi + patternProperties: + '^interconnect-[a-z0-9]+$': false - if: properties: compatible: const: qcom,sm6115-snoc - then: properties: clocks: @@ -87,7 +97,6 @@ allOf: - description: UFS-NoC AXI clock. - description: USB-NoC AXI clock. - description: IPA clock. - clock-names: items: - const: cpu_axi @@ -95,20 +104,6 @@ allOf: - const: usb_axi - const: ipa - - if: - properties: - compatible: - enum: - - qcom,sm6115-bimc - - qcom,sm6115-clk-virt - - qcom,sm6115-mmrt-virt - - qcom,sm6115-mmnrt-virt - - then: - properties: - clocks: false - clock-names: false - unevaluatedProperties: false examples: @@ -149,4 +144,6 @@ examples: compatible = "qcom,sm6115-cnoc"; reg = <0x01900000 0x8200>; #interconnect-cells = <1>; + clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>; + clock-names = "usb_axi"; }; diff --git a/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml b/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml index 5e26e48c7217..0203959c8995 100644 --- a/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml +++ b/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml @@ -23,7 +23,7 @@ description: | The each AXI bus has the owned source clock but, has not the only owned power line. The power line might be shared among one more sub-blocks. So, we can divide into two type of device as the role of each sub-block. There are two - type of bus devices as following:: + type of bus devices as following: - parent bus device - passive bus device @@ -44,8 +44,8 @@ description: | able to support the bus frequency for all Exynos SoCs. Detailed correlation between sub-blocks and power line according - to Exynos SoC:: - - In case of Exynos3250, there are two power line as following:: + to Exynos SoC: + - In case of Exynos3250, there are two power line as following: VDD_MIF |--- DMC (Dynamic Memory Controller) VDD_INT |--- LEFTBUS (parent device) @@ -89,7 +89,7 @@ description: | |L5 |200000 |200000 |400000 |300000 | ||1000000 | ---------------------------------------------------------- - - In case of Exynos4210, there is one power line as following:: + - In case of Exynos4210, there is one power line as following: VDD_INT |--- DMC (parent device, Dynamic Memory Controller) |--- LEFTBUS |--- PERIL @@ -106,7 +106,7 @@ description: | |--- LCD0 |--- LCD1 - - In case of Exynos4x12, there are two power line as following:: + - In case of Exynos4x12, there are two power line as following: VDD_MIF |--- DMC (Dynamic Memory Controller) VDD_INT |--- LEFTBUS (parent device) @@ -124,7 +124,7 @@ description: | |--- LCD0 |--- ISP - - In case of Exynos5422, there are two power line as following:: + - In case of Exynos5422, there are two power line as following: VDD_MIF |--- DREX 0 (parent device, DRAM EXpress controller) |--- DREX 1 @@ -143,7 +143,7 @@ description: | |--- FSYS |--- FSYS2 - - In case of Exynos5433, there is VDD_INT power line as following:: + - In case of Exynos5433, there is VDD_INT power line as following: VDD_INT |--- G2D (parent device) |--- MSCL |--- GSCL diff --git a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml index 833a01cdd1b1..5c52e57c6647 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml @@ -32,8 +32,9 @@ properties: "#interrupt-cells": description: - SoC interrupt line index. - const: 1 + First cell is the SoC interrupt line index. Optional second cell + specifies the parent interrupt index to route to. + enum: [1, 2] reg: minItems: 1 diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml index decc43df3c83..78d8cd77b4ec 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml @@ -71,6 +71,7 @@ properties: - items: - enum: - allwinner,sun20i-d1-plic + - canaan,k230-plic - sophgo,cv1800b-plic - sophgo,cv1812h-plic - sophgo,sg2000-plic diff --git a/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml b/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml index b409b2a8b5c5..5165bfddcd54 100644 --- a/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml +++ b/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml @@ -10,10 +10,10 @@ maintainers: - Krzysztof Kozlowski description: | - The Qualcomm PM8058 contains an LED block for up to six LEDs:: three normal + The Qualcomm PM8058 contains an LED block for up to six LEDs: three normal LEDs, two "flash" LEDs and one "keypad backlight" LED. The names are quoted because sometimes these LED drivers are used for wildly different things than - flash or keypad backlight:: their names are more of a suggestion than a + flash or keypad backlight: their names are more of a suggestion than a hard-wired usecase. Hardware-wise the different LEDs support slightly different output currents. diff --git a/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml b/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml index a6aaa92dbccd..65576dfdca11 100644 --- a/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml +++ b/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml @@ -11,7 +11,7 @@ maintainers: - Krzysztof Kozlowski description: | - The device is controlled through two pins:: FL_EN and EN_SET. The pins when, + The device is controlled through two pins: FL_EN and EN_SET. The pins when, asserted high, enable flash strobe and movie mode (max 1/2 of flash current) respectively. In order to add a capability of selecting the strobe signal source (e.g. CPU or camera sensor) there is an additional switch required, @@ -39,11 +39,11 @@ properties: flash-max-microamp: description: | Maximum flash LED supply current can be calculated using following - formula:: I = 1A * 162 kOhm / Rset. + formula: I = 1A * 162 kOhm / Rset. flash-max-timeout-us: description: | - Maximum flash timeout can be calculated using following formula:: + Maximum flash timeout can be calculated using following formula: T = 8.82 * 10^9 * Ct. required: diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml index f40dc9048327..1b4ef0688ca7 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -49,6 +49,7 @@ properties: - qcom,qcs615-apss-shared - qcom,sc7180-apss-shared - qcom,sc8180x-apss-shared + - qcom,shikra-apss-shared - qcom,sm7150-apss-shared - qcom,sm8150-apss-shared - const: qcom,sdm845-apss-shared diff --git a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml index 90bfde66cc4a..03359479d926 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml @@ -19,7 +19,9 @@ properties: - items: - enum: - qcom,glymur-cpucp-mbox + - qcom,hawi-cpucp-mbox - qcom,kaanapali-cpucp-mbox + - qcom,nord-cpucp-mbox - qcom,sm8750-cpucp-mbox - const: qcom,x1e80100-cpucp-mbox - enum: diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml index f5c584cf2146..a378fe8c7148 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml @@ -26,8 +26,11 @@ properties: - enum: - qcom,eliza-ipcc - qcom,glymur-ipcc + - qcom,hawi-ipcc - qcom,kaanapali-ipcc + - qcom,maili-ipcc - qcom,milos-ipcc + - qcom,nord-ipcc - qcom,qcs8300-ipcc - qcom,qdu1000-ipcc - qcom,sa8255p-ipcc diff --git a/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml b/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml index 582c6c9cae48..21118e4bae0f 100644 --- a/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml +++ b/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml @@ -14,7 +14,7 @@ description: | hooked up to a pull-up GPIO line and - optionally - the HPD line is hooked up to another GPIO line. - Please note:: the maximum voltage for the CEC line is 3.63V, for the HPD and + Please note: the maximum voltage for the CEC line is 3.63V, for the HPD and 5V lines it is 5.3V. So you may need some sort of level conversion circuitry when connecting them to a GPIO line. diff --git a/Documentation/devicetree/bindings/media/rc.yaml b/Documentation/devicetree/bindings/media/rc.yaml index dedc5a4b81ec..43d84296e70d 100644 --- a/Documentation/devicetree/bindings/media/rc.yaml +++ b/Documentation/devicetree/bindings/media/rc.yaml @@ -122,10 +122,10 @@ properties: - rc-proteus-2309 - rc-purpletv - rc-pv951 - - rc-rc5-tv - rc-rc6-mce - rc-real-audio-220-32-keys - rc-reddo + - rc-siemens-gigaset-rc20 - rc-snapstream-firefly - rc-streamzap - rc-su3000 @@ -147,11 +147,11 @@ properties: - rc-tt-1500 - rc-twinhan-dtv-cab-ci - rc-twinhan1027 + - rc-vega-s9x - rc-videomate-k100 - rc-videomate-s350 - rc-videomate-tv-pvr - rc-videostrong-kii-pro - - rc-vega-s9x - rc-wetek-hub - rc-wetek-play2 - rc-winfast diff --git a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml index 1f135618e3b6..c6f91e7bc8aa 100644 --- a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml +++ b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml @@ -28,7 +28,6 @@ properties: fan-supply: description: Phandle to the regulator that powers the fan. - $ref: /schemas/types.yaml#/definitions/phandle required: - compatible diff --git a/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml b/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml index 92a72a99fd79..940262898353 100644 --- a/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml +++ b/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml @@ -29,7 +29,7 @@ properties: type: object additionalProperties: false patternProperties: - "^(ldo(1[0-6]|[1-9])|buck[1-5])$": + "^(ldo(1[0-6]|[1-9])|buck[1-5]|vbus)$": type: object $ref: /schemas/regulator/regulator.yaml# description: LDO or buck regulator. diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml index 644c42b5e2e5..809be2756a0c 100644 --- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml +++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml @@ -80,6 +80,7 @@ properties: - qcom,pmcx0102 - qcom,pmd8028 - qcom,pmd9635 + - qcom,pmg1110 - qcom,pmh0101 - qcom,pmh0104 - qcom,pmh0110 diff --git a/Documentation/devicetree/bindings/mfd/syscon-common.yaml b/Documentation/devicetree/bindings/mfd/syscon-common.yaml index 14a08e7bc8bd..2d5eef5add54 100644 --- a/Documentation/devicetree/bindings/mfd/syscon-common.yaml +++ b/Documentation/devicetree/bindings/mfd/syscon-common.yaml @@ -32,6 +32,7 @@ properties: compatible: contains: const: syscon + # Always require a specific compatible for syscon minItems: 2 maxItems: 5 # Should be enough @@ -52,11 +53,44 @@ allOf: contains: const: simple-mfd then: + # Always require a specific compatible for syscon with simple-mfd properties: compatible: minItems: 3 maxItems: 5 + - if: + properties: + compatible: + contains: + const: simple-bus + then: + # simple-bus conflicts with syscon - if a device is a system controller + # with miscellaneous registers, then it has at least one dedicated + # function thus it is not a simple bus. Allow existing exceptions. + if: + properties: + compatible: + not: + contains: + # This list CANNOT grow + enum: + - cznic,turris1x-cpld + - img,pistachio-cr-periph + - ti,am3352-scm-conf + - ti,am4372-scm-conf + - ti,dm814-scm-conf + - ti,dm8168-scm-conf + - ti,dra7-scm-conf + - ti,omap2-scm-conf + - ti,omap3-scm-conf + - ti,omap4-sysc-padconf-global + - ti,omap5-scm-wkup-conf + - ti,omap5-sysc-padconf-global + then: + required: + - incorrect-usage-of-simple-bus-and-syscon + additionalProperties: true examples: diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml index e22867088063..fe882d3f828b 100644 --- a/Documentation/devicetree/bindings/mfd/syscon.yaml +++ b/Documentation/devicetree/bindings/mfd/syscon.yaml @@ -19,122 +19,6 @@ description: | maintainers: - Lee Jones -# Need a select with all compatibles listed for compatibility with older -# dtschema (<2024.02), so this will not be selected for other schemas having -# syscon fallback. -select: - properties: - compatible: - contains: - enum: - - airoha,en7581-pbus-csr - - al,alpine-sysfabric-service - - allwinner,sun8i-a83t-system-controller - - allwinner,sun8i-h3-system-controller - - allwinner,sun8i-v3s-system-controller - - allwinner,sun50i-a64-system-controller - - altr,l3regs - - altr,sdr-ctl - - amd,pensando-elba-syscon - - amlogic,meson-mx-assist - - amlogic,meson-mx-bootrom - - amlogic,meson8-analog-top - - amlogic,meson8b-analog-top - - amlogic,meson8-pmu - - amlogic,meson8b-pmu - - apm,merlin-poweroff-mailbox - - apm,mustang-poweroff-mailbox - - apm,xgene-csw - - apm,xgene-efuse - - apm,xgene-mcb - - apm,xgene-rb - - apm,xgene-scu - - atmel,sama5d2-sfrbu - - atmel,sama5d3-nfc-io - - atmel,sama5d3-sfrbu - - atmel,sama5d4-sfrbu - - axis,artpec6-syscon - - brcm,cru-clkset - - brcm,sr-cdru - - brcm,sr-mhb - - cirrus,ep7209-syscon1 - - cirrus,ep7209-syscon2 - - cirrus,ep7209-syscon3 - - cnxt,cx92755-uc - - econet,en751221-chip-scu - - freecom,fsg-cs2-system-controller - - fsl,imx93-aonmix-ns-syscfg - - fsl,imx93-wakeupmix-syscfg - - fsl,ls1088a-reset - - fsl,vf610-anatop - - fsl,vf610-mscm-cpucfg - - hisilicon,dsa-subctrl - - hisilicon,hi6220-sramctrl - - hisilicon,hip04-ppe - - hisilicon,pcie-sas-subctrl - - hisilicon,peri-subctrl - - hpe,gxp-sysreg - - loongson,ls1b-syscon - - loongson,ls1c-syscon - - lsi,axxia-syscon - - marvell,armada-3700-cpu-misc - - marvell,armada-3700-nb-pm - - marvell,armada-3700-avs - - marvell,armada-3700-usb2-host-device-misc - - marvell,armada-3700-usb2-host-misc - - marvell,dove-global-config - - mediatek,mt2701-pctl-a-syscfg - - mediatek,mt2712-pctl-a-syscfg - - mediatek,mt6397-pctl-pmic-syscfg - - mediatek,mt7981-topmisc - - mediatek,mt7988-topmisc - - mediatek,mt8135-pctl-a-syscfg - - mediatek,mt8135-pctl-b-syscfg - - mediatek,mt8173-pctl-a-syscfg - - mediatek,mt8365-syscfg - - microchip,lan966x-cpu-syscon - - microchip,mpfs-control-scb - - microchip,mpfs-sysreg-scb - - microchip,sam9x60-sfr - - microchip,sama7d65-ddr3phy - - microchip,sama7d65-sfrbu - - microchip,sama7g5-ddr3phy - - mscc,ocelot-cpu-syscon - - mstar,msc313-pmsleep - - nuvoton,ma35d1-sys - - nuvoton,wpcm450-shm - - nxp,s32g2-gpr - - nxp,s32g3-gpr - - qcom,apq8064-mmss-sfpb - - qcom,apq8064-sps-sic - - rockchip,px30-qos - - rockchip,rk3036-qos - - rockchip,rk3066-qos - - rockchip,rk3128-qos - - rockchip,rk3228-qos - - rockchip,rk3288-qos - - rockchip,rk3368-qos - - rockchip,rk3399-qos - - rockchip,rk3528-qos - - rockchip,rk3562-qos - - rockchip,rk3568-qos - - rockchip,rk3576-qos - - rockchip,rk3588-qos - - rockchip,rv1126-qos - - st,spear1340-misc - - stericsson,nomadik-pmu - - starfive,jh7100-sysmain - - ti,am62-opp-efuse-table - - ti,am62-usb-phy-ctrl - - ti,am625-dss-oldi-io-ctrl - - ti,am62p-cpsw-mac-efuse - - ti,am654-dss-oldi-io-ctrl - - ti,j784s4-acspcie-proxy-ctrl - - ti,j784s4-pcie-ctrl - - ti,keystone-pllctrl - required: - - compatible - properties: compatible: oneOf: diff --git a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml index 2876fdd7c6e6..24fc0752c11a 100644 --- a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml +++ b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml @@ -26,6 +26,7 @@ properties: - enum: - qcom,glymur-fastrpc - qcom,hawi-fastrpc + - qcom,nord-fastrpc - const: qcom,kaanapali-fastrpc label: diff --git a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml index 27c4060f2f91..223fcc9f651f 100644 --- a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml +++ b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml @@ -85,7 +85,7 @@ properties: description: | The value of CIU TX and RX clock phase shift value for HS400 mode operation. - Valid values for SDR and DDR CIU clock timing:: + Valid values for SDR and DDR CIU clock timing: - valid value for tx phase shift and rx phase shift is 0 to 7. - when CIU clock divider value is set to 3, all possible 8 phase shift values can be used. diff --git a/Documentation/devicetree/bindings/mtd/nand-chip.yaml b/Documentation/devicetree/bindings/mtd/nand-chip.yaml index 8800d1d07266..effdc4f99017 100644 --- a/Documentation/devicetree/bindings/mtd/nand-chip.yaml +++ b/Documentation/devicetree/bindings/mtd/nand-chip.yaml @@ -23,6 +23,15 @@ properties: description: Contains the chip-select IDs. + nand-randomizer: + description: | + Control the data randomizer feature. + 0: Disable randomizer. + 1: Enable randomizer. + If absent, the current hardware state is left unchanged. + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1] + required: - reg diff --git a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml index fc445ad5a1f1..63084f762373 100644 --- a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml +++ b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml @@ -63,11 +63,29 @@ properties: - const: ptp_ref interrupts: - maxItems: 1 + minItems: 1 + maxItems: 17 interrupt-names: + minItems: 1 items: - const: macirq + - const: tx-queue-0 + - const: tx-queue-1 + - const: tx-queue-2 + - const: tx-queue-3 + - const: tx-queue-4 + - const: tx-queue-5 + - const: tx-queue-6 + - const: tx-queue-7 + - const: rx-queue-0 + - const: rx-queue-1 + - const: rx-queue-2 + - const: rx-queue-3 + - const: rx-queue-4 + - const: rx-queue-5 + - const: rx-queue-6 + - const: rx-queue-7 iommus: minItems: 1 @@ -149,10 +167,36 @@ required: - clocks - clock-names - altr,sysmgr-syscon + - interrupts + - interrupt-names allOf: - $ref: snps,dwmac.yaml# + - if: + properties: + compatible: + contains: + const: altr,socfpga-stmmac-agilex5 + then: + properties: + interrupts: + minItems: 17 + + interrupt-names: + minItems: 17 + + dma-coherent: true + else: + properties: + interrupts: + maxItems: 1 + + interrupt-names: + maxItems: 1 + + dma-coherent: false + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/net/qcom,ipq9574-ppe.yaml b/Documentation/devicetree/bindings/net/qcom,ipq9574-ppe.yaml index 753f370b7605..6d0b21a10732 100644 --- a/Documentation/devicetree/bindings/net/qcom,ipq9574-ppe.yaml +++ b/Documentation/devicetree/bindings/net/qcom,ipq9574-ppe.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Qualcomm IPQ packet process engine (PPE) maintainers: - - Luo Jie + - Luo Jie - Lei Wei - Suruchi Agarwal - Pavithra R diff --git a/Documentation/devicetree/bindings/net/renesas,ether.yaml b/Documentation/devicetree/bindings/net/renesas,ether.yaml index f0a52f47f95a..dd7187f12a67 100644 --- a/Documentation/devicetree/bindings/net/renesas,ether.yaml +++ b/Documentation/devicetree/bindings/net/renesas,ether.yaml @@ -121,8 +121,7 @@ examples: #size-cells = <0>; phy1: ethernet-phy@1 { - compatible = "ethernet-phy-id0022.1537", - "ethernet-phy-ieee802.3-c22"; + compatible = "ethernet-phy-id0022.1537"; reg = <1>; interrupt-parent = <&irqc0>; interrupts = <0 IRQ_TYPE_LEVEL_LOW>; diff --git a/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml new file mode 100644 index 000000000000..c52f8d4bec39 --- /dev/null +++ b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvmem/airoha,smc-efuses.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Airoha SMC eFuses + +description: | + Airoha new SoC AN7581 expose banks of eFuse accessible + via specific SMC commands. + + 2 different bank of eFuse or 64 cells of 32 bit are exposed + read-only used to give information on HW Revision, PHY Calibration, + Device Model, Private Key... + +maintainers: + - Christian Marangi + +properties: + compatible: + enum: + - airoha,an7581-efuses + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + '^efuse-bank@[0-1]$': + type: object + + allOf: + - $ref: nvmem.yaml# + + properties: + reg: + description: Identify the eFuse bank. + enum: [0, 1] + + required: + - reg + + unevaluatedProperties: false + +required: + - compatible + - '#address-cells' + - '#size-cells' + +additionalProperties: false + +examples: + - | + efuse { + compatible = "airoha,an7581-efuses"; + #address-cells = <1>; + #size-cells = <0>; + + efuse-bank@0 { + reg = <0>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml index f97c6beb4766..c03e96afe564 100644 --- a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml +++ b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml @@ -25,6 +25,7 @@ properties: - const: microchip,lan9662-otpc - enum: - microchip,lan9662-otpc + - microchip,lan9691-otpc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml index 2ab047f2bb69..8134ddb54e13 100644 --- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml +++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml @@ -19,6 +19,8 @@ properties: - enum: - qcom,apq8064-qfprom - qcom,apq8084-qfprom + - qcom,glymur-qfprom + - qcom,hawi-qfprom - qcom,ipq5018-qfprom - qcom,ipq5332-qfprom - qcom,ipq5424-qfprom @@ -27,6 +29,7 @@ properties: - qcom,ipq8074-qfprom - qcom,ipq9574-qfprom - qcom,kaanapali-qfprom + - qcom,milos-qfprom - qcom,msm8226-qfprom - qcom,msm8916-qfprom - qcom,msm8917-qfprom @@ -48,6 +51,7 @@ properties: - qcom,sdm630-qfprom - qcom,sdm670-qfprom - qcom,sdm845-qfprom + - qcom,shikra-qfprom - qcom,sm6115-qfprom - qcom,sm6350-qfprom - qcom,sm6375-qfprom diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml index 9d1349855b42..4f56e8e4f100 100644 --- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml @@ -58,24 +58,48 @@ properties: items: - description: builtin MSI controller. - description: builtin DMA controller. + - description: PCIe event interrupt. + - description: builtin AER SPI standalone interrupt line. + - description: builtin PME SPI standalone interrupt line. interrupt-names: minItems: 1 items: - const: msi - const: dma + - const: intr + - const: aer + - const: pme reset-gpio: + deprecated: true description: Should specify the GPIO for controlling the PCI bus device reset signal. It's not polarity aware and defaults to active-low reset sequence (L=reset state, H=operation state) (optional required). + This property is deprecated, instead of referencing this property from the + host bridge node, use the reset-gpios property from the root port node. reset-gpio-active-high: + deprecated: true description: If present then the reset sequence using the GPIO specified in the "reset-gpio" property is reversed (H=reset state, L=operation state) (optional required). + This property is deprecated along with the reset-gpio property above, use + the reset-gpios property from the root port node. type: boolean + pcie@0: + description: + Describe the i.MX6 PCIe Root Port. + type: object + $ref: /schemas/pci/pci-pci-bridge.yaml# + + properties: + reg: + maxItems: 1 + + unevaluatedProperties: false + required: - compatible - reg @@ -231,11 +255,31 @@ allOf: - const: ref - const: extref # Optional + - if: + properties: + compatible: + enum: + - fsl,imx6q-pcie + - fsl,imx6sx-pcie + - fsl,imx6qp-pcie + - fsl,imx7d-pcie + - fsl,imx8mm-pcie + - fsl,imx8mp-pcie + - fsl,imx8mq-pcie + - fsl,imx8q-pcie + then: + properties: + interrupts: + maxItems: 2 + interrupt-names: + maxItems: 2 + unevaluatedProperties: false examples: - | #include + #include #include pcie: pcie@1ffc000 { @@ -262,5 +306,18 @@ examples: <&clks IMX6QDL_CLK_LVDS1_GATE>, <&clks IMX6QDL_CLK_PCIE_REF_125M>; clock-names = "pcie", "pcie_bus", "pcie_phy"; + + pcie_port0: pcie@0 { + compatible = "pciclass,0604"; + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + reset-gpios = <&gpio7 12 GPIO_ACTIVE_LOW>; + }; }; ... diff --git a/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml b/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml index 54e2890ae631..394bb46b38e6 100644 --- a/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml @@ -27,16 +27,20 @@ properties: - const: snps,dw-pcie reg: + minItems: 3 items: - description: Controller control and status registers. - description: PCIe configuration registers. - description: Controller application registers. + - description: Internal Address Translation Unit (iATU) registers. reg-names: + minItems: 3 items: - const: dbi - const: config - const: app + - const: atu ranges: maxItems: 1 @@ -95,8 +99,9 @@ examples: #size-cells = <2>; reg = <0xd0e00000 0x1000>, <0xd2000000 0x800000>, - <0xd0a41000 0x1000>; - reg-names = "dbi", "config", "app"; + <0xd0a41000 0x1000>, + <0xd0ec0000 0x1000>; + reg-names = "dbi", "config", "app", "atu"; linux,pci-domain = <0>; max-link-speed = <4>; bus-range = <0x00 0x08>; diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie.yaml index 0b8c78ec4f91..c009a7a52bc6 100644 --- a/Documentation/devicetree/bindings/pci/mediatek-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/mediatek-pcie.yaml @@ -14,6 +14,7 @@ properties: oneOf: - enum: - airoha,an7583-pcie + - econet,en7528-pcie - mediatek,mt2712-pcie - mediatek,mt7622-pcie - mediatek,mt7629-pcie @@ -226,6 +227,31 @@ allOf: mediatek,pbus-csr: false + - if: + properties: + compatible: + contains: + const: econet,en7528-pcie + then: + properties: + clocks: + maxItems: 1 + + clock-names: + maxItems: 1 + + resets: false + + reset-names: false + + power-domains: false + + mediatek,pbus-csr: false + + required: + - phys + - phy-names + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml new file mode 100644 index 000000000000..2c999ca6b205 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/qcom,hawi-pcie.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Hawi PCI Express Root Complex + +maintainers: + - Bjorn Andersson + - Manivannan Sadhasivam + +description: + Qualcomm Hawi SoC (and compatible) PCIe root complex controller is based on + the Synopsys DesignWare PCIe IP. + +properties: + compatible: + const: qcom,hawi-pcie + + reg: + items: + - description: Qualcomm specific registers + - description: DesignWare PCIe registers + - description: External local bus interface registers + - description: ATU address space + - description: PCIe configuration space + - description: MHI registers + + reg-names: + items: + - const: parf + - const: dbi + - const: elbi + - const: atu + - const: config + - const: mhi + + clocks: + items: + - description: PCIe Auxiliary clock + - description: PCIe Configuration clock + - description: PCIe Master AXI clock + - description: PCIe Slave AXI clock + - description: PCIe Slave Q2A AXI clock + - description: PCIe Aggre NoC AXI clock + - description: PCIe Config NoC AXI clock + + clock-names: + items: + - const: aux + - const: cfg + - const: bus_master + - const: bus_slave + - const: slave_q2a + - const: noc_aggr + - const: cnoc_sf_axi + + interrupts: + minItems: 9 + maxItems: 9 + + interrupt-names: + items: + - const: msi0 + - const: msi1 + - const: msi2 + - const: msi3 + - const: msi4 + - const: msi5 + - const: msi6 + - const: msi7 + - const: global + + resets: + items: + - description: PCIe core reset + - description: PCIe link down reset + + reset-names: + items: + - const: pci + - const: link_down + +required: + - power-domains + - resets + - reset-names + +allOf: + - $ref: qcom,pcie-common.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + pcie@1c00000 { + compatible = "qcom,hawi-pcie"; + reg = <0 0x01c00000 0 0x3000>, + <0 0x40000000 0 0xf1d>, + <0 0x40000f20 0 0xa8>, + <0 0x40001000 0 0x1000>, + <0 0x40100000 0 0x100000>, + <0 0x01c03000 0 0x1000>; + reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; + ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x3d00000>; + + bus-range = <0x00 0xff>; + device_type = "pci"; + linux,pci-domain = <0>; + num-lanes = <2>; + + #address-cells = <3>; + #size-cells = <2>; + + clocks = <&gcc_pcie_0_aux_clk>, + <&gcc_pcie_0_cfg_ahb_clk>, + <&gcc_pcie_0_mstr_axi_clk>, + <&gcc_pcie_0_slv_axi_clk>, + <&gcc_pcie_0_slv_q2a_axi_clk>, + <&gcc_aggre_noc_pcie_axi_clk>, + <&gcc_cnoc_pcie_sf_axi_clk>; + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a", + "noc_aggr", + "cnoc_sf_axi"; + + dma-coherent; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", "msi1", "msi2", "msi3", + "msi4", "msi5", "msi6", "msi7", "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc 0 0 GIC_ESPI 213 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ + <0 0 0 2 &intc 0 0 GIC_ESPI 214 IRQ_TYPE_LEVEL_HIGH>, /* int_b */ + <0 0 0 3 &intc 0 0 GIC_ESPI 215 IRQ_TYPE_LEVEL_HIGH>, /* int_c */ + <0 0 0 4 &intc 0 0 GIC_ESPI 216 IRQ_TYPE_LEVEL_HIGH>; /* int_d */ + + interconnects = <&pcie_anoc_master_pcie_0 QCOM_ICC_TAG_ALWAYS + &mc_virt_slave_ebi1 QCOM_ICC_TAG_ALWAYS>, + <&gem_noc_master_appss_proc QCOM_ICC_TAG_ACTIVE_ONLY + &cnoc_main_slave_pcie_0 QCOM_ICC_TAG_ACTIVE_ONLY>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &apps_smmu 0x1000 0x1>, + <0x100 &apps_smmu 0x1001 0x1>; + + pinctrl-0 = <&pcie0_default_state>; + pinctrl-names = "default"; + + power-domains = <&gcc_pcie_0_phy_gdsc>; + + resets = <&gcc_pcie_0_bcr>, + <&gcc_pcie_0_link_down_bcr>; + reset-names = "pci", "link_down"; + + msi-map = <0x0 &gic_its 0x1000 0x1>, + <0x100 &gic_its 0x1001 0x1>; + msi-map-mask = <0xff00>; + + pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + phys = <&pcie0_phy>; + wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>; + reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml index 4be342cc04e1..533079811881 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml @@ -17,8 +17,10 @@ properties: - qcom,pcie-ipq9574 - items: - enum: + - qcom,pcie-ipq5210 - qcom,pcie-ipq5332 - qcom,pcie-ipq5424 + - qcom,pcie-ipq9650 - const: qcom,pcie-ipq9574 reg: @@ -82,6 +84,19 @@ required: allOf: - $ref: qcom,pcie-common.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,pcie-ipq5210 + - qcom,pcie-ipq9650 + then: + properties: + interrupts: + minItems: 9 + interrupt-names: + minItems: 9 unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml index 3a94a9c1bb15..fb706b1397a3 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-sm8550.yaml @@ -20,6 +20,7 @@ properties: - const: qcom,pcie-sm8550 - items: - enum: + - qcom,eliza-pcie - qcom,kaanapali-pcie - qcom,sar2130p-pcie - qcom,pcie-sm8650 @@ -91,6 +92,55 @@ required: allOf: - $ref: qcom,pcie-common.yaml# + - if: + properties: + compatible: + contains: + const: qcom,eliza-pcie + then: + properties: + reg: + minItems: 6 + reg-names: + minItems: 6 + + - if: + properties: + compatible: + contains: + const: qcom,eliza-pcie + then: + properties: + clocks: + minItems: 8 + maxItems: 8 + clock-names: + minItems: 8 + maxItems: 8 + + - if: + properties: + compatible: + contains: + const: qcom,eliza-pcie + then: + properties: + interrupts: + minItems: 9 + interrupt-names: + minItems: 9 + + - if: + properties: + compatible: + contains: + const: qcom,eliza-pcie + then: + properties: + resets: + minItems: 2 + reset-names: + minItems: 2 unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml index a67108c48feb..09d78bd1a577 100644 --- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml @@ -4,21 +4,28 @@ $id: http://devicetree.org/schemas/pci/renesas,r9a08g045-pcie.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Renesas RZ/G3S PCIe host controller +title: Renesas RZ/G3S PCIe host controller (and similar SoCs) maintainers: - Claudiu Beznea -description: - Renesas RZ/G3{E,S} PCIe host controllers comply with PCIe - Base Specification 4.0 and support up to 5 GT/s (Gen2) for RZ/G3S and - up to 8 GT/s (Gen3) for RZ/G3E. +description: | + PCIe host controller found in Renesas RZ/G3S and similar SoCs complies + with PCIe Base Specification 4.0 and supports different link speeds + depending on the SoC variant: + - Gen2 (5 GT/s): RZ/G3S + - Gen3 (8 GT/s): RZ/G3E, RZ/V2H(P), RZ/V2N properties: compatible: - enum: - - renesas,r9a08g045-pcie # RZ/G3S - - renesas,r9a09g047-pcie # RZ/G3E + oneOf: + - enum: + - renesas,r9a08g045-pcie # RZ/G3S + - renesas,r9a09g047-pcie # RZ/G3E + - renesas,r9a09g057-pcie # RZ/V2H(P) + - items: + - const: renesas,r9a09g056-pcie # RZ/V2N + - const: renesas,r9a09g047-pcie reg: maxItems: 1 @@ -133,7 +140,13 @@ properties: - clkl1pm clock request state - power off information in L2 state - errors (fatal, non-fatal, correctable) - $ref: /schemas/types.yaml#/definitions/phandle + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle to system controller + - description: PCIe controller index + enum: [0, 1] + minItems: 1 patternProperties: "^pcie@0,[0-0]$": @@ -152,6 +165,7 @@ patternProperties: enum: - 0x0033 - 0x0039 + - 0x003b clocks: items: @@ -213,7 +227,9 @@ allOf: properties: compatible: contains: - const: renesas,r9a09g047-pcie + enum: + - renesas,r9a09g047-pcie + - renesas,r9a09g057-pcie then: properties: interrupts: @@ -228,6 +244,25 @@ allOf: maxItems: 1 reset-names: maxItems: 1 + - if: + properties: + compatible: + contains: + const: renesas,r9a09g057-pcie + then: + properties: + num-lanes: + enum: [2, 4] + renesas,sysc: + items: + - minItems: 2 + required: + - num-lanes + else: + properties: + renesas,sysc: + items: + - maxItems: 1 unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml b/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml index f8b7ca57fff1..ab482488b047 100644 --- a/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml +++ b/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml @@ -30,6 +30,8 @@ properties: device-id: const: 0x2042 + dma-coherent: true + msi-parent: true allOf: @@ -60,5 +62,6 @@ examples: vendor-id = <0x1f1c>; device-id = <0x2042>; cdns,no-bar-match-nbits = <48>; + dma-coherent; msi-parent = <&msi>; }; diff --git a/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml b/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml new file mode 100644 index 000000000000..512b935bf5d1 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/ultrarisc,dp1000-pcie.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: UltraRISC DP1000 PCIe Host Controller + +description: + UltraRISC DP1000 SoC PCIe host controller is based on the DesignWare PCIe IP. + +maintainers: + - Xincheng Zhang + - Jia Wang + +allOf: + - $ref: /schemas/pci/snps,dw-pcie.yaml# + +properties: + compatible: + const: ultrarisc,dp1000-pcie + + reg: + items: + - description: Data Bus Interface (DBI) registers. + - description: PCIe configuration space region. + + reg-names: + items: + - const: dbi + - const: config + + num-lanes: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [4, 16] + description: Number of lanes to use. + + interrupts: + items: + - description: MSI interrupt + - description: Legacy INTA interrupt + - description: Legacy INTB interrupt + - description: Legacy INTC interrupt + - description: Legacy INTD interrupt + + interrupt-names: + items: + - const: msi + - const: inta + - const: intb + - const: intc + - const: intd + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pcie@21000000 { + compatible = "ultrarisc,dp1000-pcie"; + reg = <0x0 0x21000000 0x0 0x01000000>, + <0x0 0x4fff0000 0x0 0x00010000>; + reg-names = "dbi", "config"; + ranges = <0x81000000 0x0 0x4fbf0000 0x0 0x4fbf0000 0x0 0x00400000>, + <0x82000000 0x0 0x40000000 0x0 0x40000000 0x0 0x0fbf0000>, + <0xc3000000 0x40 0x00000000 0x40 0x00000000 0xd 0x00000000>; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + device_type = "pci"; + dma-coherent; + bus-range = <0x0 0xff>; + num-lanes = <16>; + interrupt-parent = <&plic>; + interrupts = <43>, <44>, <45>, <46>, <47>; + interrupt-names = "msi", "inta", "intb", "intc", "intd"; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0x0 0x0 0x0 0x1 &plic 44>, + <0x0 0x0 0x0 0x2 &plic 45>, + <0x0 0x0 0x0 0x3 &plic 46>, + <0x0 0x0 0x0 0x4 &plic 47>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml b/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml new file mode 100644 index 000000000000..61700b80e93f --- /dev/null +++ b/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/axiado,ax3000-emmc-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Axiado AX3000 Arasan eMMC PHY + +maintainers: + - SriNavmani A + - Tzu-Hao Wei + - Prasad Bolisetty + +properties: + compatible: + const: axiado,ax3000-emmc-phy + + reg: + maxItems: 1 + + "#phy-cells": + const: 0 + +required: + - compatible + - reg + - "#phy-cells" + +additionalProperties: false + +examples: + - | + phy@80801c00 { + compatible = "axiado,ax3000-emmc-phy"; + reg = <0x80801c00 0x1000>; + #phy-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml new file mode 100644 index 000000000000..987d396c1c64 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/econet,en751221-pcie-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: EcoNet PCI-Express PHY for EcoNet EN751221 and EN7528 + +maintainers: + - Caleb James DeLisle + +description: + The PCIe PHY supports physical layer functionality for PCIe Gen1 and + Gen1/Gen2 ports. On these SoCs, port 0 is a Gen1-only port while + port 1 is Gen1/Gen2 capable. + +properties: + compatible: + enum: + - econet,en751221-pcie-gen1 + - econet,en751221-pcie-gen2 + - econet,en7528-pcie-gen1 + - econet,en7528-pcie-gen2 + + reg: + maxItems: 1 + + "#phy-cells": + const: 0 + +required: + - compatible + - reg + - "#phy-cells" + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <1>; + #size-cells = <1>; + + pcie-phy@1faf2000 { + compatible = "econet,en7528-pcie-gen1"; + reg = <0x1faf2000 0x1000>; + #phy-cells = <0>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/phy/fsl,lynx-10g.yaml b/Documentation/devicetree/bindings/phy/fsl,lynx-10g.yaml new file mode 100644 index 000000000000..2e7d0abfa71a --- /dev/null +++ b/Documentation/devicetree/bindings/phy/fsl,lynx-10g.yaml @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/fsl,lynx-10g.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale Lynx 10G SerDes PHY + +maintainers: + - Vladimir Oltean + +description: + The 10G Lynx is a multi-protocol SerDes block which handles networking, PCIe, + SATA and other high-speed interfaces. It is present on most QorIQ and + Layerscape SoCs. The register map is common, but the integration is + SoC-specific, with the differences consisting in register endianness, the + number of lanes, protocol converters available per lane and their location in + the PCCR registers. Some SoCs have multiple SerDes blocks and those differ in + their protocol capabilities per lane. + +properties: + compatible: + enum: + - fsl,ls1028a-serdes + - fsl,ls1046a-serdes1 + - fsl,ls1046a-serdes2 + - fsl,ls1088a-serdes1 + - fsl,ls1088a-serdes2 + - fsl,ls2088a-serdes1 + - fsl,ls2088a-serdes2 + + reg: + maxItems: 1 + + big-endian: true + + "#phy-cells": + const: 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^phy@[0-7]$": + type: object + description: SerDes lane (single RX/TX differential pair) + + properties: + reg: + minimum: 0 + maximum: 7 + description: Lane index as seen in register map + + "#phy-cells": + const: 0 + + required: + - reg + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - reg + - "#phy-cells" + - "#address-cells" + - "#size-cells" + +allOf: + - if: + properties: + compatible: + contains: + enum: + - fsl,ls1028a-serdes + - fsl,ls1046a-serdes1 + - fsl,ls1046a-serdes2 + - fsl,ls1088a-serdes1 + - fsl,ls1088a-serdes2 + then: + patternProperties: + "^phy@[0-7]$": + properties: + reg: + minimum: 0 + maximum: 3 + - if: + properties: + compatible: + enum: + - fsl,ls1046a-serdes1 + - fsl,ls1046a-serdes2 + then: + required: + - big-endian + else: + properties: + big-endian: false + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + serdes@1ea0000 { + compatible = "fsl,ls1028a-serdes"; + reg = <0x0 0x1ea0000 0x0 0xffff>; + #address-cells = <1>; + #size-cells = <0>; + #phy-cells = <1>; + + phy@0 { + reg = <0>; + #phy-cells = <0>; + }; + + phy@1 { + reg = <1>; + #phy-cells = <0>; + }; + + phy@2 { + reg = <2>; + #phy-cells = <0>; + }; + + phy@3 { + reg = <3>; + #phy-cells = <0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml b/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml index e96229c2f8fb..d73591315d4b 100644 --- a/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml +++ b/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml @@ -9,10 +9,37 @@ title: Freescale Lynx 28G SerDes PHY maintainers: - Ioana Ciornei +description: + The Lynx 28G is a multi-lane, multi-protocol SerDes (PCIe, SATA, Ethernet) + present in multiple instances on NXP LX2160A and LX2162A SoCs. All instances + share a common register map and programming model, however they differ in + supported protocols per lane in a way that is not detectable by said + programming model without prior knowledge. The distinction is made through + the compatible string. + properties: compatible: - enum: - - fsl,lynx-28g + oneOf: + - const: fsl,lynx-28g + deprecated: true + description: + Legacy compatibility string for Lynx 28G SerDes. Any assumption + regarding whether a certain lane supports a certain protocol may + be incorrect. Deprecated except when used as a fallback. Use + device-specific strings instead. + - items: + - const: fsl,lx2160a-serdes1 + - const: fsl,lynx-28g + - items: + - const: fsl,lx2160a-serdes2 + - const: fsl,lynx-28g + - items: + - const: fsl,lx2162a-serdes1 + - const: fsl,lynx-28g + - items: + - const: fsl,lx2162a-serdes2 + - const: fsl,lynx-28g + - const: fsl,lx2160a-serdes3 reg: maxItems: 1 @@ -51,6 +78,21 @@ required: - reg - "#phy-cells" +allOf: + # LX2162A SerDes 1 has fewer lanes than the others + - if: + properties: + compatible: + contains: + const: fsl,lx2162a-serdes1 + then: + patternProperties: + "^phy@[0-7]$": + properties: + reg: + minimum: 4 + maximum: 7 + additionalProperties: false examples: @@ -60,7 +102,7 @@ examples: #size-cells = <2>; serdes@1ea0000 { - compatible = "fsl,lynx-28g"; + compatible = "fsl,lx2160a-serdes1", "fsl,lynx-28g"; reg = <0x0 0x1ea0000 0x0 0x1e30>; #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/phy/nxp,tja1145.yaml b/Documentation/devicetree/bindings/phy/nxp,tja1145.yaml new file mode 100644 index 000000000000..fb068f7d774f --- /dev/null +++ b/Documentation/devicetree/bindings/phy/nxp,tja1145.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/nxp,tja1145.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TJA1145 CAN transceiver + +maintainers: + - Dimitri Fedrau + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + const: nxp,tja1145 + + interrupts: + maxItems: 1 + + reg: + maxItems: 1 + + "#phy-cells": + const: 0 + + spi-cpha: true + + spi-max-frequency: + maximum: 4000000 + + spi-cs-setup-delay-ns: + minimum: 50 + default: 50 + + spi-cs-hold-delay-ns: + minimum: 50 + default: 50 + + spi-cs-inactive-delay-ns: + minimum: 250 + default: 250 + + vcc-supply: + description: + CAN transceiver supply voltage + + vio-supply: + description: + Supply voltage for I/O level adaptor + + vbat-supply: + description: + Battery supply voltage + +required: + - compatible + - reg + - "#phy-cells" + - spi-cpha + +additionalProperties: false + +examples: + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + can-phy@0 { + compatible = "nxp,tja1145"; + interrupt-parent = <&gpio0>; + interrupts = <31 IRQ_TYPE_LEVEL_LOW>; + reg = <0>; + #phy-cells = <0>; + spi-cpha; + spi-max-frequency = <4000000>; + spi-cs-setup-delay-ns = <50>; + spi-cs-hold-delay-ns = <50>; + spi-cs-inactive-delay-ns = <250>; + vcc-supply = <®_5v0>; + vio-supply = <®_3v3>; + vbat-supply = <®_24v0>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index 4a1daae3d8d4..0bf8bf4f66ac 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -74,6 +74,7 @@ allOf: compatible: enum: - qcom,glymur-dp-phy + - qcom,sa8775p-edp-phy - qcom,x1e80100-dp-phy then: properties: diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index 39851ba9de43..449c2a7e5fec 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -30,6 +30,10 @@ properties: - qcom,sdm660-qusb2-phy - qcom,sm4250-qusb2-phy - qcom,sm6115-qusb2-phy + - items: + - enum: + - qcom,ipq5210-qusb2-phy + - const: qcom,ipq6018-qusb2-phy - items: - enum: - qcom,sc7180-qusb2-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml index 3a35120a77ec..108cf9dc86ea 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml @@ -16,6 +16,8 @@ description: properties: compatible: enum: + - qcom,eliza-qmp-gen3x1-pcie-phy + - qcom,eliza-qmp-gen3x2-pcie-phy - qcom,glymur-qmp-gen4x2-pcie-phy - qcom,glymur-qmp-gen5x4-pcie-phy - qcom,kaanapali-qmp-gen3x2-pcie-phy @@ -136,13 +138,22 @@ allOf: items: - description: port a - description: port b - required: - - qcom,4ln-config-sel else: properties: reg: maxItems: 1 + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-qmp-gen3x4-pcie-phy + - qcom,x1e80100-qmp-gen4x4-pcie-phy + then: + required: + - qcom,4ln-config-sel + - if: properties: compatible: @@ -181,6 +192,8 @@ allOf: compatible: contains: enum: + - qcom,eliza-qmp-gen3x1-pcie-phy + - qcom,eliza-qmp-gen3x2-pcie-phy - qcom,glymur-qmp-gen4x2-pcie-phy - qcom,glymur-qmp-gen5x4-pcie-phy - qcom,qcs8300-qmp-gen4x2-pcie-phy @@ -206,6 +219,8 @@ allOf: compatible: contains: enum: + - qcom,eliza-qmp-gen3x1-pcie-phy + - qcom,eliza-qmp-gen3x2-pcie-phy - qcom,glymur-qmp-gen4x2-pcie-phy - qcom,glymur-qmp-gen5x4-pcie-phy - qcom,kaanapali-qmp-gen3x2-pcie-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml index 9616c736b6d4..b2c5c9a375a3 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml @@ -31,6 +31,7 @@ properties: - items: - enum: - qcom,eliza-qmp-ufs-phy + - qcom,nord-qmp-ufs-phy - const: qcom,sm8650-qmp-ufs-phy - items: - enum: diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb3-uni-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb3-uni-phy.yaml index 623c2f8c7d22..01342823e57f 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb3-uni-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb3-uni-phy.yaml @@ -15,26 +15,32 @@ description: properties: compatible: - enum: - - qcom,glymur-qmp-usb3-uni-phy - - qcom,ipq5424-qmp-usb3-phy - - qcom,ipq6018-qmp-usb3-phy - - qcom,ipq8074-qmp-usb3-phy - - qcom,ipq9574-qmp-usb3-phy - - qcom,msm8996-qmp-usb3-phy - - qcom,qcs8300-qmp-usb3-uni-phy - - qcom,qdu1000-qmp-usb3-uni-phy - - qcom,sa8775p-qmp-usb3-uni-phy - - qcom,sc8180x-qmp-usb3-uni-phy - - qcom,sc8280xp-qmp-usb3-uni-phy - - qcom,sdm845-qmp-usb3-uni-phy - - qcom,sdx55-qmp-usb3-uni-phy - - qcom,sdx65-qmp-usb3-uni-phy - - qcom,sdx75-qmp-usb3-uni-phy - - qcom,sm8150-qmp-usb3-uni-phy - - qcom,sm8250-qmp-usb3-uni-phy - - qcom,sm8350-qmp-usb3-uni-phy - - qcom,x1e80100-qmp-usb3-uni-phy + oneOf: + - items: + - enum: + - qcom,glymur-qmp-usb3-uni-phy + - qcom,ipq5424-qmp-usb3-phy + - qcom,ipq6018-qmp-usb3-phy + - qcom,ipq8074-qmp-usb3-phy + - qcom,ipq9574-qmp-usb3-phy + - qcom,msm8996-qmp-usb3-phy + - qcom,qcs8300-qmp-usb3-uni-phy + - qcom,qdu1000-qmp-usb3-uni-phy + - qcom,sa8775p-qmp-usb3-uni-phy + - qcom,sc8180x-qmp-usb3-uni-phy + - qcom,sc8280xp-qmp-usb3-uni-phy + - qcom,sdm845-qmp-usb3-uni-phy + - qcom,sdx55-qmp-usb3-uni-phy + - qcom,sdx65-qmp-usb3-uni-phy + - qcom,sdx75-qmp-usb3-uni-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + - qcom,x1e80100-qmp-usb3-uni-phy + - items: + - enum: + - qcom,ipq5210-qmp-usb3-phy + - const: qcom,ipq9574-qmp-usb3-phy reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml index 3d537b7f9985..4eff92343ce4 100644 --- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml @@ -16,6 +16,10 @@ description: properties: compatible: oneOf: + - items: + - enum: + - qcom,eliza-qmp-usb3-dp-phy + - const: qcom,sm8650-qmp-usb3-dp-phy - items: - enum: - qcom,kaanapali-qmp-usb3-dp-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml index 854f70af0a6c..096f6b546632 100644 --- a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml @@ -17,6 +17,7 @@ properties: oneOf: - items: - enum: + - qcom,eliza-snps-eusb2-phy - qcom,milos-snps-eusb2-phy - qcom,sar2130p-snps-eusb2-phy - qcom,sdx75-snps-eusb2-phy diff --git a/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml b/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml index 58e735b5dd05..f50fc69fbbe4 100644 --- a/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml +++ b/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml @@ -20,6 +20,7 @@ properties: - rockchip,rk3328-usb2phy - rockchip,rk3366-usb2phy - rockchip,rk3399-usb2phy + - rockchip,rk3528-usb2phy - rockchip,rk3562-usb2phy - rockchip,rk3568-usb2phy - rockchip,rk3576-usb2phy @@ -41,11 +42,15 @@ properties: maxItems: 3 clock-names: - minItems: 1 - items: + oneOf: - const: phyclk - - const: aclk - - const: aclk_slv + - items: + - const: phyclk + - const: pclk + - items: + - const: phyclk + - const: aclk + - const: aclk_slv assigned-clocks: description: @@ -65,6 +70,9 @@ properties: description: Muxed interrupt for both ports maxItems: 1 + power-domains: + maxItems: 1 + resets: maxItems: 2 @@ -145,6 +153,21 @@ anyOf: - host-port allOf: + - if: + properties: + compatible: + contains: + enum: + - rockchip,rk3528-usb2phy + - rockchip,rk3568-usb2phy + - rockchip,rv1108-usb2phy + then: + required: + - rockchip,usbgrf + else: + properties: + rockchip,usbgrf: false + - if: properties: compatible: @@ -204,6 +227,19 @@ allOf: clock-names: maxItems: 1 + - if: + properties: + compatible: + contains: + enum: + - rockchip,rk3528-usb2phy + then: + properties: + clocks: + minItems: 2 + clock-names: + minItems: 2 + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml index 16967ef8e9ec..87b6a35b2626 100644 --- a/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml +++ b/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml @@ -13,14 +13,14 @@ maintainers: description: | For samsung,s5pv210-mipi-video-phy compatible PHYs the second cell in the - PHY specifier identifies the PHY and its meaning is as follows:: + PHY specifier identifies the PHY and its meaning is as follows: 0 - MIPI CSIS 0, 1 - MIPI DSIM 0, 2 - MIPI CSIS 1, 3 - MIPI DSIM 1. samsung,exynos5420-mipi-video-phy and samsung,exynos5433-mipi-video-phy - support additional fifth PHY:: + support additional fifth PHY: 4 - MIPI CSIS 2. properties: diff --git a/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml index d9f22a801cbf..7db7605a82e2 100644 --- a/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml @@ -14,7 +14,7 @@ maintainers: description: | The first phandle argument in the PHY specifier identifies the PHY, its meaning is compatible dependent. For the currently supported SoCs (Exynos4210 - and Exynos4212) it is as follows:: + and Exynos4212) it is as follows: 0 - USB device ("device"), 1 - USB host ("host"), 2 - HSIC0 ("hsic0"), diff --git a/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml index 4562e0468f4f..a1b3d9e6a094 100644 --- a/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml +++ b/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml @@ -14,7 +14,7 @@ maintainers: description: | For samsung,exynos5250-usbdrd-phy and samsung,exynos5420-usbdrd-phy compatible PHYs, the second cell in the PHY specifier identifies the - PHY id, which is interpreted as follows:: + PHY id, which is interpreted as follows: 0 - UTMI+ type phy, 1 - PIPE3 type phy. diff --git a/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml index 43eaca90d88c..18025e5f60d6 100644 --- a/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml @@ -4,14 +4,16 @@ $id: http://devicetree.org/schemas/phy/spacemit,usb2-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: SpacemiT K1 SoC USB 2.0 PHY +title: SpacemiT K1/K3 SoC USB 2.0 PHY maintainers: - Ze Huang properties: compatible: - const: spacemit,k1-usb2-phy + enum: + - spacemit,k1-usb2-phy + - spacemit,k3-usb2-phy reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml b/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml new file mode 100644 index 000000000000..ca4605f1d664 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/ti,ds125df111.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI DS125DF111 Retimer PHY + +description: + This binding describes the TI DS125DF111 multi-protocol Retimer PHY. + +maintainers: + - Ioana Ciornei + +properties: + compatible: + const: ti,ds125df111 + + reg: + maxItems: 1 + + "#phy-cells": + const: 1 + description: | + The phandle's argument in the PHY specifier selects one of the two + channels of the retimer + +required: + - compatible + - reg + - "#phy-cells" + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + phy@18 { + compatible = "ti,ds125df111"; + reg = <0x18>; + #phy-cells = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml index be41b4547ec6..60b644a4c639 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml @@ -47,15 +47,20 @@ description: | properties: compatible: - enum: - - ti,am3352-phy-gmii-sel - - ti,dra7xx-phy-gmii-sel - - ti,am43xx-phy-gmii-sel - - ti,dm814-phy-gmii-sel - - ti,am654-phy-gmii-sel - - ti,j7200-cpsw5g-phy-gmii-sel - - ti,j721e-cpsw9g-phy-gmii-sel - - ti,j784s4-cpsw9g-phy-gmii-sel + oneOf: + - enum: + - ti,am3352-phy-gmii-sel + - ti,dra7xx-phy-gmii-sel + - ti,am43xx-phy-gmii-sel + - ti,dm814-phy-gmii-sel + - ti,am654-phy-gmii-sel + - ti,j7200-cpsw5g-phy-gmii-sel + - ti,j721e-cpsw9g-phy-gmii-sel + - ti,j784s4-cpsw9g-phy-gmii-sel + - items: + - enum: + - ti,j722s-phy-gmii-sel + - const: ti,am654-phy-gmii-sel reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml index 3f16ff14484d..0653252c18d8 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml @@ -12,13 +12,18 @@ maintainers: properties: compatible: - enum: - - ti,j721e-wiz-16g - - ti,j721e-wiz-10g - - ti,j721s2-wiz-10g - - ti,am64-wiz-10g - - ti,j7200-wiz-10g - - ti,j784s4-wiz-10g + oneOf: + - enum: + - ti,j721e-wiz-16g + - ti,j721e-wiz-10g + - ti,j721s2-wiz-10g + - ti,am64-wiz-10g + - ti,j7200-wiz-10g + - ti,j784s4-wiz-10g + - items: + - enum: + - ti,j722s-wiz-10g + - const: ti,am64-wiz-10g power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml index 76944fd14e2c..fe7cef4fef6a 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml @@ -356,6 +356,7 @@ patternProperties: - I3C8 - I3C9 - JTAGM1 + - JTAGM1TRST - LPC0 - LPC1 - LTPI diff --git a/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml index a24286e4def6..36f2393fa406 100644 --- a/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -# Copyright 2022 NXP +# Copyright 2022, 2026 NXP %YAML 1.2 --- $id: http://devicetree.org/schemas/pinctrl/nxp,s32g2-siul2-pinctrl.yaml# @@ -17,8 +17,10 @@ description: | SIUL2_0 @ 0x4009c000 SIUL2_1 @ 0x44010000 - Every SIUL2 region has multiple register types, and here only MSCR and - IMCR registers need to be revealed for kernel to configure pinmux. + Every SIUL2 region has multiple register types. MSCR and IMCR registers + need to be revealed for kernel to configure pinmux. PGPDO and PGPDI + registers are used for GPIO output/input operations. EIRQ registers + are used for external interrupt configuration. Please note that some register indexes are reserved in S32G2, such as MSCR102-MSCR111, MSCR123-MSCR143, IMCR84-IMCR118 and IMCR398-IMCR429. @@ -29,14 +31,22 @@ properties: - nxp,s32g2-siul2-pinctrl reg: + minItems: 6 description: | - A list of MSCR/IMCR register regions to be reserved. + A list of MSCR/IMCR/PGPDO/PGPDI/EIRQ register regions to be reserved. - MSCR (Multiplexed Signal Configuration Register) An MSCR register can configure the associated pin as either a GPIO pin or a function output pin depends on the selected signal source. - IMCR (Input Multiplexed Signal Configuration Register) An IMCR register can configure the associated pin as function input pin depends on the selected signal source. + - PGPDO (Parallel GPIO Pad Data Out Register) + A PGPDO register is used to set the output value of a GPIO pin. + - PGPDI (Parallel GPIO Pad Data In Register) + A PGPDI register is used to read the input value of a GPIO pin. + - EIRQ (External Interrupt Request) + EIRQ registers are used to configure and manage external interrupts. + items: - description: MSCR registers group 0 in SIUL2_0 - description: MSCR registers group 1 in SIUL2_1 @@ -44,6 +54,28 @@ properties: - description: IMCR registers group 0 in SIUL2_0 - description: IMCR registers group 1 in SIUL2_1 - description: IMCR registers group 2 in SIUL2_1 + - description: PGPDO registers in SIUL2_0 + - description: PGPDI registers in SIUL2_0 + - description: PGPDO registers in SIUL2_1 + - description: PGPDI registers in SIUL2_1 + - description: EIRQ registers in SIUL2_1 + + gpio-controller: true + + "#gpio-cells": + const: 2 + + gpio-ranges: + minItems: 1 + maxItems: 4 + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + interrupts: + maxItems: 1 patternProperties: '-pins$': @@ -86,11 +118,38 @@ required: - compatible - reg +oneOf: + - description: Legacy pinctrl-only node + properties: + reg: + maxItems: 6 + + gpio-controller: false + "#gpio-cells": false + gpio-ranges: false + interrupt-controller: false + "#interrupt-cells": false + interrupts: false + + - description: Pinctrl node with GPIO and external interrupt support + required: + - gpio-controller + - "#gpio-cells" + - gpio-ranges + - interrupt-controller + - "#interrupt-cells" + - interrupts + properties: + reg: + minItems: 11 + additionalProperties: false examples: - | - pinctrl@4009c240 { + #include + + pinctrl: pinctrl@4009c240 { compatible = "nxp,s32g2-siul2-pinctrl"; /* MSCR0-MSCR101 registers on siul2_0 */ @@ -104,7 +163,26 @@ examples: /* IMCR119-IMCR397 registers on siul2_1 */ <0x44010c1c 0x45c>, /* IMCR430-IMCR495 registers on siul2_1 */ - <0x440110f8 0x108>; + <0x440110f8 0x108>, + /* PGPDO registers on siul2_0 */ + <0x4009d700 0x10>, + /* PGPDI registers on siul2_0 */ + <0x4009d740 0x10>, + /* PGPDO registers on siul2_1 */ + <0x44011700 0x18>, + /* PGPDI registers on siul2_1 */ + <0x44011740 0x18>, + /* EIRQ registers on siul2_1 */ + <0x44010010 0x34>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 0 102>, + <&pinctrl 112 112 79>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupts = ; llce-can0-pins { llce-can0-grp0 { diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml new file mode 100644 index 000000000000..64fe90b2391b --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,maili-tlmm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Maili TLMM block + +maintainers: + - Jingyi Wang + +description: + Top Level Mode Multiplexer pin controller in Qualcomm Maili SoC. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,maili-tlmm + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + gpio-reserved-ranges: + minItems: 1 + maxItems: 113 + + gpio-line-names: + maxItems: 226 + +patternProperties: + "-state$": + oneOf: + - $ref: "#/$defs/qcom-maili-tlmm-state" + - patternProperties: + "-pins$": + $ref: "#/$defs/qcom-maili-tlmm-state" + additionalProperties: false + +$defs: + qcom-maili-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state + unevaluatedProperties: false + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-5])$" + - enum: [ ufs_reset, sdc2_clk, sdc2_cmd, sdc2_data ] + minItems: 1 + maxItems: 36 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + enum: [ gpio, aoss_cti, atest_char, atest_usb, audio_ext_mclk, + audio_ref_clk, cam_mclk, cci_async_in, cci_i2c0, cci_i2c1, + cci_i2c2, cci_i2c3, cci_timer, coex_espmi, coex_uart1_rx, + coex_uart1_tx, dbg_out_clk, ddr_bist, ddr_pxi, dp_hot, egpio, + gcc_gp, gnss_adc, host2wlan_sol, host_rst, i2chub0_se0, + i2chub0_se1, i2chub0_se2, i2chub0_se3, i2chub0_se4, i2s0, i2s1, + ibi_i3c, ibi_i3c_qup5_se0, jitter_bist, mdp_esync0, mdp_esync1, + mdp_esync2, mdp_vsync, mdp_vsync_e, mdp_vsync_p, mdp_vsync0_out, + mdp_vsync1_out, mdp_vsync2_out, mdp_vsync3_out, mdp_vsync5_out, + modem_pps_in, modem_pps_out, nav_gpio, nav_gpio0, nav_gpio3, + nav_rffe, pcie0_clk_req_n, pcie1_clk_req_n, pcie1_rst_n, + phase_flag, pll_bist_sync, pll_clk_aux, qdss_cti, qlink, qspi, + qspi_clk, qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se3, + qup1_se4, qup1_se5, qup1_se6, qup1_se7, qup2_se0, qup2_se1, + qup2_se2, qup2_se3, qup2_se4_01, qup2_se4_23, qup3_se0, + qup3_se1, qup3_se2, qup3_se3, qup3_se4, qup3_se5, qup4_se0, + qup4_se1, qup4_se2, qup4_se3_01, qup4_se3_23, qup4_se3_l3, + qup4_se4_01, qup4_se4_23, qup4_se4_l3, qup5_se0, rng_rosc, + sd_write_protect, sdc2_clk, sdc2_cmd, sdc2_data, sdc2_rclk, + sdc4_clk, sdc4_cmd, sdc4_data, sys_throttle, tb_trig_sdc, + tmess_rng, tsense_clm, tsense_pwm, uim0, uim1, usb0_hs, usb_phy, + vfr, vsense_trigger_mirnat, wcn_sw ] + + required: + - pins + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + #include + + tlmm: pinctrl@f100000 { + compatible = "qcom,maili-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&tlmm 0 0 227>; + interrupt-controller; + #interrupt-cells = <2>; + + qup-uart7-state { + pins = "gpio62", "gpio63"; + function = "qup1_se7"; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index b8109e6c2a10..c27e24cb206f 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -62,6 +62,7 @@ properties: - qcom,pmc8380-gpio - qcom,pmcx0102-gpio - qcom,pmd8028-gpio + - qcom,pmg1110-gpio - qcom,pmh0101-gpio - qcom,pmh0104-gpio - qcom,pmh0110-gpio @@ -155,6 +156,7 @@ allOf: - qcom,pm8450-gpio - qcom,pm8916-gpio - qcom,pmd8028-gpio + - qcom,pmg1110-gpio - qcom,pmk8350-gpio - qcom,pmr735a-gpio - qcom,pmr735b-gpio @@ -496,6 +498,7 @@ $defs: - gpio1-gpio22 for pma8084 - gpio1-gpio14 for pmcx0102 - gpio1-gpio4 for pmd8028 + - gpio1-gpio4 for pmg1110 - gpio1-gpio18 for pmh0101 - gpio1-gpio8 for pmh0104 - gpio1-gpio14 for pmh0110 diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml index 7b006009ca0e..5e35686eeed3 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml @@ -18,7 +18,7 @@ description: | All the pin controller nodes should be represented in the aliases node using the following format 'pinctrl{n}' where n is a unique number for the alias. - The controller supports three types of interrupts:: + The controller supports three types of interrupts: - External GPIO interrupts (see interrupts property in pin controller node); - External wake-up interrupts - multiplexed (capable of waking up the system diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml index 0744a867c79e..2eadffe9ae14 100644 --- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml +++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml @@ -36,6 +36,7 @@ properties: - qcom,msm8996-rpmpd - qcom,msm8998-rpmpd - qcom,nord-rpmhpd + # - qcom,nordau-rpmhpd (driver support only, binding entry not yet added) - qcom,qcm2290-rpmpd - qcom,qcs404-rpmpd - qcom,qcs615-rpmhpd diff --git a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml index 347571e2545a..b67aa170b2c1 100644 --- a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml +++ b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml @@ -13,7 +13,7 @@ maintainers: description: | The R-Car (RZ/G) System Controller provides power management for the CPU cores and various coprocessors. - The power domain IDs for consumers are defined in header files:: + The power domain IDs for consumers are defined in header files: include/dt-bindings/power/r8*-sysc.h properties: diff --git a/Documentation/devicetree/bindings/power/reset/restart-handler.yaml b/Documentation/devicetree/bindings/power/reset/restart-handler.yaml index 965a834a3dbe..00c00ec5ec81 100644 --- a/Documentation/devicetree/bindings/power/reset/restart-handler.yaml +++ b/Documentation/devicetree/bindings/power/reset/restart-handler.yaml @@ -18,12 +18,12 @@ properties: priority: $ref: /schemas/types.yaml#/definitions/uint32 description: | - A priority ranging from 0 to 255 according to the following guidelines:: - 0:: Restart handler of last resort, with limited restart capabilities. - 128:: Typical, default restart handler; use if no other restart handler + A priority ranging from 0 to 255 according to the following guidelines: + 0: Restart handler of last resort, with limited restart capabilities. + 128: Typical, default restart handler; use if no other restart handler is expected to be available, and/or if restart functionality is sufficient to restart the entire system. - 255:: Highest priority restart handler, will preempt all other restart handlers. + 255: Highest priority restart handler, will preempt all other restart handlers. minimum: 0 maximum: 255 diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml index b704f05ea454..b886495c1396 100644 --- a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml +++ b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml @@ -22,13 +22,13 @@ description: | Certain regulators support "regulator-initial-mode" and "regulator-mode". The valid modes list is defined in the dt-bindings/regulator/maxim,max77802.h - and their meaning is:: + and their meaning is: 1 - Normal regulator voltage output mode. 3 - Low Power which reduces the quiescent current down to only 1uA The standard "regulator-mode" property can only be used for regulators that support changing their mode to Low Power Mode during suspend. These - regulators are:: bucks 2-4 and LDOs 1-35. Also, it only takes effect if the + regulators are: bucks 2-4 and LDOs 1-35. Also, it only takes effect if the regulator has been enabled for the given suspend state using "regulator-on-in-suspend" and has not been disabled for that state using "regulator-off-in-suspend". diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml index d93594304651..de1815ed0ab3 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml @@ -68,6 +68,7 @@ description: | properties: compatible: enum: + # - qcom,kai_mv-rpmh-regulators (driver support only, binding entry not yet added) - qcom,pm6150-rpmh-regulators - qcom,pm6150l-rpmh-regulators - qcom,pm6350-rpmh-regulators @@ -99,6 +100,7 @@ properties: - qcom,pmh0110-rpmh-regulators - qcom,pmi8998-rpmh-regulators - qcom,pmm8155au-rpmh-regulators + # - qcom,pmm8650au-rpmh-regulators (driver support only, binding entry not yet added) - qcom,pmm8654au-rpmh-regulators - qcom,pmr735a-rpmh-regulators - qcom,pmr735b-rpmh-regulators diff --git a/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml index 40f9223d4c27..0bbf7c806fbc 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml @@ -13,9 +13,6 @@ description: The REFGEN (reference voltage generator) regulator provides reference voltage for on-chip IPs (like PHYs) on some Qualcomm SoCs. -allOf: - - $ref: regulator.yaml# - properties: compatible: oneOf: @@ -39,16 +36,44 @@ properties: - const: qcom,sm8250-refgen-regulator - enum: + - qcom,ipq9650-refgen-regulator - qcom,sdm845-refgen-regulator - qcom,sm8250-refgen-regulator reg: maxItems: 1 + clocks: + items: + - description: Core reference clock + - description: AHB interface clock + + clock-names: + items: + - const: core + - const: hclk + required: - compatible - reg +allOf: + - $ref: regulator.yaml# + + - if: + properties: + compatible: + contains: + const: qcom,ipq9650-refgen-regulator + then: + required: + - clocks + - clock-names + else: + properties: + clocks: false + clock-names: false + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml b/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml index 022c1f197364..4c3236be8709 100644 --- a/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml +++ b/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml @@ -21,7 +21,7 @@ description: | conduction mode (FCCM). The definition of modes is in the datasheet which is available in below link - and their meaning is:: + and their meaning is: 0 - Auto mode for power saving, which reducing the switching frequency at light load condition to maintain high frequency. 1 - FCCM to meet the strict voltage regulation accuracy, which keeping constant switching frequency. @@ -42,11 +42,14 @@ properties: richtek,mtp-sel-high: type: boolean + deprecated: true description: vout register selection based on this boolean value. false - Using DVS0 register setting to adjust vout true - Using DVS1 register setting to adjust vout + The property is now deprecated. Will be identified by RG HW register. + regulators: type: object additionalProperties: false @@ -100,7 +103,6 @@ examples: compatible = "richtek,rtq2208"; reg = <0x10>; interrupts-extended = <&gpio26 0 IRQ_TYPE_LEVEL_LOW>; - richtek,mtp-sel-high; regulators { buck-a { diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml index cc4ceb32e9d6..68b5e579e2d3 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml @@ -38,7 +38,9 @@ patternProperties: type: object description: Properties for single LDO/BUCK regulator. - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# properties: regulator-name: @@ -51,15 +53,6 @@ patternProperties: description: GPIO used to control ldo4 state (when ldo4 is controlled by GPIO). - rohm,dvs-run-voltage: - description: - PMIC "RUN" state voltage in uV when PMIC HW states are used. See - comments below for bucks/LDOs which support this. 0 means - regulator should be disabled at RUN state. - $ref: /schemas/types.yaml#/definitions/uint32 - minimum: 0 - maximum: 3300000 - rohm,dvs-snvs-voltage: description: Whether to keep regulator enabled at "SNVS" state or not. @@ -71,21 +64,15 @@ patternProperties: minimum: 0 maximum: 3300000 + rohm,dvs-run-voltage: + minimum: 0 + maximum: 3300000 + rohm,dvs-suspend-voltage: - description: - PMIC "SUSPEND" state voltage in uV when PMIC HW states are used. See - comments below for bucks/LDOs which support this. 0 means - regulator should be disabled at SUSPEND state. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-lpsr-voltage: - description: - PMIC "LPSR" state voltage in uV when PMIC HW states are used. See - comments below for bucks/LDOs which support this. 0 means - regulator should be disabled at LPSR state. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml index d898800d6bca..8cba397678a6 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml @@ -40,7 +40,9 @@ patternProperties: type: object description: Properties for single BUCK regulator. - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# properties: regulator-name: @@ -49,34 +51,18 @@ patternProperties: should be "buck1", ..., "buck7" rohm,dvs-run-voltage: - description: - PMIC default "RUN" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-idle-voltage: - description: - PMIC default "IDLE" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-suspend-voltage: - description: - PMIC default "SUSPEND" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-lpsr-voltage: - description: - PMIC default "LPSR" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml index 29b350a4f88a..efb38dd15145 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml @@ -44,7 +44,9 @@ patternProperties: "^BUCK[1-8]$": type: object - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# description: Properties for single BUCK regulator. @@ -55,28 +57,16 @@ patternProperties: should be "buck1", ..., "buck8" rohm,dvs-run-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "RUN" state voltage in uV. See below table for - bucks which support this. 0 means disabled. rohm,dvs-idle-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "IDLE" state voltage in uV. See below table for - bucks which support this. 0 means disabled. rohm,dvs-suspend-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "SUSPEND" state voltage in uV. See below table for - bucks which support this. 0 means disabled. # Supported default DVS states: # diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml index 7ba4ccf723d8..ef6f69be24ae 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml @@ -43,7 +43,9 @@ patternProperties: "^BUCK[1-6]$": type: object - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# description: Properties for single BUCK regulator. @@ -54,28 +56,16 @@ patternProperties: should be "buck1", ..., "buck6" rohm,dvs-run-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "RUN" state voltage in uV. See below table for - bucks which support this. 0 means disabled. rohm,dvs-idle-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "IDLE" state voltage in uV. See below table for - bucks which support this. 0 means disabled. rohm,dvs-suspend-voltage: - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 1300000 - description: - PMIC default "SUSPEND" state voltage in uV. See below table for - bucks which support this. 0 means disabled. # Supported default DVS states: # diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml index 5518082129bd..76747bf2bbdd 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml @@ -26,41 +26,27 @@ patternProperties: type: object description: Properties for single LDO regulator. - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# properties: regulator-name: pattern: "^ldo([1-9]|1[0-1])$" rohm,dvs-run-voltage: - description: - PMIC default "RUN" state voltage in uV. See below table for - LDOs which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-idle-voltage: - description: - PMIC default "IDLE" state voltage in uV. See below table for - LDOs which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-suspend-voltage: - description: - PMIC default "SUSPEND" state voltage in uV. See below table for - LDOs which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-lpsr-voltage: - description: - PMIC default "deep-idle" state voltage in uV. See below table for - LDOs which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 @@ -82,7 +68,9 @@ patternProperties: type: object description: Properties for single BUCK regulator. - $ref: regulator.yaml# + allOf: + - $ref: regulator.yaml# + - $ref: rohm,pmic-states.yaml# properties: regulator-name: @@ -97,34 +85,18 @@ patternProperties: maximum: 300000 rohm,dvs-run-voltage: - description: - PMIC default "RUN" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-idle-voltage: - description: - PMIC default "IDLE" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-suspend-voltage: - description: - PMIC default "SUSPEND" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 rohm,dvs-lpsr-voltage: - description: - PMIC default "deep-idle" state voltage in uV. See below table for - bucks which support this. 0 means disabled. - $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3300000 diff --git a/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml b/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml new file mode 100644 index 000000000000..f09f021bdde4 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/rohm,pmic-states.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ROHM PMICs' hardware state machine descriptions + +maintainers: + - Matti Vaittinen + +description: + Many of the ROHM PMICs have internal state-machine designed to provide + correct voltages for different SOC states. In many cases the states and + transitions are SOC specific, but ones described here can be found from a + few of the different ROHM PMICs. + +properties: + rohm,dvs-run-voltage: + description: + PMIC "RUN" state voltage in uV when PMIC HW states are used. 0 means the + regulator should be disabled at RUN state. Some regulators do not support + setting HW state specific voltage but do support enable/disable control. + For them any positive value means the regulator should be enabled without + touching the voltage. + $ref: /schemas/types.yaml#/definitions/uint32 + + rohm,dvs-suspend-voltage: + description: + PMIC "SUSPEND" state voltage in uV when PMIC HW states are used. 0 means + the regulator should be disabled at SUSPEND state. Some regulators do not + support setting HW state specific voltage but do support enable/disable + control. For them any positive value means the regulator should be + enabled without touching the voltage. + $ref: /schemas/types.yaml#/definitions/uint32 + + rohm,dvs-idle-voltage: + description: + PMIC "IDLE" state voltage in uV. 0 means the regulator should be disabled + at IDLE state. Some regulators do not support setting HW state specific + voltage but do support enable/disable control. For them any positive + value means the regulator should be enabled without touching the voltage. + $ref: /schemas/types.yaml#/definitions/uint32 + + rohm,dvs-lpsr-voltage: + description: + PMIC "LPSR" state voltage in uV when PMIC HW states are used. 0 means the + regulator should be disabled at LPSR state. Some regulators do not + support setting HW state specific voltage but do support enable/disable + control. For them any positive value means the regulator should be + enabled without touching the voltage. Some PMICs' documentation may refer + to this state as a "deep-idle state". + $ref: /schemas/types.yaml#/definitions/uint32 + +# The schema is expected to be reference by PMIC schemas, which may introduce +# other properties that must be allowed. +additionalProperties: true diff --git a/Documentation/devicetree/bindings/regulator/sprd,sc2730-regulator.yaml b/Documentation/devicetree/bindings/regulator/sprd,sc2730-regulator.yaml new file mode 100644 index 000000000000..ab945c46b08e --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/sprd,sc2730-regulator.yaml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/sprd,sc2730-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Unisoc SC2730 Power Management IC regulators + +maintainers: + - Otto Pflüger + +patternProperties: + "^dcdc-(core|cpu|gen[0-1]|gpu|mem|memq|modem|sram)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + "^ldo-avdd(12|18)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + "^ldo-vdd(18-dcxo|28)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + "^ldo-vdd(emmccore|kpled|ldo[0-2]|sd(core|io)|sim[0-2]|usb33|wcn|wifipa)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + "^ldo-vddcam(a0|a1|d0|d1|io|mot)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + "^ldo-vddrf(1v25|18)$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + +additionalProperties: false +... diff --git a/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml b/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml index ce8ec0119469..c18f71b64889 100644 --- a/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml +++ b/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml @@ -28,6 +28,9 @@ properties: - fsl,imx8qxp-cm4 - fsl,imx8ulp-cm33 - fsl,imx93-cm33 + - fsl,imx94-cm33s + - fsl,imx94-cm70 + - fsl,imx94-cm71 - fsl,imx95-cm7 clocks: diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml index 16a245fe2738..7e8ecae8e6cb 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml @@ -32,6 +32,14 @@ properties: reg: maxItems: 1 + clocks: + items: + - description: XO clock + + clock-names: + items: + - const: xo + cx-supply: true px-supply: @@ -49,6 +57,30 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + + interrupt-names: + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - memory-region @@ -57,56 +89,6 @@ unevaluatedProperties: false allOf: - $ref: /schemas/remoteproc/qcom,pas-common.yaml# - - if: - properties: - compatible: - contains: - enum: - - qcom,msm8226-adsp-pil - - qcom,msm8953-adsp-pil - - qcom,msm8974-adsp-pil - - qcom,msm8996-adsp-pil - - qcom,msm8996-slpi-pil - - qcom,msm8998-adsp-pas - - qcom,msm8998-slpi-pas - - qcom,sdm660-adsp-pas - - qcom,sdm660-cdsp-pas - - qcom,sdm845-adsp-pas - - qcom,sdm845-cdsp-pas - - qcom,sdm845-slpi-pas - then: - properties: - clocks: - items: - - description: XO clock - clock-names: - items: - - const: xo - - - if: - properties: - compatible: - contains: - enum: - - qcom,msm8226-adsp-pil - - qcom,msm8953-adsp-pil - - qcom,msm8974-adsp-pil - - qcom,msm8996-adsp-pil - - qcom,msm8996-slpi-pil - - qcom,msm8998-adsp-pas - - qcom,msm8998-slpi-pas - - qcom,sdm660-adsp-pas - - qcom,sdm660-cdsp-pas - - qcom,sdm845-adsp-pas - - qcom,sdm845-cdsp-pas - - qcom,sdm845-slpi-pas - then: - properties: - interrupts: - maxItems: 5 - interrupt-names: - maxItems: 5 - - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml new file mode 100644 index 000000000000..8fd6913e414d --- /dev/null +++ b/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/remoteproc/qcom,kaanapali-soccp-pas.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Kaanapali SoCCP Peripheral Authentication Service + +maintainers: + - Jingyi Wang + +description: + The SoC Control Processor (SoCCP) is a small RISC-V MCU that controls USB + Type-C, battery charging and various other functions on Qualcomm SoCs, somewhat + analogous to traditional PC Embedded Controllers. This document describes + the Peripheral Authentication Service that loads and boots firmware for SoCCP. + +properties: + compatible: + oneOf: + - items: + - enum: + - qcom,glymur-soccp-pas + - qcom,hawi-soccp-pas + - qcom,maili-soccp-pas + - const: qcom,kaanapali-soccp-pas + - enum: + - qcom,kaanapali-soccp-pas + + reg: + maxItems: 1 + + clocks: + items: + - description: XO clock + + clock-names: + items: + - const: xo + + power-domains: + items: + - description: CX power domain + - description: MX power domain + + power-domain-names: + items: + - const: cx + - const: mx + + firmware-name: + items: + - description: Firmware name of the SoC Control Processor + - description: Firmware name of the SoCCP Devicetree + + memory-region: + items: + - description: Memory region for main Firmware authentication + - description: Memory region for Devicetree Firmware authentication + + interrupts: + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Pong interrupt + + interrupt-names: + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: pong + + qcom,smem-states: + minItems: 2 + description: States used by the AP to signal the SoC Control Processor + + qcom,smem-state-names: + minItems: 2 + description: The names of the state bits used for SMP2P output + +required: + - compatible + - reg + - memory-region + - power-domains + - power-domain-names + +allOf: + - $ref: /schemas/remoteproc/qcom,pas-common.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + #include + #define IPCC_MPROC_SOCCP + + remoteproc@d00000 { + compatible = "qcom,kaanapali-soccp-pas"; + reg = <0x00d00000 0x200000>; + + clocks = <&rpmhcc RPMH_CXO_CLK>; + clock-names = "xo"; + + interrupts-extended = <&intc GIC_SPI 167 IRQ_TYPE_EDGE_RISING>, + <&soccp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, + <&soccp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, + <&soccp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, + <&soccp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>, + <&soccp_smp2p_in 9 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "wdog", + "fatal", + "ready", + "handover", + "stop-ack", + "pong"; + + memory-region = <&soccp_mem>, + <&soccp_dtb_mem_mem>; + + firmware-name = "qcom/kaanapali/soccp.mbn", + "qcom/kaanapali/soccp_dtb.mbn"; + + power-domains = <&rpmhpd RPMHPD_CX>, + <&rpmhpd RPMHPD_MX>; + power-domain-names = "cx", + "mx"; + + qcom,smem-states = <&soccp_smp2p_out 0>, + <&soccp_smp2p_out 8>; + qcom,smem-state-names = "stop", + "ping"; + + glink-edge { + interrupts-extended = <&ipcc IPCC_MPROC_SOCCP + IPCC_MPROC_SIGNAL_GLINK_QMP + IRQ_TYPE_EDGE_RISING>; + mboxes = <&ipcc IPCC_MPROC_SOCCP + IPCC_MPROC_SIGNAL_GLINK_QMP>; + + label = "soccp"; + qcom,remote-pid = <19>; + + /* ... */ + }; + }; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml index e5cce0d05fc6..99d7337e58ec 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml @@ -34,12 +34,22 @@ properties: - const: xo interrupts: - minItems: 6 - maxItems: 6 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt interrupt-names: - minItems: 6 - maxItems: 6 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack qcom,qmp: $ref: /schemas/types.yaml#/definitions/phandle @@ -59,6 +69,14 @@ properties: - description: Memory region for core Firmware authentication - description: Memory region for Devicetree Firmware authentication + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml index 4607b459131b..11faf655f530 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml @@ -26,23 +26,11 @@ properties: interrupts: minItems: 5 - items: - - description: Watchdog interrupt - - description: Fatal interrupt - - description: Ready interrupt - - description: Handover interrupt - - description: Stop acknowledge interrupt - - description: Shutdown acknowledge interrupt + maxItems: 6 interrupt-names: minItems: 5 - items: - - const: wdog - - const: fatal - - const: ready - - const: handover - - const: stop-ack - - const: shutdown-ack + maxItems: 6 iommus: maxItems: 1 @@ -58,8 +46,16 @@ properties: qcom,smem-states: $ref: /schemas/types.yaml#/definitions/phandle-array description: States used by the AP to signal the Hexagon core + minItems: 1 items: - - description: Stop the modem + - description: Stop the remoteproc + items: + - description: Phandle to the Shared Memory Point 2 Point device + handling the communication with a remote processor + - description: Single bit index to toggle in the value sent to + the remote processor + maximum: 32 + - description: Ping the remoteproc items: - description: Phandle to the Shared Memory Point 2 Point device handling the communication with a remote processor @@ -69,8 +65,10 @@ properties: qcom,smem-state-names: description: The names of the state bits used for SMP2P output + minItems: 1 items: - const: stop + - const: ping smd-edge: $ref: /schemas/remoteproc/qcom,smd-edge.yaml# diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml index ad45fd00ae34..bf9bf1af9ff1 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml @@ -32,10 +32,20 @@ properties: - const: xo interrupts: - maxItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt interrupt-names: - maxItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack power-domains: false power-domain-names: false @@ -49,6 +59,14 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,rpm-proc.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,rpm-proc.yaml index 540bdfca53d9..823304afaa98 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,rpm-proc.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,rpm-proc.yaml @@ -87,6 +87,7 @@ properties: - qcom,qcm2290-rpm-proc - qcom,qcs404-rpm-proc - qcom,sdm660-rpm-proc + - qcom,shikra-rpm-proc - qcom,sm6115-rpm-proc - qcom,sm6125-rpm-proc - qcom,sm6375-rpm-proc diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml index bcd2bcf96e24..dda2d144b720 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml @@ -59,10 +59,28 @@ properties: - description: Memory region for main Firmware authentication interrupts: - maxItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt interrupt-names: - maxItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output required: - compatible diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml index 66b455d0a8e3..b20780e5e26b 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml @@ -48,6 +48,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml index 8227527c1d77..4bbe4a986c7c 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml @@ -45,6 +45,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml index 8c4abde74915..8c16b01c53e4 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml @@ -30,10 +30,22 @@ properties: - const: xo interrupts: - minItems: 6 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt interrupt-names: - minItems: 6 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack power-domains: items: @@ -59,6 +71,14 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml new file mode 100644 index 000000000000..34a2f15e9361 --- /dev/null +++ b/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/remoteproc/qcom,shikra-pas.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Shikra SoC Peripheral Authentication Service + +maintainers: + - Bibek Kumar Patro + - Komal Bajaj + +description: + Qualcomm Shikra SoC Peripheral Authentication Service loads and boots + firmware on the Qualcomm DSP Hexagon cores. + +properties: + compatible: + enum: + - qcom,shikra-cdsp-pas + - qcom,shikra-lpaicp-pas + - qcom,shikra-mpss-pas + + reg: + maxItems: 1 + + clocks: + items: + - description: XO clock + + clock-names: + items: + - const: xo + + memory-region: + minItems: 1 + maxItems: 2 + + smd-edge: false + + firmware-name: + minItems: 1 + items: + - description: Firmware name of the Hexagon core + - description: Firmware name of the Hexagon Devicetree + + glink-edge: + $ref: /schemas/remoteproc/qcom,glink-edge.yaml# + description: + Qualcomm G-Link subnode which represents communication edge, channels + and devices related to the remoteproc core. + unevaluatedProperties: false + + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: States used by the AP to signal the Hexagon core + items: + - description: Stop the remote processor + items: + - description: Phandle to the Shared Memory Point 2 Point device + handling the communication with a remote processor + - description: Single bit index to toggle in the value sent to + the remote processor + maximum: 32 + + qcom,smem-state-names: + description: The names of the state bits used for SMP2P output + items: + - const: stop + +required: + - compatible + - reg + - memory-region + +allOf: + - $ref: /schemas/remoteproc/qcom,pas-common.yaml# + + - if: + properties: + compatible: + enum: + - qcom,shikra-cdsp-pas + - qcom,shikra-mpss-pas + then: + properties: + interrupts: + minItems: 6 + interrupt-names: + minItems: 6 + memory-region: + maxItems: 1 + firmware-name: + maxItems: 1 + power-domains: + items: + - description: CX power domain + power-domain-names: + items: + - const: cx + + - if: + properties: + compatible: + enum: + - qcom,shikra-lpaicp-pas + then: + properties: + interrupts: + maxItems: 5 + interrupt-names: + maxItems: 5 + memory-region: + minItems: 2 + firmware-name: + minItems: 2 + power-domains: false + power-domain-names: false + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + #include + #include + #include + + remoteproc@b300000 { + compatible = "qcom,shikra-cdsp-pas"; + reg = <0x0b300000 0x100000>; + + interrupts-extended = <&intc GIC_SPI 265 IRQ_TYPE_EDGE_RISING>, + <&cdsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, + <&cdsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, + <&cdsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, + <&cdsp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>, + <&cdsp_smp2p_in 7 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "wdog", "fatal", "ready", + "handover", "stop-ack", "shutdown-ack"; + + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>; + clock-names = "xo"; + + interconnects = <&mem_noc MASTER_AMPSS_M0 RPM_ALWAYS_TAG + &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>; + + power-domains = <&rpmpd RPMHPD_CX>; + power-domain-names = "cx"; + + memory-region = <&cdsp_mem>; + + qcom,smem-states = <&cdsp_smp2p_out 0>; + qcom,smem-state-names = "stop"; + + firmware-name = "qcom/shikra/cdsp.mbn"; + + glink-edge { + interrupts = ; + mboxes = <&apcs_glb 4>; + qcom,remote-pid = <5>; + label = "cdsp"; + }; + }; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml index eeb6a8aafeb9..454ba82bd6f1 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml @@ -51,6 +51,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml index c1a3cc308bdb..42e02c64347a 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml @@ -45,6 +45,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml index 7286b2baa19f..274f87880e2e 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml @@ -39,8 +39,36 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + smd-edge: false + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml index a8cddf7e2fe1..5a7c5f8c92d1 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml @@ -61,6 +61,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml index 6d09823153fc..72d0db5698c5 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml @@ -55,6 +55,34 @@ properties: maxItems: 1 description: Firmware name for the Hexagon core + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml index 1e4db0c9fcf9..1a8539d6b788 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml @@ -17,6 +17,7 @@ properties: compatible: oneOf: - enum: + - qcom,nord-adsp-pas - qcom,sdx75-mpss-pas - qcom,sm8550-adsp-pas - qcom,sm8550-cdsp-pas @@ -30,12 +31,14 @@ properties: - items: - enum: - qcom,glymur-adsp-pas + - qcom,hawi-adsp-pas - qcom,kaanapali-adsp-pas - qcom,sm8750-adsp-pas - const: qcom,sm8550-adsp-pas - items: - enum: - qcom,glymur-cdsp-pas + - qcom,hawi-cdsp-pas - qcom,kaanapali-cdsp-pas - const: qcom,sm8550-cdsp-pas - items: @@ -73,6 +76,34 @@ properties: - description: DSM Memory region 2 - description: Memory region for Qlink Logging + interrupts: + minItems: 5 + items: + - description: Watchdog interrupt + - description: Fatal interrupt + - description: Ready interrupt + - description: Handover interrupt + - description: Stop acknowledge interrupt + - description: Shutdown acknowledge interrupt + + interrupt-names: + minItems: 5 + items: + - const: wdog + - const: fatal + - const: ready + - const: handover + - const: stop-ack + - const: shutdown-ack + + qcom,smem-states: + maxItems: 1 + description: States used by the AP to signal the Hexagon core + + qcom,smem-state-names: + maxItems: 1 + description: The names of the state bits used for SMP2P output + required: - compatible - reg @@ -104,8 +135,11 @@ allOf: enum: - qcom,glymur-adsp-pas - qcom,glymur-cdsp-pas + - qcom,hawi-adsp-pas + - qcom,hawi-cdsp-pas - qcom,kaanapali-adsp-pas - qcom,kaanapali-cdsp-pas + - qcom,nord-adsp-pas - qcom,sm8750-adsp-pas then: properties: @@ -210,6 +244,23 @@ allOf: - const: lcx - const: lmx + - if: + properties: + compatible: + contains: + enum: + - qcom,nord-adsp-pas + then: + properties: + power-domains: + items: + - description: CX power domain + - description: MX power domain + power-domain-names: + items: + - const: cx + - const: mx + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml b/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml index c7d5e58330d6..5ab19d70a344 100644 --- a/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml +++ b/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml @@ -128,6 +128,10 @@ patternProperties: - description: vring1 additionalItems: true + firmware-name: + maxItems: 1 + description: default firmware to load + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/reserved-memory/intel,wakeup-mailbox.yaml b/Documentation/devicetree/bindings/reserved-memory/intel,wakeup-mailbox.yaml new file mode 100644 index 000000000000..4362bc058df4 --- /dev/null +++ b/Documentation/devicetree/bindings/reserved-memory/intel,wakeup-mailbox.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/reserved-memory/intel,wakeup-mailbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Wakeup Mailbox for Intel processors + +description: | + The Wakeup Mailbox provides a mechanism for the operating system to wake up + secondary CPUs on Intel processors. It is an alternative to the INIT-!INIT- + SIPI sequence used on most x86 systems. + + The structure and operation of the mailbox is described in the Multiprocessor + Wakeup Structure of the ACPI specification version 6.6 section 5.2.12.19 [1]. + + The implementation of the mailbox in platform firmware is described in the + Intel TDX Virtual Firmware Design Guide section 4.3.5 [2]. + + 1: https://uefi.org/specs/ACPI/6.6/05_ACPI_Software_Programming_Model.html#multiprocessor-wakeup-structure + 2: https://www.intel.com/content/www/us/en/content-details/733585/intel-tdx-virtual-firmware-design-guide.html + +maintainers: + - Ricardo Neri + +allOf: + - $ref: reserved-memory.yaml + +properties: + compatible: + const: intel,wakeup-mailbox + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + reserved-memory { + #address-cells = <2>; + #size-cells = <1>; + + wakeup-mailbox@ffff0000 { + compatible = "intel,wakeup-mailbox"; + reg = <0x0 0xffff0000 0x1000>; + }; + }; diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml index 41fd11f70a49..f9854ff43ac6 100644 --- a/Documentation/devicetree/bindings/riscv/canaan.yaml +++ b/Documentation/devicetree/bindings/riscv/canaan.yaml @@ -10,7 +10,7 @@ maintainers: - Damien Le Moal description: - Canaan Kendryte K210 SoC-based boards + Canaan Kendryte SoC-based boards properties: $nodename: @@ -42,6 +42,12 @@ properties: - items: - const: canaan,kendryte-k210 + - items: + - enum: + - canaan,canmv-k230 + - canaan,k230-usip-lp3-evb + - const: canaan,kendryte-k230 + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/rtc/epson,rx6110.txt b/Documentation/devicetree/bindings/rtc/epson,rx6110.txt deleted file mode 100644 index 3dc313e01f77..000000000000 --- a/Documentation/devicetree/bindings/rtc/epson,rx6110.txt +++ /dev/null @@ -1,39 +0,0 @@ -Epson RX6110 Real Time Clock -============================ - -The Epson RX6110 can be used with SPI or I2C busses. The kind of -bus depends on the SPISEL pin and can not be configured via software. - -I2C mode --------- - -Required properties: - - compatible: should be: "epson,rx6110" - - reg : the I2C address of the device for I2C - -Example: - - rtc: rtc@32 { - compatible = "epson,rx6110" - reg = <0x32>; - }; - -SPI mode --------- - -Required properties: - - compatible: should be: "epson,rx6110" - - reg: chip select number - - spi-cs-high: RX6110 needs chipselect high - - spi-cpha: RX6110 works with SPI shifted clock phase - - spi-cpol: RX6110 works with SPI inverse clock polarity - -Example: - - rtc: rtc@3 { - compatible = "epson,rx6110" - reg = <3> - spi-cs-high; - spi-cpha; - spi-cpol; - }; diff --git a/Documentation/devicetree/bindings/rtc/epson,rx6110.yaml b/Documentation/devicetree/bindings/rtc/epson,rx6110.yaml new file mode 100644 index 000000000000..55086ac7d1e2 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/epson,rx6110.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/epson,rx6110.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Epson RX6110 Real Time Clock + +description: + The Epson RX6110 can be used with SPI or I2C busses. The kind of bus depends + on the SPISEL pin and cannot be configured via software. + +maintainers: + - Alexandre Belloni + +allOf: + - $ref: rtc.yaml# + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + const: epson,rx6110 + + reg: + maxItems: 1 + + spi-cs-high: true + spi-cpha: true + spi-cpol: true + +required: + - compatible + - reg + +dependencies: + spi-cs-high: [ spi-cpha, spi-cpol ] + spi-cpha: [ spi-cs-high, spi-cpol ] + spi-cpol: [ spi-cs-high, spi-cpha ] + +unevaluatedProperties: false + +examples: + # I2C mode + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + rtc@32 { + compatible = "epson,rx6110"; + reg = <0x32>; + }; + }; + + # SPI mode + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + + rtc@3 { + compatible = "epson,rx6110"; + reg = <3>; + spi-cs-high; + spi-cpha; + spi-cpol; + }; + }; diff --git a/Documentation/devicetree/bindings/rtc/rtc-ds1307.yaml b/Documentation/devicetree/bindings/rtc/rtc-ds1307.yaml index 98d10e680144..9b2796804f07 100644 --- a/Documentation/devicetree/bindings/rtc/rtc-ds1307.yaml +++ b/Documentation/devicetree/bindings/rtc/rtc-ds1307.yaml @@ -31,6 +31,7 @@ properties: - epson,rx8025 - isil,isl12057 - epson,rx8130 + - epson,rx8901 - items: - enum: diff --git a/Documentation/devicetree/bindings/rtc/st,m41t93.yaml b/Documentation/devicetree/bindings/rtc/st,m41t93.yaml new file mode 100644 index 000000000000..bdd995c5c1f4 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/st,m41t93.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/st,m41t93.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ST M41T93 RTC and compatible + +maintainers: + - Akhilesh Patil + +description: + ST M41T93 is spi based Real Time Clock (RTC) with time, date, + alarm, watchdog, square wave clock output, 8 bit timer and + 7 bytes of user SRAM. + +properties: + compatible: + enum: + - st,m41t93 + + reg: + maxItems: 1 + + "#clock-cells": + const: 0 + +required: + - compatible + - reg + +allOf: + - $ref: rtc.yaml + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + +examples: + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + rtc@0 { + compatible = "st,m41t93"; + reg = <0>; + #clock-cells = <0>; + spi-max-frequency = <2000000>; + }; + }; + diff --git a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml index 722176c831aa..f4d0eed98a08 100644 --- a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml @@ -30,6 +30,8 @@ properties: - aspeed,ast2500-rtc # ASPEED BMC ast2600 Real-time Clock - aspeed,ast2600-rtc + # ASPEED BMC ast2700 Real-time Clock + - aspeed,ast2700-rtc # Conexant Digicolor Real Time Clock Controller - cnxt,cx92755-rtc # I2C, 32-Bit Binary Counter Watchdog RTC with Trickle Charger and Reset Input/Output diff --git a/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml b/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml index 788ef5c1c446..bc967ead2350 100644 --- a/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml +++ b/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml @@ -17,7 +17,7 @@ description: | software perspective it's mostly compatible with the MSM serial UART except that it supports reading and writing multiple characters at a time. - Note:: Aliases may be defined to ensure the correct ordering of the UARTs. + Note: Aliases may be defined to ensure the correct ordering of the UARTs. The alias serialN will result in the UART being assigned port N. If any serialN alias exists, then an alias must exist for each enabled UART. The serialN aliases should be in a .dts file instead of in a .dtsi file. diff --git a/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml b/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml index c8f01923cb25..8496f822dfa5 100644 --- a/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml +++ b/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml @@ -14,9 +14,16 @@ allOf: properties: compatible: - enum: - - qcom,sa8255p-geni-uart - - qcom,sa8255p-geni-debug-uart + oneOf: + - enum: + - qcom,sa8255p-geni-uart + - qcom,sa8255p-geni-debug-uart + - items: + - const: qcom,sa8797p-geni-uart + - const: qcom,sa8255p-geni-uart + - items: + - const: qcom,sa8797p-geni-debug-uart + - const: qcom,sa8255p-geni-debug-uart reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/serial/rs485.txt b/Documentation/devicetree/bindings/serial/rs485.txt deleted file mode 100644 index a7fe93efc4a5..000000000000 --- a/Documentation/devicetree/bindings/serial/rs485.txt +++ /dev/null @@ -1 +0,0 @@ -See rs485.yaml diff --git a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml index 685c1eceb782..49f51b002879 100644 --- a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml +++ b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml @@ -78,6 +78,7 @@ properties: - starfive,jh7100-hsuart - starfive,jh7100-uart - starfive,jh7110-uart + - ultrarisc,dp1000-uart - const: snps,dw-apb-uart - const: snps,dw-apb-uart diff --git a/Documentation/devicetree/bindings/slimbus/slimbus.yaml b/Documentation/devicetree/bindings/slimbus/slimbus.yaml index 5a941610ce4e..3910327c8ded 100644 --- a/Documentation/devicetree/bindings/slimbus/slimbus.yaml +++ b/Documentation/devicetree/bindings/slimbus/slimbus.yaml @@ -29,7 +29,7 @@ patternProperties: description: | Every SLIMbus controller node can contain zero or more child nodes representing slave devices on the bus. Every SLIMbus slave device is - uniquely determined by the enumeration address containing 4 fields:: + uniquely determined by the enumeration address containing 4 fields: Manufacturer ID, Product code, Device index, and Instance value for the device. @@ -48,7 +48,7 @@ patternProperties: reg: maxItems: 1 description: | - Pair of (device index, instande ID), where:: + Pair of (device index, instande ID), where: - Device index, which uniquely identifies multiple devices within a single component. - Instance ID, can be used for the cases where multiple devices of diff --git a/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml b/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml index 39d4637c2d08..b1200e6940ac 100644 --- a/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml +++ b/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml @@ -24,6 +24,8 @@ properties: - amlogic,meson-sm1-clk-measure - amlogic,c3-clk-measure - amlogic,s4-clk-measure + - amlogic,a1-clk-measure + - amlogic,t7-clk-measure reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml index d828c2e82965..124f5c206ee3 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml @@ -26,6 +26,12 @@ properties: reg: maxItems: 1 + '#address-cells': + const: 1 + + '#size-cells': + const: 1 + '#power-domain-cells': const: 1 @@ -92,6 +98,11 @@ properties: - compatible - ports + bridge@20: + type: object + $ref: /schemas/display/bridge/fsl,ldb.yaml# + unevaluatedProperties: false + allOf: - if: properties: @@ -112,6 +123,7 @@ allOf: - const: lcdif - const: isi - const: csi + bridge@20: false - if: properties: compatible: @@ -163,6 +175,8 @@ examples: <&clk IMX93_CLK_MIPI_DSI_GATE>; clock-names = "apb", "axi", "nic", "disp", "cam", "pxp", "lcdif", "isi", "csi", "dsi"; + #address-cells = <1>; + #size-cells = <1>; #power-domain-cells = <1>; dpi-bridge { @@ -190,4 +204,29 @@ examples: }; }; }; + + bridge@20 { + compatible = "fsl,imx93-ldb"; + reg = <0x20 0x4>, <0x24 0x4>; + reg-names = "ldb", "lvds"; + clocks = <&clk IMX93_CLK_LVDS_GATE>; + clock-names = "ldb"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + endpoint { + remote-endpoint = <&lcdif_to_ldb>; + }; + }; + + port@1 { + reg = <1>; + }; + }; + }; }; diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml index 8eaa04431d74..e778fa3a6c92 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml @@ -30,6 +30,7 @@ properties: - qcom,hawi-aoss-qmp - qcom,kaanapali-aoss-qmp - qcom,milos-aoss-qmp + - qcom,nord-aoss-qmp - qcom,qcs615-aoss-qmp - qcom,qcs8300-aoss-qmp - qcom,qdu1000-aoss-qmp diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml index bdf482db32aa..b663be3ea5a1 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml @@ -40,7 +40,7 @@ properties: $ref: /schemas/types.yaml#/definitions/string-array description: | Protection domain service name and path for APR service (if supported). - Possible values are:: + Possible values are: "avs/audio", "msm/adsp/audio_pd". "kernel/elf_loader", "msm/modem/wlan_pd". "tms/servreg", "msm/adsp/audio_pd". diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml index 26d9bc773ec5..1889139a3f7a 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml @@ -23,7 +23,7 @@ description: | with a few variations that are captured by the properties here. A TCS may be triggered from Linux or triggered by the F/W after all the CPUs - have powered off to facilitate idle power saving. TCS could be classified as:: + have powered off to facilitate idle power saving. TCS could be classified as: ACTIVE - Triggered by Linux SLEEP - Triggered by F/W WAKE - Triggered by F/W @@ -76,7 +76,7 @@ properties: items: items: - description: | - TCS type:: + TCS type: - ACTIVE_TCS - SLEEP_TCS - WAKE_TCS @@ -152,7 +152,7 @@ examples: - | // For a TCS whose RSC base address is 0x179C0000 and is at a DRV id of // 2, the register offsets for DRV2 start at 0D00, the register - // calculations are like this:: + // calculations are like this: // DRV0: 0x179C0000 // DRV2: 0x179C0000 + 0x10000 = 0x179D0000 // DRV2: 0x179C0000 + 0x10000 * 2 = 0x179E0000 @@ -182,7 +182,7 @@ examples: - | // For a TCS whose RSC base address is 0xAF20000 and is at DRV id of 0, the // register offsets for DRV0 start at 01C00, the register calculations are - // like this:: + // like this: // DRV0: 0xAF20000 // TCS-OFFSET: 0x1C00 #include diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml index 270bcd079f88..bd1d32898461 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml @@ -57,6 +57,7 @@ properties: - qcom,rpm-qcm2290 - qcom,rpm-qcs404 - qcom,rpm-sdm660 + - qcom,rpm-shikra - qcom,rpm-sm6115 - qcom,rpm-sm6125 - qcom,rpm-sm6375 diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml index 4fcae6bedfff..72a7f8cb09ba 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml @@ -28,7 +28,7 @@ properties: $ref: /schemas/types.yaml#/definitions/phandle description: | Reference to a node specifying the wcnss "ccu" and "dxe" register blocks. - The node must be compatible with one of the following:: + The node must be compatible with one of the following: - qcom,riva" - qcom,pronto" diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml index 5ddd31f30f26..57c9d3c57021 100644 --- a/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml +++ b/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml @@ -12,10 +12,10 @@ maintainers: description: | Guidelines for new compatibles for SoC blocks/components. - When adding new compatibles in new bindings, use the format:: + When adding new compatibles in new bindings, use the format: renesas,SoC-IP - For example:: + For example: renesas,r8a77965-csi2 When adding new compatibles to existing bindings, use the format in the diff --git a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml index 9c63dbcd4d77..fcdb9ffd2644 100644 --- a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml +++ b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml @@ -37,6 +37,7 @@ properties: - samsung,exynos8895-peric1-sysreg - samsung,exynos990-peric0-sysreg - samsung,exynos990-peric1-sysreg + - samsung,exynosautov920-hsi0-sysreg - samsung,exynosautov920-hsi2-sysreg - samsung,exynosautov920-peric0-sysreg - samsung,exynosautov920-peric1-sysreg diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml new file mode 100644 index 000000000000..2a142b32acf5 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/cirrus,cs35l36.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Cirrus Logic CS35L36 Speaker Amplifier + +maintainers: + - David Rhodes + - patches@opensource.cirrus.com + +description: + CS35L36 is a boosted mono Class D amplifier + +allOf: + - $ref: dai-common.yaml# + +properties: + compatible: + enum: + - cirrus,cs35l36 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + VA-supply: + description: Voltage regulator of analog internal section + + VP-supply: + description: Voltage regulator of boost converter + + reset-gpios: + maxItems: 1 + + cirrus,boost-ctl-millivolt: + description: Boost converter output voltage (step 50) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 2550 + maximum: 12000 + + cirrus,boost-peak-milliamp: + description: Boost-converter peak current limit (step 50) + $ref: /schemas/types.yaml#/definitions/uint32 + default: 4500 + minimum: 1600 + maximum: 4500 + + cirrus,boost-ind-nanohenry: + description: Initial inductor estimation reference value (1000=1μH, 1200=1.2μH) + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1000 + + cirrus,multi-amp-mode: + description: Hi-Z ASP port when more than one amplifier in system + type: boolean + + cirrus,boost-ctl-select: + description: Boost converter control source selection + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1 + enum: + - 0 # Control Port + - 1 # Class + - 2 # Sync + + cirrus,amp-pcm-inv: + description: Invert incoming PCM data when true + type: boolean + + cirrus,imon-pol-inv: + description: Invert polarity of outbound IMON feedback when true + type: boolean + + cirrus,vmon-pol-inv: + description: Invert polarity of outbound VMON feedback when true + type: boolean + + cirrus,dcm-mode-enable: + description: Enable boost converter automatic Discontinuous Conduction Mode + type: boolean + + cirrus,weak-fet-disable: + description: Reduce output driver strength in Weak-FET Drive Mode when true + type: boolean + + cirrus,classh-wk-fet-delay-ms: + description: Weak-FET entry delay + default: 100 + enum: [0, 5, 10, 50, 100, 200, 500, 1000] + + cirrus,classh-weak-fet-thld-millivolt: + description: Weak-FET drive threshold + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700] + + cirrus,temp-warn-threshold: + description: Overtemperature warning threshold + $ref: /schemas/types.yaml#/definitions/uint32 + default: 2 + enum: + - 0 # 105°C + - 1 # 115°C + - 2 # 125°C + - 3 # 135°C + + cirrus,irq-drive-select: + description: Interrupt output driver type + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1 + enum: + - 0 # open-drain + - 1 # push-pull + + cirrus,irq-gpio-select: + description: Programmable IRQ pin selection + $ref: /schemas/types.yaml#/definitions/uint32 + enum: + - 0 # PDM_DATA/SWIRE_SD/INT + - 1 # GPIO + + cirrus,vpbr-config: + $ref: "#/$defs/vpbr-config" + +$defs: + vpbr-config: + description: Brownout prevention configuration sub-node + type: object + additionalProperties: false + + properties: + cirrus,vpbr-en: + description: VBST brownout prevention enable + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0 + enum: + - 0 # disabled + - 1 # enabled + + cirrus,vpbr-thld: + description: Initial VPBR threshold voltage + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 2 + maximum: 31 + + cirrus,vpbr-atk-rate: + description: Attenuation attack step rate + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 7 + + cirrus,vpbr-atk-vol: + description: VP brownout prevention step size + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 7 + + cirrus,vpbr-max-attn: + description: Maximum attenuation during VP brownout prevention (dB) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 15 + + cirrus,vpbr-wait: + description: Delay between brownout clearance and attenuation release (ms) + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1 + enum: + - 0 # 10 + - 1 # 100 + - 2 # 250 + - 3 # 500 + + cirrus,vpbr-rel-rate: + description: Attenuation release step rate + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 7 + + cirrus,vpbr-mute-en: + description: Mute audio if maximum attenuation reached + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 1 + +required: + - compatible + - reg + - interrupts + - VA-supply + - cirrus,boost-ctl-millivolt + - cirrus,boost-peak-milliamp + +unevaluatedProperties: false + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + codec@40 { + compatible = "cirrus,cs35l36"; + reg = <0x40>; + VA-supply = <&dummy_vreg>; + VP-supply = <&dummy_vreg>; + reset-gpios = <&gpio0 54 GPIO_ACTIVE_HIGH>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + + cirrus,boost-ind-nanohenry = <1000>; + cirrus,boost-ctl-millivolt = <10000>; + cirrus,boost-peak-milliamp = <4500>; + cirrus,boost-ctl-select = <0>; + cirrus,classh-wk-fet-delay-ms = <100>; + cirrus,classh-weak-fet-thld-millivolt = <100>; + cirrus,temp-warn-threshold = <1>; + cirrus,multi-amp-mode; + cirrus,irq-drive-select = <1>; + cirrus,irq-gpio-select = <1>; + + cirrus,vpbr-config { + cirrus,vpbr-en = <0>; + cirrus,vpbr-thld = <5>; + cirrus,vpbr-atk-rate = <2>; + cirrus,vpbr-atk-vol = <1>; + cirrus,vpbr-max-attn = <9>; + cirrus,vpbr-wait = <1>; + cirrus,vpbr-rel-rate = <5>; + cirrus,vpbr-mute-en = <0>; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/sound/cs35l36.txt b/Documentation/devicetree/bindings/sound/cs35l36.txt deleted file mode 100644 index d34117b8558e..000000000000 --- a/Documentation/devicetree/bindings/sound/cs35l36.txt +++ /dev/null @@ -1,168 +0,0 @@ -CS35L36 Speaker Amplifier - -Required properties: - - - compatible : "cirrus,cs35l36" - - - reg : the I2C address of the device for I2C - - - VA-supply, VP-supply : power supplies for the device, - as covered in - Documentation/devicetree/bindings/regulator/regulator.txt. - - - cirrus,boost-ctl-millivolt : Boost Voltage Value. Configures the boost - converter's output voltage in mV. The range is from 2550mV to 12000mV with - increments of 50mV. - (Default) VP - - - cirrus,boost-peak-milliamp : Boost-converter peak current limit in mA. - Configures the peak current by monitoring the current through the boost FET. - Range starts at 1600mA and goes to a maximum of 4500mA with increments of - 50mA. - (Default) 4.50 Amps - - - cirrus,boost-ind-nanohenry : Inductor estimation LBST reference value. - Seeds the digital boost converter's inductor estimation block with the initial - inductance value to reference. - - 1000 = 1uH (Default) - 1200 = 1.2uH - -Optional properties: - - cirrus,multi-amp-mode : Boolean to determine if there are more than - one amplifier in the system. If more than one it is best to Hi-Z the ASP - port to prevent bus contention on the output signal - - - cirrus,boost-ctl-select : Boost converter control source selection. - Selects the source of the BST_CTL target VBST voltage for the boost - converter to generate. - 0x00 - Control Port Value - 0x01 - Class H Tracking (Default) - 0x10 - MultiDevice Sync Value - - - cirrus,amp-pcm-inv : Boolean to determine Amplifier will invert incoming - PCM data - - - cirrus,imon-pol-inv : Boolean to determine Amplifier will invert the - polarity of outbound IMON feedback data - - - cirrus,vmon-pol-inv : Boolean to determine Amplifier will invert the - polarity of outbound VMON feedback data - - - cirrus,dcm-mode-enable : Boost converter automatic DCM Mode enable. - This enables the digital boost converter to operate in a low power - (Discontinuous Conduction) mode during low loading conditions. - - - cirrus,weak-fet-disable : Boolean : The strength of the output drivers is - reduced when operating in a Weak-FET Drive Mode and must not be used to drive - a large load. - - - cirrus,classh-wk-fet-delay : Weak-FET entry delay. Controls the delay - (in ms) before the Class H algorithm switches to the weak-FET voltage - (after the audio falls and remains below the value specified in WKFET_AMP_THLD). - - 0 = 0ms - 1 = 5ms - 2 = 10ms - 3 = 50ms - 4 = 100ms (Default) - 5 = 200ms - 6 = 500ms - 7 = 1000ms - - - cirrus,classh-weak-fet-thld-millivolt : Weak-FET amplifier drive threshold. - Configures the signal threshold at which the PWM output stage enters - weak-FET operation. The range is 50mV to 700mV in 50mV increments. - - - cirrus,temp-warn-threshold : Amplifier overtemperature warning threshold. - Configures the threshold at which the overtemperature warning condition occurs. - When the threshold is met, the overtemperature warning attenuation is applied - and the TEMP_WARN_EINT interrupt status bit is set. - If TEMP_WARN_MASK = 0, INTb is asserted. - - 0 = 105C - 1 = 115C - 2 = 125C (Default) - 3 = 135C - - - cirrus,irq-drive-select : Selects the driver type of the selected interrupt - output. - - 0 = Open-drain - 1 = Push-pull (Default) - - - cirrus,irq-gpio-select : Selects the pin to serve as the programmable - interrupt output. - - 0 = PDM_DATA / SWIRE_SD / INT (Default) - 1 = GPIO - -Optional properties for the "cirrus,vpbr-config" Sub-node - - - cirrus,vpbr-en : VBST brownout prevention enable. Configures whether the - VBST brownout prevention algorithm is enabled or disabled. - - 0 = VBST brownout prevention disabled (default) - 1 = VBST brownout prevention enabled - - See Section 7.31.1 VPBR Config for configuration options & further details - - - cirrus,vpbr-thld : Initial VPBR threshold. Configures the VP brownout - threshold voltage - - - cirrus,cirrus,vpbr-atk-rate : Attenuation attack step rate. Configures the - amount delay between consecutive volume attenuation steps when a brownout - condition is present and the VP brownout condition is in an attacking state. - - - cirrus,vpbr-atk-vol : VP brownout prevention step size. Configures the VP - brownout prevention attacking attenuation step size when operating in either - digital volume or analog gain modes. - - - cirrus,vpbr-max-attn : Maximum attenuation that the VP brownout prevention - can apply to the audio signal. - - - cirrus,vpbr-wait : Configures the delay time between a brownout condition - no longer being present and the VP brownout prevention entering an attenuation - release state. - - - cirrus,vpbr-rel-rate : Attenuation release step rate. Configures the delay - between consecutive volume attenuation release steps when a brownout condition - is not longer present and the VP brownout is in an attenuation release state. - - - cirrus,vpbr-mute-en : During the attack state, if the vpbr-max-attn value - is reached, the error condition still remains, and this bit is set, the audio - is muted. - -Example: - -cs35l36: cs35l36@40 { - compatible = "cirrus,cs35l36"; - reg = <0x40>; - VA-supply = <&dummy_vreg>; - VP-supply = <&dummy_vreg>; - reset-gpios = <&gpio0 54 0>; - interrupt-parent = <&gpio8>; - interrupts = <3 IRQ_TYPE_LEVEL_LOW>; - - cirrus,boost-ind-nanohenry = <1000>; - cirrus,boost-ctl-millivolt = <10000>; - cirrus,boost-peak-milliamp = <4500>; - cirrus,boost-ctl-select = <0x00>; - cirrus,weak-fet-delay = <0x04>; - cirrus,weak-fet-thld = <0x01>; - cirrus,temp-warn-threshold = <0x01>; - cirrus,multi-amp-mode; - cirrus,irq-drive-select = <0x01>; - cirrus,irq-gpio-select = <0x01>; - - cirrus,vpbr-config { - cirrus,vpbr-en = <0x00>; - cirrus,vpbr-thld = <0x05>; - cirrus,vpbr-atk-rate = <0x02>; - cirrus,vpbr-atk-vol = <0x01>; - cirrus,vpbr-max-attn = <0x09>; - cirrus,vpbr-wait = <0x01>; - cirrus,vpbr-rel-rate = <0x05>; - cirrus,vpbr-mute-en = <0x00>; - }; -}; diff --git a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml index 61e8babed402..dc7f4afbb777 100644 --- a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml +++ b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml @@ -8,19 +8,22 @@ title: Loongson 7axxx/2kxxx ASoC audio sound card driver maintainers: - Yingkun Meng + - Binbin Zhou description: The binding describes the sound card present in loongson 7axxx/2kxxx platform. The sound card is an ASoC component which uses Loongson I2S controller to transfer the audio data. +allOf: + - $ref: sound-card-common.yaml# + properties: compatible: - const: loongson,ls-audio-card - - model: - $ref: /schemas/types.yaml#/definitions/string - description: User specified audio sound card name + enum: + - loongson,ls-audio-card # Loongson-2K1000/Loongson-2K2000/LS7A + - loongson,ls2k0300-forever-pi-audio-card # CTCISZ Forever Pi + - loongson,ls2k0300-dl2k0300b-audio-card # ATK-DL2K0300B mclk-fs: $ref: simple-card.yaml#/definitions/mclk-fs @@ -45,14 +48,25 @@ properties: required: - sound-dai + spkr-en-gpios: + maxItems: 1 + description: The GPIO that enables the speakers + + hp-ctl-gpios: + maxItems: 1 + description: The GPIO that control the headphones + + hp-det-gpios: + maxItems: 1 + description: The GPIO that detect headphones are plugged in + required: - compatible - - model - mclk-fs - cpu - codec -additionalProperties: false +unevaluatedProperties: false examples: - | @@ -68,3 +82,28 @@ examples: sound-dai = <&es8323>; }; }; + + - | + #include + + sound { + compatible = "loongson,ls2k0300-dl2k0300b-audio-card"; + model = "loongson-audio"; + mclk-fs = <512>; + hp-det-gpios = <&gpio 81 GPIO_ACTIVE_HIGH>; + spkr-en-gpios = <&gpio 86 GPIO_ACTIVE_HIGH>; + hp-ctl-gpios = <&gpio 87 GPIO_ACTIVE_HIGH>; + audio-routing = + "Headphone", "LOUT1", + "Headphone", "ROUT1", + "Speaker", "LOUT2", + "Speaker", "ROUT2"; + + cpu { + sound-dai = <&i2s>; + }; + + codec { + sound-dai = <&es8388>; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml b/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml index da79510bb2d9..51e23c189f7a 100644 --- a/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml +++ b/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml @@ -14,9 +14,12 @@ allOf: properties: compatible: - const: loongson,ls2k1000-i2s + enum: + - loongson,ls2k0300-i2s + - loongson,ls2k1000-i2s reg: + minItems: 1 items: - description: Loongson I2S controller Registers. - description: APB DMA config register for Loongson I2S controller. @@ -49,6 +52,23 @@ required: unevaluatedProperties: false +if: + properties: + compatible: + contains: + enum: + - loongson,ls2k1000-i2s + +then: + properties: + reg: + minItems: 2 + +else: + properties: + reg: + maxItems: 1 + examples: - | #include diff --git a/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml b/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml new file mode 100644 index 000000000000..1b7451655476 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/mediatek,mtk-btcvsd-snd.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mediatek ALSA BT SCO CVSD/MSBC + +maintainers: + - Luca Leonardo Scorcia + +properties: + compatible: + const: mediatek,mtk-btcvsd-snd + + reg: + items: + - description: PKV region + - description: SRAM_BANK2 region + + interrupts: + items: + - description: BT-SCO interrupt + + mediatek,infracfg: + $ref: /schemas/types.yaml#/definitions/phandle + description: The phandle of the infracfg controller + + mediatek,offset: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Array of register offsets and masks + items: + - description: infra_misc_offset + - description: infra_conn_bt_cvsd_mask + - description: cvsd_mcu_read_offset + - description: cvsd_mcu_write_offset + - description: cvsd_packet_indicator_offset + +required: + - compatible + - reg + - interrupts + - mediatek,infracfg + - mediatek,offset + +additionalProperties: false + +examples: + - | + #include + + mtk-btcvsd-snd@18000000 { + compatible = "mediatek,mtk-btcvsd-snd"; + reg = <0x18000000 0x1000>, + <0x18080000 0x8000>; + interrupts = ; + mediatek,infracfg = <&infrasys>; + mediatek,offset = <0xf00 0x800 0xfd0 0xfd4 0xfd8>; + }; diff --git a/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt b/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt deleted file mode 100644 index 679e44839b48..000000000000 --- a/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt +++ /dev/null @@ -1,24 +0,0 @@ -Mediatek ALSA BT SCO CVSD/MSBC Driver - -Required properties: -- compatible = "mediatek,mtk-btcvsd-snd"; -- reg: register location and size of PKV and SRAM_BANK2 -- interrupts: should contain BTSCO interrupt -- mediatek,infracfg: the phandles of INFRASYS -- mediatek,offset: Array contains of register offset and mask - infra_misc_offset, - infra_conn_bt_cvsd_mask, - cvsd_mcu_read_offset, - cvsd_mcu_write_offset, - cvsd_packet_indicator_offset - -Example: - - mtk-btcvsd-snd@18000000 { - compatible = "mediatek,mtk-btcvsd-snd"; - reg=<0 0x18000000 0 0x1000>, - <0 0x18080000 0 0x8000>; - interrupts = ; - mediatek,infracfg = <&infrasys>; - mediatek,offset = <0xf00 0x800 0xfd0 0xfd4 0xfd8>; - }; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml index 47a105a97ecf..bc8c8ba24f9c 100644 --- a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml @@ -45,7 +45,7 @@ patternProperties: $ref: /schemas/types.yaml#/definitions/uint32 enum: [0, 1, 2] description: | - The direction of the dai stream:: + The direction of the dai stream: - Q6ASM_DAI_TX_RX (0) for both tx and rx - Q6ASM_DAI_TX (1) for only tx (Capture/Encode) - Q6ASM_DAI_RX (2) for only rx (Playback/Decode) diff --git a/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml b/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml index 9447a2f371b5..8e6973fa229c 100644 --- a/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml +++ b/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml @@ -90,7 +90,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-sinterval-low: $ref: /schemas/types.yaml#/definitions/uint8-array @@ -101,7 +101,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-sinterval: $ref: /schemas/types.yaml#/definitions/uint16-array @@ -112,7 +112,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-offset1: $ref: /schemas/types.yaml#/definitions/uint8-array @@ -123,7 +123,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-offset2: $ref: /schemas/types.yaml#/definitions/uint8-array @@ -134,7 +134,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-lane-control: $ref: /schemas/types.yaml#/definitions/uint8-array @@ -145,7 +145,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 qcom,ports-block-pack-mode: $ref: /schemas/types.yaml#/definitions/uint8-array @@ -158,7 +158,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 items: oneOf: - minimum: 0 @@ -175,7 +175,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 items: oneOf: - minimum: 0 @@ -192,7 +192,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 items: oneOf: - minimum: 0 @@ -208,7 +208,7 @@ properties: or applicable for the respective data port. More info in MIPI Alliance SoundWire 1.0 Specifications. minItems: 3 - maxItems: 16 + maxItems: 17 items: oneOf: - minimum: 0 diff --git a/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml b/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml new file mode 100644 index 000000000000..f0b708e1ccbb --- /dev/null +++ b/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/cavium,octeon-3010-spi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Cavium, Inc. OCTEON SoC SPI master controller + +description: + The Cavium OCTEON SPI controller is an SPI master controller found in + OCTEON SoCs. + +maintainers: + - Rob Herring + +allOf: + - $ref: spi-controller.yaml# + +properties: + compatible: + const: cavium,octeon-3010-spi + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + spi@1070000001000 { + compatible = "cavium,octeon-3010-spi"; + reg = <0x10700 0x00001000 0x0 0x100>; + interrupts = <0 58>; + #address-cells = <1>; + #size-cells = <0>; + + eeprom@0 { + compatible = "st,m95256", "atmel,at25"; + reg = <0>; + spi-max-frequency = <5000000>; + spi-cpha; + spi-cpol; + pagesize = <64>; + size = <32768>; + address-width = <16>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml index 8ebebcebca16..95a5bd894e93 100644 --- a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml +++ b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml @@ -50,10 +50,15 @@ properties: - enum: - mscc,ocelot-spi - mscc,jaguar2-spi - - renesas,rzn1-spi - sophgo,sg2042-spi - thead,th1520-spi - const: snps,dw-apb-ssi + - description: Vendor controllers which use snps,dwc-ssi-2.00a as fallback + items: + - enum: + - starfive,jhb100-spi + - const: snps,dwc-ssi-2.00a + - const: snps,dwc-ssi-1.01a - description: Intel Keem Bay SPI Controller const: intel,keembay-ssi - description: Intel Mount Evans Integrated Management Complex SPI Controller @@ -88,6 +93,9 @@ properties: - const: ssi_clk - const: pclk + power-domains: + maxItems: 1 + resets: maxItems: 1 diff --git a/Documentation/devicetree/bindings/spi/spi-octeon.txt b/Documentation/devicetree/bindings/spi/spi-octeon.txt deleted file mode 100644 index 431add192342..000000000000 --- a/Documentation/devicetree/bindings/spi/spi-octeon.txt +++ /dev/null @@ -1,33 +0,0 @@ -Cavium, Inc. OCTEON SOC SPI master controller. - -Required properties: -- compatible : "cavium,octeon-3010-spi" -- reg : The register base for the controller. -- interrupts : One interrupt, used by the controller. -- #address-cells : <1>, as required by generic SPI binding. -- #size-cells : <0>, also as required by generic SPI binding. - -Child nodes as per the generic SPI binding. - -Example: - - spi@1070000001000 { - compatible = "cavium,octeon-3010-spi"; - reg = <0x10700 0x00001000 0x0 0x100>; - interrupts = <0 58>; - #address-cells = <1>; - #size-cells = <0>; - - eeprom@0 { - compatible = "st,m95256", "atmel,at25"; - reg = <0>; - spi-max-frequency = <5000000>; - spi-cpha; - spi-cpol; - - pagesize = <64>; - size = <32768>; - address-width = <16>; - }; - }; - diff --git a/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml b/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml index 3f1a27efff80..ee57739b73b8 100644 --- a/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml +++ b/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml @@ -50,6 +50,9 @@ properties: minItems: 1 maxItems: 2 + power-domains: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml index 3b5005b96c6d..1593a1183a36 100644 --- a/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml +++ b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml @@ -25,6 +25,7 @@ properties: oneOf: - items: - enum: + - qcom,hawi-spmi-pmic-arb - qcom,kaanapali-spmi-pmic-arb - const: qcom,glymur-spmi-pmic-arb - enum: diff --git a/Documentation/devicetree/bindings/sram/qcom,imem.yaml b/Documentation/devicetree/bindings/sram/qcom,imem.yaml index c63026904061..eaa8ef367ff4 100644 --- a/Documentation/devicetree/bindings/sram/qcom,imem.yaml +++ b/Documentation/devicetree/bindings/sram/qcom,imem.yaml @@ -89,6 +89,26 @@ required: - compatible - reg +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,apq8064-imem + - qcom,msm8226-imem + - qcom,msm8974-imem + then: + properties: + reboot-mode: + patternProperties: + # Negative look-ahead to disallow unsupported modes. The '$' has to be + # part of lookahead group to work, instead of trailing outside of (). + "^mode-(?!(bootloader$|normal$|recovery$))": false + else: + properties: + reboot-mode: false + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml index eb695698a03e..5bf27c5f63e2 100644 --- a/Documentation/devicetree/bindings/sram/sram.yaml +++ b/Documentation/devicetree/bindings/sram/sram.yaml @@ -35,6 +35,7 @@ properties: - nvidia,tegra194-sysram - nvidia,tegra234-sysram - qcom,eliza-imem + - qcom,glymur-imem - qcom,hawi-imem - qcom,kaanapali-imem - qcom,milos-imem diff --git a/Documentation/devicetree/bindings/thermal/amlogic,thermal.yaml b/Documentation/devicetree/bindings/thermal/amlogic,thermal.yaml index e28612510d67..8cfa44dcda58 100644 --- a/Documentation/devicetree/bindings/thermal/amlogic,thermal.yaml +++ b/Documentation/devicetree/bindings/thermal/amlogic,thermal.yaml @@ -87,9 +87,12 @@ examples: amlogic,ao-secure = <&sec_AO>; }; - | + #include + #include + temperature-sensor@20000 { compatible = "amlogic,t7-thermal"; - reg = <0x0 0x20000 0x0 0x50>; + reg = <0x20000 0x50>; interrupts = ; clocks = <&clkc_periphs CLKID_TS>; #thermal-sensor-cells = <0>; diff --git a/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml b/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml index 29a08b0729ee..3f333db72a71 100644 --- a/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml +++ b/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml @@ -40,7 +40,7 @@ properties: description: | The Exynos TMU supports generating interrupts when reaching given temperature thresholds. Number of supported thermal trip points depends - on the SoC (only first trip points defined in DT will be configured):: + on the SoC (only first trip points defined in DT will be configured): - most of SoC: 4 - samsung,exynos5433-tmu: 8 - samsung,exynos7-tmu: 8 @@ -52,7 +52,7 @@ properties: - description: | Shared TMU registers. - Note:: On Exynos5420, the TRIMINFO register is misplaced for TMU + Note: On Exynos5420, the TRIMINFO register is misplaced for TMU channels 2, 3 and 4 Use "samsung,exynos5420-tmu-ext-triminfo" in cases, there is a misplaced register, also provide clock to access that base. diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index 105e3ef539e6..3673836e14de 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -50,6 +50,7 @@ properties: - items: - enum: - allwinner,sun20i-d1-clint + - canaan,k230-clint - sophgo,cv1800b-clint - sophgo,cv1812h-clint - sophgo,sg2000-clint diff --git a/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml index 75fae9f1eba7..db165a235cb6 100644 --- a/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml +++ b/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml @@ -11,7 +11,11 @@ maintainers: properties: compatible: - const: qcom,sa8255p-ufshc + oneOf: + - const: qcom,sa8255p-ufshc + - items: + - const: qcom,sa8797p-ufshc + - const: qcom,sa8255p-ufshc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml index f28641c6e68f..b441f0d26081 100644 --- a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml +++ b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml @@ -16,7 +16,9 @@ select: contains: enum: - qcom,eliza-ufshc + - qcom,hawi-ufshc - qcom,kaanapali-ufshc + - qcom,nord-ufshc - qcom,sm8650-ufshc - qcom,sm8750-ufshc required: @@ -27,7 +29,9 @@ properties: items: - enum: - qcom,eliza-ufshc + - qcom,hawi-ufshc - qcom,kaanapali-ufshc + - qcom,nord-ufshc - qcom,sm8650-ufshc - qcom,sm8750-ufshc - const: qcom,ufshc @@ -74,6 +78,7 @@ allOf: contains: enum: - qcom,eliza-ufshc + - qcom,nord-ufshc then: properties: reg: diff --git a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml index a7eb7ad85a94..710ce493f3b6 100644 --- a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml +++ b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml @@ -19,6 +19,7 @@ properties: - samsung,exynos7-ufs - samsung,exynosautov9-ufs - samsung,exynosautov9-ufs-vh + - samsung,exynosautov920-ufs - tesla,fsd-ufs reg: diff --git a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml index a199e5ba6416..e8082c5c05a2 100644 --- a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml +++ b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml @@ -4,29 +4,37 @@ $id: http://devicetree.org/schemas/usb/cdns,usb3.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Cadence USBSS-DRD controller +title: Cadence USBSS and USBSSP DRD controller maintainers: - Pawel Laszczak +description: + Cadence USB dual-role controller. Covers USBSS (SuperSpeed, USB 3.0) and + USBSSP (SuperSpeed Plus, USB 3.1 gen2x1). Both variants share the same + DRD/OTG register interface, so the driver auto-detects the controller + version at runtime. + properties: compatible: - const: cdns,usb3 + oneOf: + - const: cdns,usb3 + - items: + - {} + - const: cdns,cdnsp reg: - items: - - description: OTG controller registers - - description: XHCI Host controller registers - - description: DEVICE controller registers + minItems: 2 + maxItems: 3 reg-names: + minItems: 2 + maxItems: 3 items: - - const: otg - - const: xhci - - const: dev + enum: [ otg, xhci, dev ] interrupts: - minItems: 3 + minItems: 2 items: - description: XHCI host controller interrupt - description: Device controller interrupt @@ -35,7 +43,7 @@ properties: cleared by xhci core, this interrupt is optional interrupt-names: - minItems: 3 + minItems: 2 items: - const: host - const: peripheral @@ -49,7 +57,7 @@ properties: cdns3 to type C connector. maximum-speed: - enum: [super-speed, high-speed, full-speed] + enum: [super-speed-plus, super-speed, high-speed, full-speed] phys: minItems: 1 @@ -87,6 +95,47 @@ allOf: - $ref: usb-drd.yaml# - $ref: usb-xhci.yaml# + - if: + properties: + compatible: + contains: + const: cdns,cdnsp + then: + properties: + reg: + items: + - description: XHCI Host controller registers + - description: DEVICE controller registers + reg-names: + items: + - const: xhci + - const: dev + interrupts: + maxItems: 2 + interrupt-names: + items: + - const: host + - const: peripheral + dr_mode: + enum: [host, peripheral] + else: + properties: + reg: + items: + - description: OTG controller registers + - description: XHCI Host controller registers + - description: DEVICE controller registers + reg-names: + items: + - const: otg + - const: xhci + - const: dev + interrupts: + minItems: 3 + maxItems: 4 + interrupt-names: + minItems: 3 + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml index fec04702f530..620b564914d4 100644 --- a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml +++ b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml @@ -45,11 +45,11 @@ properties: clocks: minItems: 1 - maxItems: 2 + maxItems: 3 clock-names: minItems: 1 - maxItems: 2 + maxItems: 3 operating-points-v2: description: A phandle to the OPP table containing the performance states. @@ -92,6 +92,27 @@ allOf: - $ref: chipidea,usb2-common.yaml# - $ref: usb-hcd.yaml# - $ref: usb-drd.yaml# + - if: + properties: + compatible: + contains: + const: qcom,ci-hdrc + then: + properties: + clocks: + minItems: 2 + clock-names: + minItems: 2 + items: + - const: iface + - const: core + - const: fs + else: + properties: + clocks: + maxItems: 2 + clock-names: + maxItems: 2 unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml index 9a94b2a74a1e..e8b8c03f87a0 100644 --- a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml +++ b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml @@ -15,6 +15,7 @@ properties: - usb5e3,608 - usb5e3,610 - usb5e3,620 + - usb5e3,625 - usb5e3,626 reg: true @@ -26,6 +27,10 @@ properties: description: The regulator that provides 3.3V or 5.0V core power to the hub. + vdd12-supply: + description: + The regulator that provides 1.2V power to the hub. + peer-hub: true ports: @@ -56,6 +61,7 @@ allOf: properties: peer-hub: false vdd-supply: false + vdd12-supply: false - if: properties: @@ -68,6 +74,18 @@ allOf: properties: peer-hub: true vdd-supply: true + vdd12-supply: false + + - if: + properties: + compatible: + contains: + enum: + - usb5e3,625 + then: + properties: + peer-hub: true + vdd-supply: true unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml index a7f58114c02e..90daee616880 100644 --- a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml @@ -92,14 +92,14 @@ properties: clocks: description: | - Several clocks are used, depending on the variant. Typical ones are:: - - cfg_noc:: System Config NOC clock. - - core:: Master/Core clock, has to be >= 125 MHz for SS operation and >= + Several clocks are used, depending on the variant. Typical ones are: + - cfg_noc: System Config NOC clock. + - core: Master/Core clock, has to be >= 125 MHz for SS operation and >= 60MHz for HS operation. - - iface:: System bus AXI clock. - - sleep:: Sleep clock, used for wakeup when USB3 core goes into low + - iface: System bus AXI clock. + - sleep: Sleep clock, used for wakeup when USB3 core goes into low power mode (U3). - - mock_utmi:: Mock utmi clock needed for ITP/SOF generation in host + - mock_utmi: Mock utmi clock needed for ITP/SOF generation in host mode. Its frequency should be 19.2MHz. minItems: 1 maxItems: 9 diff --git a/Documentation/devicetree/bindings/usb/qcom,pmic-typec.yaml b/Documentation/devicetree/bindings/usb/qcom,pmic-typec.yaml index 6d3fa2bc9cee..ba790c4488b7 100644 --- a/Documentation/devicetree/bindings/usb/qcom,pmic-typec.yaml +++ b/Documentation/devicetree/bindings/usb/qcom,pmic-typec.yaml @@ -79,22 +79,17 @@ properties: - const: fr-swap vdd-vbus-supply: - description: VBUS power supply. + deprecated: true + description: use connector/vbus-supply instead. vdd-pdphy-supply: description: VDD regulator supply to the PDPHY. - port: - $ref: /schemas/graph.yaml#/properties/port - description: - Contains a port which produces data-role switching messages. - required: - compatible - reg - interrupts - interrupt-names - - vdd-vbus-supply allOf: - if: diff --git a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml index 8201656b41ed..d99af9f413d0 100644 --- a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml @@ -87,14 +87,14 @@ properties: clocks: description: | - Several clocks are used, depending on the variant. Typical ones are:: - - cfg_noc:: System Config NOC clock. - - core:: Master/Core clock, has to be >= 125 MHz for SS operation and >= + Several clocks are used, depending on the variant. Typical ones are: + - cfg_noc: System Config NOC clock. + - core: Master/Core clock, has to be >= 125 MHz for SS operation and >= 60MHz for HS operation. - - iface:: System bus AXI clock. - - sleep:: Sleep clock, used for wakeup when USB3 core goes into low + - iface: System bus AXI clock. + - sleep: Sleep clock, used for wakeup when USB3 core goes into low power mode (U3). - - mock_utmi:: Mock utmi clock needed for ITP/SOF generation in host + - mock_utmi: Mock utmi clock needed for ITP/SOF generation in host mode. Its frequency should be 19.2MHz. minItems: 1 maxItems: 9 diff --git a/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt b/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt deleted file mode 100644 index 6f8115db2ea9..000000000000 --- a/Documentation/devicetree/bindings/usb/richtek,rt1711h.txt +++ /dev/null @@ -1,44 +0,0 @@ -Richtek RT1711H TypeC PD Controller. - -Required properties: - - compatible : Must be "richtek,rt1711h". - - reg : Must be 0x4e, it's slave address of RT1711H. - - interrupts : where a is the interrupt number and b represents an - encoding of the sense and level information for the interrupt. - -Required sub-node: -- connector: The "usb-c-connector" attached to the tcpci chip, the bindings - of connector node are specified in - Documentation/devicetree/bindings/connector/usb-connector.yaml - -Example : -rt1711h@4e { - compatible = "richtek,rt1711h"; - reg = <0x4e>; - interrupt-parent = <&gpio26>; - interrupts = <0 IRQ_TYPE_LEVEL_LOW>; - - usb_con: connector { - compatible = "usb-c-connector"; - label = "USB-C"; - data-role = "dual"; - power-role = "dual"; - try-power-role = "sink"; - source-pdos = ; - sink-pdos = ; - op-sink-microwatt = <10000000>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@1 { - reg = <1>; - usb_con_ss: endpoint { - remote-endpoint = <&usb3_data_ss>; - }; - }; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/usb/rockchip,dwc3.yaml b/Documentation/devicetree/bindings/usb/rockchip,dwc3.yaml index fd1b13c0ed6b..0554dbc4b854 100644 --- a/Documentation/devicetree/bindings/usb/rockchip,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/rockchip,dwc3.yaml @@ -26,6 +26,7 @@ select: contains: enum: - rockchip,rk3328-dwc3 + - rockchip,rk3528-dwc3 - rockchip,rk3562-dwc3 - rockchip,rk3568-dwc3 - rockchip,rk3576-dwc3 @@ -38,6 +39,7 @@ properties: items: - enum: - rockchip,rk3328-dwc3 + - rockchip,rk3528-dwc3 - rockchip,rk3562-dwc3 - rockchip,rk3568-dwc3 - rockchip,rk3576-dwc3 @@ -135,6 +137,7 @@ allOf: compatible: contains: enum: + - rockchip,rk3528-dwc3 - rockchip,rk3568-dwc3 - rockchip,rk3576-dwc3 then: diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 6b9fb6a6bf0b..85a27937832e 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -225,10 +225,14 @@ patternProperties: description: Axis Communications AB "^ayaneo,.*": description: Anyun Intelligent Technology (Hong Kong) Co., Ltd + "^ayntec,.*": + description: AYN Technologies Co., Ltd. "^azoteq,.*": description: Azoteq (Pty) Ltd "^azw,.*": description: Shenzhen AZW Technology Co., Ltd. + "^baijie,.*": + description: Shenzhen Baijie Technology Co., Ltd. "^baikal,.*": description: BAIKAL ELECTRONICS, JSC "^bananapi,.*": @@ -491,6 +495,8 @@ patternProperties: description: Emtop Embedded Solutions "^eeti,.*": description: eGalax_eMPIA Technology Inc + "^efinix,.*": + description: Efinix, Inc. "^egnite,.*": description: egnite GmbH "^einfochips,.*": @@ -602,6 +608,8 @@ patternProperties: description: Flipkart Inc. "^focaltech,.*": description: FocalTech Systems Co.,Ltd + "^focuslcds,.*": + description: Focus Display Solutions, Inc. "^forlinx,.*": description: Baoding Forlinx Embedded Technology Co., Ltd. "^foursemi,.*": @@ -1361,6 +1369,8 @@ patternProperties: description: Shenzhen QiShenglong Industrialist Co., Ltd. "^qnap,.*": description: QNAP Systems, Inc. + "^qstcorp,.*": + description: QST Corporation Limited "^quanta,.*": description: Quanta Computer Inc. "^radxa,.*": @@ -1409,6 +1419,8 @@ patternProperties: description: Embest RIoT "^riscv,.*": description: RISC-V Foundation + "^riverdi,.*": + description: Riverdi Sp. z o.o "^rockchip,.*": description: Rockchip Electronics Co., Ltd. "^rocktech,.*": @@ -1923,6 +1935,8 @@ patternProperties: description: YSH & ATIL "^yones-toptech,.*": description: Yones Toptech Co., Ltd. + "^youyeetoo,.*": + description: Hong Kong Cybodev Tech Limited (youyeetoo) "^ys,.*": description: Shenzhen Yashi Changhua Intelligent Technology Co., Ltd. "^ysoft,.*": diff --git a/Documentation/devicetree/bindings/w1/omap-hdq.txt b/Documentation/devicetree/bindings/w1/omap-hdq.txt deleted file mode 100644 index 913c5f91a0f9..000000000000 --- a/Documentation/devicetree/bindings/w1/omap-hdq.txt +++ /dev/null @@ -1,22 +0,0 @@ -* OMAP HDQ One wire bus master controller - -Required properties: -- compatible : should be "ti,omap3-1w" or "ti,am4372-hdq" -- reg : Address and length of the register set for the device -- interrupts : interrupt line. -- ti,hwmods : "hdq1w" - -Optional properties: -- ti,mode: should be "hdq": HDQ mode "1w": one-wire mode. - If not specified HDQ mode is implied. - -Example: - -- From omap3.dtsi - hdqw1w: 1w@480b2000 { - compatible = "ti,omap3-1w"; - reg = <0x480b2000 0x1000>; - interrupts = <58>; - ti,hwmods = "hdq1w"; - ti,mode = "hdq"; - }; diff --git a/Documentation/devicetree/bindings/w1/ti,hdq.yaml b/Documentation/devicetree/bindings/w1/ti,hdq.yaml new file mode 100644 index 000000000000..fd402c0510db --- /dev/null +++ b/Documentation/devicetree/bindings/w1/ti,hdq.yaml @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/w1/ti,hdq.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments OMAP HDQ One Wire Bus Master Controller + +maintainers: + - Eduard Bostina + +properties: + compatible: + enum: + - ti,omap3-1w + - ti,am4372-hdq + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: fck + + ti,hwmods: + description: Name of the hwmod associated to the hdq + $ref: /schemas/types.yaml#/definitions/string + const: hdq1w + + ti,mode: + description: | + Operation mode. "hdq" for HDQ mode, "1w" for One-Wire mode. + If not specified, HDQ mode is implied. + $ref: /schemas/types.yaml#/definitions/string + enum: + - hdq + - 1w + default: hdq + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + onewire@0 { + compatible = "ti,am4372-hdq"; + reg = <0x0 0x1000>; + interrupts = ; + clocks = <&func_12m_clk>; + clock-names = "fck"; + }; + - | + onewire@480b2000 { + compatible = "ti,omap3-1w"; + reg = <0x480b2000 0x1000>; + interrupts = <58>; + ti,hwmods = "hdq1w"; + ti,mode = "hdq"; + }; diff --git a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml index 74117f5726a7..4ff61102e407 100644 --- a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml @@ -27,6 +27,7 @@ properties: - qcom,apss-wdt-ipq9574 - qcom,apss-wdt-ipq9650 - qcom,apss-wdt-kaanapali + - qcom,apss-wdt-maili - qcom,apss-wdt-msm8226 - qcom,apss-wdt-msm8974 - qcom,apss-wdt-msm8994 diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst index 8b6a5888cb11..3b182cfdf135 100644 --- a/Documentation/driver-api/basics.rst +++ b/Documentation/driver-api/basics.rst @@ -90,6 +90,9 @@ Reference counting .. kernel-doc:: lib/refcount.c :export: +.. kernel-doc:: include/linux/percpu-refcount.h +.. kernel-doc:: lib/percpu-refcount.c + Atomics ------- diff --git a/Documentation/driver-api/cxl/linux/dax-driver.rst b/Documentation/driver-api/cxl/linux/dax-driver.rst index 10d953a2167b..72c85a0f8606 100644 --- a/Documentation/driver-api/cxl/linux/dax-driver.rst +++ b/Documentation/driver-api/cxl/linux/dax-driver.rst @@ -35,9 +35,9 @@ will be exposed to the kernel page allocator in the user-selected memory zone. The :code:`memmap_on_memory` setting (both global and DAX device local) -dictates where the kernell will allocate the :code:`struct folio` descriptors +dictates where the kernel will allocate the :code:`struct folio` descriptors for this memory will come from. If :code:`memmap_on_memory` is set, memory hotplug will set aside a portion of the memory block capacity to allocate folios. If unset, the memory is allocated via a normal :code:`GFP_KERNEL` -allocation - and as a result will most likely land on the local NUM node of the +allocation - and as a result will most likely land on the local NUMA node of the CPU executing the hotplug operation. diff --git a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst index d491e385d61a..5ee5d4a3596d 100644 --- a/Documentation/driver-api/dmaengine/client.rst +++ b/Documentation/driver-api/dmaengine/client.rst @@ -80,6 +80,10 @@ The details of these operations are: - slave_sg: DMA a list of scatter gather buffers from/to a peripheral + - config_sg: Similar with slave_sg, just pass down dma_slave_config + struct to avoid calling dmaengine_slave_config() every time adjusting the + burst length or the FIFO address is needed. + - peripheral_dma_vec: DMA an array of scatter gather buffers from/to a peripheral. Similar to slave_sg, but uses an array of dma_vec structures instead of a scatterlist. @@ -106,6 +110,11 @@ The details of these operations are: unsigned int sg_len, enum dma_data_direction direction, unsigned long flags); + struct dma_async_tx_descriptor *dmaengine_prep_config_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, struct dma_slave_config *config); + struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( struct dma_chan *chan, const struct dma_vec *vecs, size_t nents, enum dma_data_direction direction, diff --git a/Documentation/driver-api/iio/buffers.rst b/Documentation/driver-api/iio/buffers.rst index 63f364e862d1..8779022e3da5 100644 --- a/Documentation/driver-api/iio/buffers.rst +++ b/Documentation/driver-api/iio/buffers.rst @@ -37,9 +37,10 @@ directory contains attributes of the following form: * :file:`index`, the scan_index of the channel. * :file:`type`, description of the scan element data storage within the buffer and hence the form in which it is read from user space. - Format is [be|le]:[s|u]bits/storagebits[Xrepeat][>>shift] . + Format is [be|le]:[f|s|u]bits/storagebits[Xrepeat][>>shift] . * *be* or *le*, specifies big or little endian. + * *f*, specifies if floating-point. * *s* or *u*, specifies if signed (2's complement) or unsigned. * *bits*, is the number of valid data bits. * *storagebits*, is the number of bits (after padding) that it occupies in the @@ -78,7 +79,7 @@ fields in iio_chan_spec definition:: /* other members */ int scan_index struct { - char sign; + char format; u8 realbits; u8 storagebits; u8 shift; @@ -98,7 +99,7 @@ following channel definition:: /* other stuff here */ .scan_index = 0, .scan_type = { - .sign = 's', + .format = IIO_SCAN_FORMAT_SIGNED_INT, .realbits = 12, .storagebits = 16, .shift = 4, diff --git a/Documentation/driver-api/iio/triggered-buffers.rst b/Documentation/driver-api/iio/triggered-buffers.rst index 417555dbbdf4..23762b06fdc6 100644 --- a/Documentation/driver-api/iio/triggered-buffers.rst +++ b/Documentation/driver-api/iio/triggered-buffers.rst @@ -29,21 +29,21 @@ A typical triggered buffer setup looks like this:: irqreturn_t sensor_trigger_handler(int irq, void *p) { - u16 buf[8]; + IIO_DECLARE_BUFFER_WITH_TS(u16, buf, 3) = { }; int i = 0; /* read data for each active channel */ - for_each_set_bit(bit, active_scan_mask, masklength) - buf[i++] = sensor_get_data(bit) + iio_for_each_active_channel(indio_dev, bit) + buf[i++] = sensor_get_data(bit); - iio_push_to_buffers_with_timestamp(indio_dev, buf, timestamp); + iio_push_to_buffers_with_ts(indio_dev, buf, sizeof(buf), timestamp); iio_trigger_notify_done(trigger); return IRQ_HANDLED; } /* setup triggered buffer, usually in probe function */ - iio_triggered_buffer_setup(indio_dev, sensor_iio_polfunc, + iio_triggered_buffer_setup(indio_dev, sensor_iio_pollfunc, sensor_trigger_handler, sensor_buffer_setup_ops); diff --git a/Documentation/driver-api/ipmi.rst b/Documentation/driver-api/ipmi.rst index f52ab2df2569..d08cee98e34a 100644 --- a/Documentation/driver-api/ipmi.rst +++ b/Documentation/driver-api/ipmi.rst @@ -495,7 +495,7 @@ tuned to the user's desired performance. The driver supports a hot add and remove of interfaces. This way, interfaces can be added or removed after the kernel is up and running. -This is done using /sys/modules/ipmi_si/parameters/hotmod, which is a +This is done using /sys/module/ipmi_si/parameters/hotmod, which is a write-only parameter. You write a string to this interface. The string has the format:: diff --git a/Documentation/driver-api/mtd/spi-nor.rst b/Documentation/driver-api/mtd/spi-nor.rst index 148fa4288760..747a326fb6c0 100644 --- a/Documentation/driver-api/mtd/spi-nor.rst +++ b/Documentation/driver-api/mtd/spi-nor.rst @@ -203,3 +203,173 @@ section, after the ``---`` marker. mtd.writesize = 1 mtd.oobsize = 0 regions = 0 + +5) If your flash supports locking, please go through the following test + procedure to make sure it correctly behaves. The below example + expects the typical situation where eraseblocks and lock sectors have + the same size. In case you enabled MTD_SPI_NOR_USE_4K_SECTORS, you + must adapt `bs` accordingly. + + Warning: These tests may hard lock your device! Make sure: + + - The device is not hard locked already (#WP strapped to low and + SR_SRWD bit set) + - If you have a WPn pin, you may want to set `no-wp` in your DT for + the time of the test, to only make use of software protection. + Otherwise, clearing the locking state depends on the WPn + signal and if it is tied to low, the flash will be permanently + locked. + + Test full chip locking and make sure expectations, the MEMISLOCKED + ioctl output, the debugfs output and experimental results are all + aligned:: + + root@1:~# alias show_sectors='grep -A4 "locked sectors" /sys/kernel/debug/spi-nor/spi0.0/params' + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -i /dev/mtd0 + Device: /dev/mtd0 + Start: 0 + Len: 0x4000000 + Lock status: unlocked + Return code: 0 + root@1:~# mtd_debug erase /dev/mtd0 0 2097152 + Erased 2097152 bytes from address 0x00000000 in flash + root@1:~# mtd_debug write /dev/mtd0 0 2097152 spi_test + Copied 2097152 bytes from spi_test to address 0x00000000 in flash + root@1:~# mtd_debug read /dev/mtd0 0 2097152 spi_read + Copied 2097152 bytes from address 0x00000000 in flash to spi_read + root@1:~# sha256sum spi* + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_read + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_test + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03ffffff | unlocked | 1024 + + root@1:~# flash_lock -l /dev/mtd0 + root@1:~# flash_lock -i /dev/mtd0 + Device: /dev/mtd0 + Start: 0 + Len: 0x4000000 + Lock status: locked + Return code: 1 + root@1:~# mtd_debug erase /dev/mtd0 0 2097152 + Erased 2097152 bytes from address 0x00000000 in flash + root@1:~# mtd_debug read /dev/mtd0 0 2097152 spi_read + Copied 2097152 bytes from address 0x00000000 in flash to spi_read + root@1:~# sha256sum spi* + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_read + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_test + root@1:~# dd if=/dev/urandom of=./spi_test2 bs=1M count=2 + 2+0 records in + 2+0 records out + root@1:~# mtd_debug write /dev/mtd0 0 2097152 spi_test2 + Copied 2097152 bytes from spi_test2 to address 0x00000000 in flash + root@1:~# mtd_debug read /dev/mtd0 0 2097152 spi_read2 + Copied 2097152 bytes from address 0x00000000 in flash to spi_read2 + root@1:~# sha256sum spi* + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_read + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_read2 + c444216a6ba2a4a66cccd60a0dd062bce4b865dd52b200ef5e21838c4b899ac8 spi_test + bea9334df51c620440f86751cba0799214a016329f1736f9456d40cf40efdc88 spi_test2 + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03ffffff | locked | 1024 + root@1:~# flash_lock -u /dev/mtd0 + + Once we trust the debugfs output we can use it to test various + situations. Check top locking/unlocking (end of the device):: + + root@1:~# size=$(cat /sys/class/mtd/mtd0/size) + root@1:~# bs=$(cat /sys/class/mtd/mtd0/erasesize) + root@1:~# nsectors=$(grep unlocked /sys/kernel/debug/spi-nor/spi0.0/params | sed -e 's/.*unlocked | //') + root@1:~# ss=$(($size / $nsectors)) + root@1:~# bps=$(($ss / $bs)) + + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -l /dev/mtd0 $(($size - (2 * $ss))) $((2 * $bps)) # last two + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03fdffff | unlocked | 1022 + 03fe0000-03ffffff | locked | 2 + root@1:~# flash_lock -u /dev/mtd0 $(($size - (2 * $ss))) $((1 * $bps)) # last one + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03feffff | unlocked | 1023 + 03ff0000-03ffffff | locked | 1 + + If the flash features 4 block protection bits (BP), we can protect + more than 4MB (typically 128 64kiB-blocks or more), with a finer + grain than locking the entire device:: + + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -l /dev/mtd0 $(($size - (2**7 * $ss))) $((2**7 * $bps)) + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-037fffff | unlocked | 896 + 03800000-03ffffff | locked | 128 + + If the flash features a Top/Bottom (TB) bit, we can protect the + beginning of the flash:: + + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -l /dev/mtd0 0 $((2 * $bps)) # first two + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-0001ffff | locked | 2 + 00020000-03ffffff | unlocked | 1022 + root@1:~# flash_lock -u /dev/mtd0 $ss $((1 * $bps)) # first one + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-0000ffff | locked | 1 + 00010000-03ffffff | unlocked | 1023 + + If the flash features a Complement (CMP) bit, we can protect with + more granularity above half of the capacity. Let's lock all but one + block, then unlock one more block:: + + root@1:~# all_but_one=$((($size / $bs) - ($ss / $bs))) + + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -l /dev/mtd0 $ss $all_but_one # all but the first + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-0000ffff | unlocked | 1 + 00010000-03ffffff | locked | 1023 + root@1:~# flash_lock -u /dev/mtd0 $ss $(($ss / $bs)) # all but the two first + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-0001ffff | unlocked | 2 + 00020000-03ffffff | locked | 1022 + root@1:~# flash_lock -u /dev/mtd0 + root@1:~# flash_lock -l /dev/mtd0 0 $all_but_one # same from the other side + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03feffff | locked | 1023 + 03ff0000-03ffffff | unlocked | 1 + root@1:~# flash_lock -u /dev/mtd0 $(($size - (2 * $ss))) $(($ss / $bs)) # all but two + root@1:~# show_sectors + software locked sectors + region (in hex) | status | #sectors + ------------------+----------+--------- + 00000000-03fdffff | locked | 1022 + 03fe0000-03ffffff | unlocked | 2 diff --git a/Documentation/driver-api/serial/serial-rs485.rst b/Documentation/driver-api/serial/serial-rs485.rst index dce061ef7647..f53043d21071 100644 --- a/Documentation/driver-api/serial/serial-rs485.rst +++ b/Documentation/driver-api/serial/serial-rs485.rst @@ -132,4 +132,4 @@ RS485 Serial Communications 6. References ============= -.. [#DT-bindings] Documentation/devicetree/bindings/serial/rs485.txt +.. [#DT-bindings] Documentation/devicetree/bindings/serial/rs485.yaml diff --git a/Documentation/driver-api/uio-howto.rst b/Documentation/driver-api/uio-howto.rst index 907ffa3b38f5..c08472dfbcfe 100644 --- a/Documentation/driver-api/uio-howto.rst +++ b/Documentation/driver-api/uio-howto.rst @@ -246,10 +246,10 @@ the members are required, others are optional. hardware interrupt number. The flags given here will be used in the call to :c:func:`request_irq()`. -- ``int (*mmap)(struct uio_info *info, struct vm_area_struct *vma)``: +- ``int (*mmap_prepare)(struct uio_info *info, struct vm_area_desc *desc)``: Optional. If you need a special :c:func:`mmap()` function, you can set it here. If this pointer is not NULL, your - :c:func:`mmap()` will be called instead of the built-in one. + ``mmap_prepare`` will be called instead of the built-in one. - ``int (*open)(struct uio_info *info, struct inode *inode)``: Optional. You might want to have your own :c:func:`open()`, diff --git a/Documentation/features/core/thread-info-in-task/arch-support.txt b/Documentation/features/core/thread-info-in-task/arch-support.txt index f3d744c76061..e26efdfbb6b4 100644 --- a/Documentation/features/core/thread-info-in-task/arch-support.txt +++ b/Documentation/features/core/thread-info-in-task/arch-support.txt @@ -12,7 +12,7 @@ | arm64: | ok | | csky: | TODO | | hexagon: | TODO | - | loongarch: | TODO | + | loongarch: | ok | | m68k: | TODO | | microblaze: | TODO | | mips: | TODO | diff --git a/Documentation/filesystems/9p.rst b/Documentation/filesystems/9p.rst index be3504ca034a..3f65db648db0 100644 --- a/Documentation/filesystems/9p.rst +++ b/Documentation/filesystems/9p.rst @@ -23,13 +23,10 @@ the 9p client is available in the form of a USENIX paper: Other applications are described in the following papers: * XCPU & Clustering - http://xcpu.org/papers/xcpu-talk.pdf * KVMFS: control file system for KVM - http://xcpu.org/papers/kvmfs.pdf * CellFS: A New Programming Model for the Cell BE - http://xcpu.org/papers/cellfs-talk.pdf * PROSE I/O: Using 9p to enable Application Partitions - http://plan9.escet.urjc.es/iwp9/cready/PROSE_iwp9_2006.pdf + http://web.archive.org/web/20110101152020/http://plan9.escet.urjc.es/iwp9/cready/PROSE_iwp9_2006.pdf * VirtFS: A Virtualization Aware File System pass-through https://kernel.org/doc/ols/2010/ols2010-pages-109-120.pdf @@ -238,6 +235,11 @@ Options cachetag cache tag to use the specified persistent cache. cache tags for existing cache sessions can be listed at /sys/fs/9p/caches. (applies only to cache=fscache) + + negtimeout the duration (in milliseconds) that negative dentries (paths + that do not actually exist) are retained in the cache. If + set to a negative value, those entries are kept indefinitely + until evicted by the buffer cache management system ============= =============================================================== Behavior diff --git a/Documentation/filesystems/erofs.rst b/Documentation/filesystems/erofs.rst index fe06308e546c..4230884fb359 100644 --- a/Documentation/filesystems/erofs.rst +++ b/Documentation/filesystems/erofs.rst @@ -7,83 +7,90 @@ EROFS - Enhanced Read-Only File System Overview ======== -EROFS filesystem stands for Enhanced Read-Only File System. It aims to form a -generic read-only filesystem solution for various read-only use cases instead -of just focusing on storage space saving without considering any side effects -of runtime performance. - -It is designed to meet the needs of flexibility, feature extendability and user -payload friendly, etc. Apart from those, it is still kept as a simple -random-access friendly high-performance filesystem to get rid of unneeded I/O -amplification and memory-resident overhead compared to similar approaches. - -It is implemented to be a better choice for the following scenarios: - - - read-only storage media or - - - part of a fully trusted read-only solution, which means it needs to be +EROFS (Enhanced Read-Only File System) is a modern, efficient, and secure +read-only kernel filesystem designed for various use cases including immutable +system images, container images, application sandbox images, and dataset +distribution. + +An immutable image filesystem can be regarded as an enhanced archive format +which allows golden images to be built once and mounted everywhere -- images are +bit-for-bit identical across all deployments and can be verified, audited, or +shared without concerns about runtime modifications (in this model, all user +writes should be redirected into another trusted filesystem, for example, via +overlayfs for copy-on-write-style redirection, by design). + +EROFS is a dedicated implementation of the image filesystem idea above, with a +flexible, hierarchical on-disk design so that needed features can be enabled on +demand. Filesystem data in the core format is strictly block-aligned in order +to perform optimally on all kinds of storage media, including block devices and +memory-backed devices. The on-disk format is easy to parse and purposely avoids +the unnecessary metadata redundancy found in generic writable filesystems, which +can suffer from extra inconsistency issues -- making it ideal for security +auditing and untrusted remote access. In addition, designs such as inline data, +inline/shared extended attributes, and optimized (de)compression provide better +space efficiency while maintaining high performance. + +In short, EROFS aims to be a better fit for the following scenarios: + + - As part of a secure immutable storage solution, where it needs to be immutable and bit-for-bit identical to the official golden image for - their releases due to security or other considerations and - - - hope to minimize extra storage space with guaranteed end-to-end performance - by using compact layout, transparent file compression and direct access, - especially for those embedded devices with limited memory and high-density - hosts with numerous containers. + each individual copy, in order to meet security, data sharing, and/or + other requirements; -Here are the main features of EROFS: + - Minimizing storage overhead with guaranteed end-to-end performance + by using compact (meta)data layout, optimized transparent data compression, + deduplication and direct access, especially for those embedded devices with + limited memory and high-density hosts with numerous containers. - - Little endian on-disk design; +Here is the list of highlights: - - Block-based distribution and file-based distribution over fscache are - supported; + - Little endian on-disk design with 48-bit block addressing, supporting up + to 1 EiB filesystem capacity with 4 KiB block size; - - Support multiple devices to refer to external blobs, which can be used - for container images; + - Two compact inode metadata layouts for space and performance efficiency: - - 32-bit block addresses for each device, therefore 16TiB address space at - most with 4KiB block size for now; + ======================== ======== ====================================== + compact extended + ======================== ======== ====================================== + Inode core metadata size 32 bytes 64 bytes + Max file size 4 GiB 16 EiB (also limited by max. vol size) + Max uids/gids 65536 4294967296 + Nanosecond timestamps no yes + Max hardlinks 65536 4294967296 + ======================== ======== ====================================== - - Two inode layouts for different requirements: + - Support tailpacking inline data for better space efficiency and reduce + unneeded I/O amplification; - ===================== ============ ====================================== - compact (v1) extended (v2) - ===================== ============ ====================================== - Inode metadata size 32 bytes 64 bytes - Max file size 4 GiB 16 EiB (also limited by max. vol size) - Max uids/gids 65536 4294967296 - Per-inode timestamp no yes (64 + 32-bit timestamp) - Max hardlinks 65536 4294967296 - Metadata reserved 8 bytes 18 bytes - ===================== ============ ====================================== + - Block-based and file-backed distribution are both supported; - - Support extended attributes as an option; + - Multiple devices to reference external data blobs: inode data can be + optionally placed into external blobs, which enables image layering and data + sharing among different filesystems; - - Support a bloom filter that speeds up negative extended attribute lookups; + - Inline and shared extended attributes with an optional bloom filter that + speeds up negative extended attribute lookups; - - Support POSIX.1e ACLs by using extended attributes; + - POSIX.1e ACLs by using extended attributes; - - Support transparent data compression as an option: - LZ4, MicroLZMA, DEFLATE and Zstandard algorithms can be used on a per-file - basis; In addition, inplace decompression is also supported to avoid bounce - compressed buffers and unnecessary page cache thrashing. + - Transparent data compression as an option: Supported algorithms (LZ4, + MicroLZMA, DEFLATE and Zstandard) can be selected on a per-inode basis. + Both the on-disk metadata and decompression runtime have been heavily + optimized to minimize the overhead for better performance. - - Support chunk-based data deduplication and rolling-hash compressed data - deduplication; + - Merging tail-end data into a special inode as fragments; - - Support tailpacking inline compared to byte-addressed unaligned metadata - or smaller block size alternatives; + - Chunk-based deduplication and rolling-hash compressed data deduplication; - - Support merging tail-end data into a special inode as fragments. + - Direct I/O and FSDAX support on uncompressed inodes for use cases such as + secure containers, loop devices, and ramdisks that do not need page caching; - - Support large folios to make use of THPs (Transparent Hugepages); + - Page cache sharing among inodes with identical content fingerprints on + the same machine. - - Support direct I/O on uncompressed files to avoid double caching for loop - devices; +For more detailed information, please refer to our documentation site: - - Support FSDAX on uncompressed images for secure containers and ramdisks in - order to get rid of unnecessary page cache. - - - Support file-based on-demand loading with the Fscache infrastructure. +- https://erofs.docs.kernel.org The following git tree provides the file system user-space tools under development, such as a formatting tool (mkfs.erofs), an on-disk consistency & @@ -91,10 +98,6 @@ compatibility checking tool (fsck.erofs), and a debugging tool (dump.erofs): - git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git -For more information, please also refer to the documentation site: - -- https://erofs.docs.kernel.org - Bugs and patches are welcome, please kindly help us and send to the following linux-erofs mailing list: @@ -127,12 +130,9 @@ dax A legacy option which is an alias for ``dax=always``. device=%s Specify a path to an extra device to be used together. directio (For file-backed mounts) Use direct I/O to access backing files, and asynchronous I/O will be enabled if supported. -fsid=%s Specify a filesystem image ID for Fscache back-end. -domain_id=%s Specify a trusted domain ID for fscache mode so that - different images with the same blobs, identified by blob IDs, - can share storage within the same trusted domain. - Also used for different filesystems with inode page sharing - enabled to share page cache within the trusted domain. +domain_id=%s Specify a trusted domain ID. Filesystems sharing the same + domain ID can share page cache across mounts when inode + page sharing is enabled. (not shown in mountinfo output) fsoffset=%llu Specify block-aligned filesystem offset for the primary device. inode_share Enable inode page sharing for this filesystem. Inodes with identical content within the same domain ID can share the diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index 7e4031631286..8c4a14ae444f 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -137,6 +137,15 @@ noacl Disable POSIX Access Control List. Note: acl is enabled active_logs=%u Support configuring the number of active logs. In the current design, f2fs supports only 2, 4, and 6 logs. Default number is 6. + When the underlying block device exposes write + streams, the default active_logs=6 configuration + maps hot, warm, and cold DATA writes to streams 1, + 2, and 3, respectively. If only one or two write + streams are available, f2fs falls back to mapping + all DATA writes to stream 1 or mapping hot/warm + to stream 1 and cold to stream 2. If no write + streams are exposed, f2fs leaves the stream + unset. disable_ext_identify Disable the extension list configured by mkfs, so f2fs is not aware of cold files such as media files. inline_xattr Enable the inline xattrs feature. diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 08d01bc62c31..c274c5eef733 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -61,7 +61,7 @@ inode_operations prototypes:: - int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t, bool); + int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t); struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst index eb846518e6ac..1a29a5afabd7 100644 --- a/Documentation/filesystems/overlayfs.rst +++ b/Documentation/filesystems/overlayfs.rst @@ -347,6 +347,22 @@ The resulting access permissions should be the same. The difference is in the time of copy (on-demand vs. up-front). +Idmapped mounts +--------------- + +The overlay mount itself can be turned into an idmapped mount by applying an +idmapping to it with mount_setattr(2) and MOUNT_ATTR_IDMAP, just like for +other filesystems that support idmapped mounts. + +The mount idmapping only changes how ownership and permissions of the overlay +inodes are presented to and interpreted for the caller. It does not change +how overlayfs accesses the underlying layers: those are still accessed with +the stashed mounter's credentials through their own mounts, which may +themselves be idmapped. The overlay mount idmapping and any layer idmapping +compose, an underlying id is first mapped according to the relevant layer +idmapping and then according to the overlay mount idmapping. + + Multiple lower layers --------------------- diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index d13f0a23c882..02522fbfd968 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -1401,3 +1401,11 @@ as with d_dispose_if_unused() these are not trivial; with this variant of API it's more explicit, since grabbing ->d_lock is caller-side, but d_dispose_if_unused() had all the same issues. It's a low-level primitive; use only if you have no alternative. + +--- + +**mandatory** + +The .create inode_operation no longer receives the 'excl' arg. It must +always assume the file does not already exist. If the filesystem needs +to be involved in non-exclusive create, it should provide atomic_open. diff --git a/Documentation/filesystems/smb/ksmbd.rst b/Documentation/filesystems/smb/ksmbd.rst index 67cb68ea6e68..672c5d3892ff 100644 --- a/Documentation/filesystems/smb/ksmbd.rst +++ b/Documentation/filesystems/smb/ksmbd.rst @@ -97,7 +97,7 @@ ACLs Partially Supported. only DACLs available, SACLs to allow future support for running as a domain member. Kerberos Supported. -Durable handle v1,v2 Planned for future. +Durable handle v1,v2 Supported. Persistent handle Planned for future. SMB2 notify Planned for future. Sparse file support Supported. @@ -111,7 +111,7 @@ DCE/RPC support Partially Supported. a few calls(NetShareEnumAll, for Witness protocol e.g.) ksmbd/nfsd interoperability Planned for future. The features that ksmbd support are Leases, Notify, ACLs and Share modes. -SMB3.1.1 Compression Planned for future. +SMB3.1.1 Compression Supported. SMB3.1.1 over QUIC Planned for future. Signing/Encryption over RDMA Planned for future. SMB3.1.1 GMAC signing support Planned for future. diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 7c753148af88..651b83b00440 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -415,7 +415,7 @@ As of kernel 2.6.22, the following members are defined: .. code-block:: c struct inode_operations { - int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, umode_t, bool); + int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, umode_t); struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); diff --git a/Documentation/gpu/automated_testing.rst b/Documentation/gpu/automated_testing.rst index 62aa3ede02a5..8a7328aef10e 100644 --- a/Documentation/gpu/automated_testing.rst +++ b/Documentation/gpu/automated_testing.rst @@ -99,7 +99,8 @@ How to enable automated testing on your tree ============================================ 1. Create a Linux tree in https://gitlab.freedesktop.org/ if you don't have one -yet +yet, by forking https://gitlab.freedesktop.org/drm/kernel (this allows GitLab +to internally track that these are the same git objects). 2. In your kernel repo's configuration (eg. https://gitlab.freedesktop.org/janedoe/linux/-/settings/ci_cd), change the diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst index 80453dda33b8..94cfc26acecc 100644 --- a/Documentation/gpu/drm-kms-helpers.rst +++ b/Documentation/gpu/drm-kms-helpers.rst @@ -171,6 +171,12 @@ Bridge Operations .. kernel-doc:: drivers/gpu/drm/drm_bridge.c :doc: bridge operations +Bridge Chain Format Selection +----------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_bridge.c + :doc: bridge chain format selection + Bridge Connector Helper ----------------------- diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst index fa69a5450f96..0dd440a14946 100644 --- a/Documentation/gpu/drm-kms.rst +++ b/Documentation/gpu/drm-kms.rst @@ -287,6 +287,12 @@ structure, ordering of committing state changes to hardware is sequenced using Read on in this chapter, and also in :ref:`drm_atomic_helper` for more detailed coverage of specific topics. +Atomic State Lifetime +--------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_atomic.c + :doc: state lifetime + Handling Driver Private State ----------------------------- @@ -604,6 +610,12 @@ Color Management Properties .. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c :doc: overview +Color Format Property +--------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_connector.c + :doc: Color format + Tile Group Property ------------------- diff --git a/Documentation/gpu/intel-display/dp-link-training.rst b/Documentation/gpu/intel-display/dp-link-training.rst new file mode 100644 index 000000000000..d0bde965021d --- /dev/null +++ b/Documentation/gpu/intel-display/dp-link-training.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: MIT +.. Copyright © 2026 Intel Corporation + +DisplayPort Link Training +========================= + +.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp_link_training.c + :doc: DisplayPort link training diff --git a/Documentation/gpu/intel-display/index.rst b/Documentation/gpu/intel-display/index.rst index 01c3d1e576b7..6fa929d82c38 100644 --- a/Documentation/gpu/intel-display/index.rst +++ b/Documentation/gpu/intel-display/index.rst @@ -38,6 +38,7 @@ driver. The display driver isn't an independent driver in that sense. fifo-underrun frontbuffer hotplug + dp-link-training plane psr snps-phy diff --git a/Documentation/gpu/nova/core/fsp.rst b/Documentation/gpu/nova/core/fsp.rst new file mode 100644 index 000000000000..52d618d22bb8 --- /dev/null +++ b/Documentation/gpu/nova/core/fsp.rst @@ -0,0 +1,142 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================== +FSP (Foundation Security Processor) and Secure Boot +=================================================== +This document describes the role of the FSP in the GPU boot sequence on +Hopper and Blackwell GPUs, and how it differs from the earlier Ampere boot +flow. It also provides a brief overview of the PRC (Product Reconfiguration +Control) protocol used to query device configuration through FSP. As with +other documents in this directory, the information is subject to change and +is intended to help developers understand the corresponding kernel code. + +What is FSP? +============ +The Foundation Security Processor (FSP) is the GPU's Internal Root of Trust +(IROT). It is a dedicated security processor that boots from immutable ROM +(Boot ROM) inside the GPU and is responsible for establishing the Chain of +Trust before any other firmware is allowed to run. + +FSP runs independently of the host CPU and starts executing as soon as the +GPU is powered on. By the time the nova-core driver is loaded, FSP has +already completed its own secure boot and is ready to accept commands from +the driver. + +Simplified boot flow (Hopper/Blackwell) +======================================= +Starting with Hopper, the boot flow is significantly simplified compared to +earlier GPU generations like Ampere. + +On an **Ampere** GPU, the boot verification chain involves multiple Falcon +engines and multiple ucode stages (see falcon.rst for details):: + + Hardware BROM (SEC2) + -> HS Booter (SEC2) + -> LS GSP-RM (GSP) + +The driver must extract ucode from VBIOS, manage SEC2 and GSP, and +orchestrate the Booter to load GSP-RM. This involves FWSEC-FRTS, devinit, +and the Booter stages. + +On **Hopper/Blackwell** GPUs, FSP replaces this multi-stage process with a +single message-driven interface:: + + FSP (hardware root of trust, boots from ROM) + -> FMC (Falcon Microcontroller, verified by FSP) + -> GSP-RM (verified and loaded by FMC) + +The driver only needs to: + +1. Wait for FSP to complete its own secure boot (polling a scratch register). +2. Send a Chain of Trust (COT) message to FSP with the FMC firmware location, + cryptographic signatures, and GSP boot parameters. +3. FSP authenticates the FMC firmware and boots it, FMC in turn loads GSP-RM. + +There is no SEC2 involvement, no Booter ucode, and no FWSEC-FRTS stage. The +entire secure boot is driven by a single FSP message exchange. + +Chain of Trust (COT) protocol +============================= +The Chain of Trust establishes a cryptographically enforced boot sequence, +ensuring the GPU reaches a known, trusted state. + +The driver communicates with FSP using a message queue (Falcon MSGQ +interface). Each message consists of an MCTP (Management Component Transport +Protocol) transport header and an NVDM (NVIDIA Vendor Defined Message) header, +followed by a protocol-specific payload. + +For Chain of Trust, the payload includes: + +- The system memory address of the FMC firmware image. +- Cryptographic material: a SHA-384 hash, RSA-3K public key, and RSA-3K + signature extracted from the FMC ELF firmware. +- FRTS (Firmware Runtime Services) region information (vidmem offset and size). +- The system memory address of the GSP boot arguments structure. + +FSP verifies the signature against the provided public key and hash, and if +verification succeeds, boots the FMC. The FMC then authenticates and launches +GSP-RM. + +The message flow is:: + + nova-core FSP + | | + | 1. Poll scratch register | + | (wait for FSP boot complete) | + | | + | 2. COT message ------------> | + | (FMC addr, signatures, | + | boot params) | + | | + | |--- Verify FMC signature + | |--- Boot FMC + | |--- FMC loads GSP-RM + | | + | 3. COT response <------------ | + | (success/error) | + | | + +FSP message format +================== +All FSP messages share a common header format consisting of two 32-bit words: + +**MCTP header** (Management Component Transport Protocol): + +- Bit 31: SOM (Start of Message) +- Bit 30: EOM (End of Message) +- Bits 29:28: Packet sequence number +- Bits 23:16: Source Endpoint ID + +**NVDM header** (NVIDIA Vendor Defined Message): + +- Bits 6:0: MCTP message type (0x7e = vendor-defined PCI) +- Bits 23:8: PCI vendor ID (0x10de = NVIDIA) +- Bits 31:24: NVDM type (0x14 = COT, 0x13 = PRC, 0x15 = FSP response) + +PRC (Product Reconfiguration Control) protocol +=============================================== +PRC is an API system exposed through FSP's Management Partition that allows +querying and modifying device configuration without firmware updates. + +Configuration parameters are called "knobs". Each knob has a unique object +ID and controls a specific device behavior. Examples include vGPU mode, ECC +enable, confidential computing mode, and NVLINK configuration. + +Each knob has two values: + +- **Active**: the currently effective value for this boot cycle. +- **Persistent**: the value stored in InfoROM, applied on subsequent boots. + +The nova-core driver uses PRC to read the vGPU mode knob (object ID 0x29) +during early boot, before firmware loading, to determine whether the GPU +should operate in vGPU mode. + +The PRC message format follows the same MCTP/NVDM header structure as COT, +with NVDM type 0x13. The payload contains: + +- A sub-command (e.g., 0x0c for read). +- Flags indicating which value to read (bit 0 = persistent, bit 1 = active). +- The knob object ID. + +The response includes the common FSP response header (with error status) +followed by the knob's 16-bit state value. diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index e39cb3163581..1783513cbd05 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -30,5 +30,6 @@ vGPU manager VFIO driver and the nova-drm driver. core/todo core/vbios core/devinit + core/fsp core/fwsec core/falcon diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst index cdddf8db35f5..14cf37590fc7 100644 --- a/Documentation/gpu/todo.rst +++ b/Documentation/gpu/todo.rst @@ -55,7 +55,7 @@ There are still drivers that use drm_simple_display_pipe. The task here is to convert them to use regular atomic helpers. Search for a driver that calls drm_simple_display_pipe_init() and inline all helpers from drm_simple_kms_helper.c into the driver, such that no simple-KMS interfaces are required. Please also -rename all inlined fucntions according to driver conventions. +rename all inlined functions according to driver conventions. Contact: Thomas Zimmermann, respective driver maintainer @@ -896,10 +896,11 @@ complection of submission. One minor feature still missing is a generic DRM IOCTL to query the error status of binary and timeline drm_syncobj. -This should probably be improved by implementing the necessary kernel interface -and adding support for that in the userspace stack. +This was already improved by implementing the necessary kernel interface +(patches are on the dri-devel mailing list) but adding support for that in the +userspace stack is still missing. -Contact: Christian König +Contact: Christian König, Michel Dänzer Level: Starter @@ -948,6 +949,47 @@ Contact: Philipp Stanner Level: Intermediate +Replace the lockless queue with a locked list +--------------------------------------------- + +drm_sched is the only user in the entire kernel of a special lockless queue, the +spsc_queue. This queue utilizes: + +- preempt_disable() +- atomic instructions +- memory barriers +- ACCESS_ONCE() + +whereas a conventional spinlock utilizes: + +- preempt_disable() +- 1 atomic instruction for taking / releasing the lock +- memory barriers + +Moreover, drm_sched_entity_push_job(), the only user of spsc_queue_push(), has +to take a lock in some situations anyways and calls to it are often serialized +with a driver lock. + +It is, thus, highly questionable whether the lockless queue grants any advantage +at all. Considering that its internals are not well documented and its correctness +is not formally proven, it seems desirable to replace the queue with a mere list +or hlist that is protected by a spinlock. + +Tasks: + +- Replace the spsc_queue in drm/sched (and those who might access the scheduler's + internal queue) with a spinlock + (h)list. +- Ideally, check with some micro benchmarks and real world tests (preferably + with amdgpu) for relevant performance regressions. +- Remove the spsc_queue from the kernel altogether. + +Contact: + +- Philipp Stanner +- Christian König + +Level: Beginner + Outside DRM =========== diff --git a/Documentation/hwmon/adm1275.rst b/Documentation/hwmon/adm1275.rst index cf923f20fa52..d8495be313b8 100644 --- a/Documentation/hwmon/adm1275.rst +++ b/Documentation/hwmon/adm1275.rst @@ -67,6 +67,30 @@ Supported chips: Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/ADM1293_1294.pdf + * ROHM Semiconductor BD12780 + + Prefix: 'bd12780' + + Addresses scanned: - + + Datasheet: https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780muv-lb-e.pdf + + * ROHM Semiconductor BD12780A + + Prefix: 'bd12780' + + Addresses scanned: - + + Datasheet: https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780amuv-lb-e.pdf + + * ROHM Semiconductor BD12790 + + Prefix: 'bd12790' + + Addresses scanned: - + + Datasheet: - + * Silergy SQ24905C Prefix: 'mc09c' diff --git a/Documentation/hwmon/chipcap2.rst b/Documentation/hwmon/chipcap2.rst index dc165becc64c..c38d87b91b69 100644 --- a/Documentation/hwmon/chipcap2.rst +++ b/Documentation/hwmon/chipcap2.rst @@ -70,4 +70,6 @@ humidity1_min_hyst: RW humidity low hystersis humidity1_max_hyst: RW humidity high hystersis humidity1_min_alarm: RO humidity low alarm indicator humidity1_max_alarm: RO humidity high alarm indicator +humidity1_label: RO descriptive name for the sensor +temp1_label: RO descriptive name for the sensor =============================== ======= ======================================== diff --git a/Documentation/hwmon/coretemp.rst b/Documentation/hwmon/coretemp.rst index f63b21f24d42..349301683381 100644 --- a/Documentation/hwmon/coretemp.rst +++ b/Documentation/hwmon/coretemp.rst @@ -44,9 +44,9 @@ Temperature known as TjMax is the maximum junction temperature of processor, which depends on the CPU model. See table below. At this temperature, protection mechanism will perform actions to forcibly cool down the processor. Alarm may be raised, if the temperature grows enough (more than TjMax) to trigger -the Out-Of-Spec bit. Following table summarizes the exported sysfs files: +the Out-Of-Spec bit. The following table summarizes the exported sysfs files: -All Sysfs entries are named with their core_id (represented here by 'X'). +All sysfs entries are named with their core_id (represented here by 'X'). ================= ======================================================== tempX_input Core temperature (in millidegrees Celsius). diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index 4aa910569c31..226789376217 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -225,6 +225,7 @@ Hardware Monitoring Kernel Drivers pmbus powerz powr1220 + prom21-xhci pt5161l pxe1610 pwm-fan @@ -252,6 +253,7 @@ Hardware Monitoring Kernel Drivers smsc47m1 sparx5-temp spd5118 + sq24860 stpddc60 surface_fan sy7636a-hwmon @@ -271,6 +273,7 @@ Hardware Monitoring Kernel Drivers tps53679 tps546d24 tsc1641 + tvs-mpfs twl4030-madc-hwmon ucd9000 ucd9200 diff --git a/Documentation/hwmon/ltc4283.rst b/Documentation/hwmon/ltc4283.rst index a650c595bc8f..99b1ee05f629 100644 --- a/Documentation/hwmon/ltc4283.rst +++ b/Documentation/hwmon/ltc4283.rst @@ -256,7 +256,7 @@ these logs can be cleared by writing in the proper reset_history attribute. ``/sys/kernel/debug/i2c/i2c-[X]/[X]-addr/`` contains the following attributes: -======================= ========================================== +============================== ========================================================== power1_failed_fault_log Set to 1 by a power1 fault occurring. power1_good_input_fault_log Set to 1 by a power1 good input fault occurring at PGIO3. in11_fet_short_fault_log Set to 1 when a FET-short fault occurs. @@ -264,4 +264,4 @@ in11_fet_bad_fault_log Set to 1 when a FET-BAD fault occurs. in0_lcrit_fault_log Set to 1 by a VIN undervoltage fault occurring. in0_crit_fault_log Set to 1 by a VIN overvoltage fault occurring. curr1_crit_fault_log Set to 1 by an overcurrent fault occurring. -======================= ========================================== +============================== ========================================================== diff --git a/Documentation/hwmon/prom21-xhci.rst b/Documentation/hwmon/prom21-xhci.rst new file mode 100644 index 000000000000..7984fb187bd8 --- /dev/null +++ b/Documentation/hwmon/prom21-xhci.rst @@ -0,0 +1,101 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver prom21-xhci +========================= + +Supported chips: + + * AMD Promontory 21 (PROM21) xHCI USB host controller + + Prefix: 'prom21_xhci' + + PCI IDs: 1022:43fc, 1022:43fd + +Author: + + - Jihong Min + +Description +----------- + +This driver exposes the temperature sensor in AMD PROM21 xHCI controllers. + +The driver binds to an auxiliary device created by the xHCI PCI driver for +supported controllers. The sensor value is accessed through a vendor-specific +index/data register pair in the controller's PCI MMIO BAR. +The auxiliary device is created by the ``xhci-pci-prom21`` PCI glue driver. +USB host operation is otherwise delegated to the common ``xhci-pci`` code. + +PROM21 is an AMD chipset IP used in single-chip or daisy-chained configurations +to build AMD 6xx/8xx series chipsets. Since the xHCI controllers are +integrated in PROM21, this temperature can also be used as a monitor for a +temperature close to the AMD chipset temperature. + +Register access +--------------- + +The temperature value is read through a vendor-specific index/data register +pair in the xHCI PCI MMIO BAR. The driver uses the following byte offsets from +the MMIO BAR base: + +======================= ===================================================== +0x3000 Vendor index register +0x3008 Vendor data register +======================= ===================================================== + +The driver saves the current vendor index register value, writes the +temperature selector ``0x0001e520`` to the vendor index register, reads the +vendor data register, and restores the previous vendor index value before +returning. The raw temperature value is the low 8 bits of the vendor data +register value. + +The hwmon core serializes this driver's callbacks, and the driver restores the +previous index value after each read. This does not provide synchronization +with firmware, SMM, ACPI AML, or any other user outside this driver. + +No public AMD reference is available for the register pair or the raw value. +The register pair was identified on an X870E system with two PROM21 xHCI +controllers. One controller was passed through to a Windows VM, and the same +controller's PCI MMIO BAR was observed from the Linux host while HWiNFO64 was +reporting the PROM21 xHCI temperature. In the test environment, the reported +temperature was very stable at idle and the displayed sensor resolution was +low, which made it possible to look for a consistently repeating MMIO response +for the same reported temperature. During observation, offset 0x3000 repeatedly +contained selector ``0x0001e520``. Writing the same selector to offset 0x3000 +from Linux and then reading offset 0x3008 reproduced the same raw value, so the +offsets are treated as a vendor index/data register pair. + +The conversion formula was empirically inferred by matching observed raw +8-bit values against HWiNFO64's reported PROM21 xHCI temperature for the same +controller. The observed mapping is: + + temp[C] = raw * 0.9066 - 78.624 + +Runtime PM +---------- + +The driver does not wake the xHCI PCI device for hwmon reads. It reads the +temperature only when the parent device is already active. A read from a +suspended device returns ``-ENODATA``. After a successful read, the driver +drops its active-only runtime PM reference and lets the PM core re-evaluate the +idle state. + +Sysfs entries +------------- + +======================= ===================================================== +temp1_input Temperature in millidegrees Celsius +======================= ===================================================== + +The hwmon device name is ``prom21_xhci``. The sysfs path depends on the hwmon +device number assigned by the kernel. Userspace can locate the device by +matching the ``name`` attribute: + +.. code-block:: sh + + for hwmon in /sys/class/hwmon/hwmon*; do + [ "$(cat "$hwmon/name")" = "prom21_xhci" ] || continue + cat "$hwmon/temp1_input" + done + +If the raw register value is invalid, ``temp1_input`` returns ``-ENODATA``. diff --git a/Documentation/hwmon/sq24860.rst b/Documentation/hwmon/sq24860.rst new file mode 100644 index 000000000000..f0182b955d8a --- /dev/null +++ b/Documentation/hwmon/sq24860.rst @@ -0,0 +1,96 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver sq24860 +===================== + +Supported chips: + + * Silergy SQ24860 + + Prefix: 'sq24860' + +Author: + + Ziming Zhu + +Description +------------ + +This driver implements support for the Silergy SQ24860 eFuse. The device is an +integrated circuit protection and power management device with a PMBus +interface. + +The device supports direct format for reading input voltage, output voltage, +auxiliary voltage, input current, input power, and temperature. + +The current and power measurement scale depends on the resistor connected +between the IMON pin and ground. The resistor value can be configured with the +``silergy,rimon-micro-ohms`` device tree property. See +``Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml`` for details. + +Due to the specificities of the chip, all history reset attributes are tied +together. Resetting the history of one sensor resets the history of all sensors. + +Sysfs entries +------------- + +The following attributes are supported. Limits are read-write; all other +attributes are read-only. + +======================= ====================================================== +in1_label "vin" +in1_input Measured input voltage. +in1_average Average measured input voltage. +in1_min Minimum input voltage limit. +in1_lcrit Critical low input voltage limit. +in1_max Maximum input voltage limit. +in1_crit Critical high input voltage limit. +in1_min_alarm Input voltage low warning alarm. +in1_lcrit_alarm Input voltage low fault alarm. +in1_max_alarm Input voltage high warning alarm. +in1_crit_alarm Input voltage high fault alarm. +in1_highest Historical maximum input voltage. +in1_lowest Historical minimum input voltage. +in1_reset_history Write any value to reset history. + +in2_label "vmon" +in2_input Measured auxiliary input voltage. + +in3_label "vout1" +in3_input Measured output voltage. +in3_average Average measured output voltage. +in3_min Minimum output voltage limit. +in3_min_alarm Output voltage low alarm. +in3_lowest Historical minimum output voltage. +in3_reset_history Write any value to reset history. + +curr1_label "iin" +curr1_input Measured input current. +curr1_average Average measured input current. +curr1_max Maximum input current warning limit. +curr1_crit Critical input over-current fault limit. +curr1_max_alarm Input current warning alarm. +curr1_crit_alarm Input over-current fault alarm. +curr1_highest Historical maximum input current. +curr1_reset_history Write any value to reset history. + +power1_label "pin" +power1_input Measured input power. +power1_average Average measured input power. +power1_max Maximum input power warning limit. +power1_alarm Input power warning alarm. +power1_input_highest Historical maximum input power. +power1_reset_history Write any value to reset history. + +temp1_input Measured temperature. +temp1_average Average measured temperature. +temp1_max Maximum temperature warning limit. +temp1_crit Critical temperature fault limit. +temp1_max_alarm Temperature warning alarm. +temp1_crit_alarm Temperature fault alarm. +temp1_highest Historical maximum temperature. +temp1_reset_history Write any value to reset history. + +samples Number of samples used for average values. +======================= ====================================================== + diff --git a/Documentation/hwmon/tvs-mpfs.rst b/Documentation/hwmon/tvs-mpfs.rst new file mode 100644 index 000000000000..1035812f363a --- /dev/null +++ b/Documentation/hwmon/tvs-mpfs.rst @@ -0,0 +1,53 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver tvs-mpfs +====================== + +Supported chips: + + * PolarFire SoC + +Authors: + + - Conor Dooley + - Lars Randers + +Description +----------- + +This driver implements support for the temperature and voltage sensors on +PolarFire SoC. The temperature reports how hot the die is, and the voltages are +the SoC's 1.05, 1.8 and 2.5 volt rails respectively. + + +Usage Notes +----------- + +update_interval has a permitted range of 0 to 8 milliseconds. + +Temperatures are read in millidegrees Celsius, but the hardware measures in +degrees Kelvin, storing the result as 11.4 fixed point data, for a maximum +value of 2047.9375 degrees Kelvin. + +Voltages are read in millivolts. The hardware measures in millivolts, storing +the value as 12.3 fixed point data, for a maximum of 4095.875 millivolts. +The minimum value reportable by the driver is 0 volts, although the hardware +is capable of measuring negative values. + +Sysfs entries +------------- + +The following attributes are supported. update_interval is read-write, as are +the enables. All other attributes are read only. + +======================= ==================================================== +temp1_label Fixed name for channel. +temp1_input Measured temperature for channel. +temp1_enable Enable/disable for channel. + +in[0-2]_label Fixed name for channel. +in[0-2]_input Measured voltage for channel. +in[0-2]_enable Enable/disable for channel. + +update_interval The interval at which the chip will update readings. +======================= ==================================================== diff --git a/Documentation/iio/ad4691.rst b/Documentation/iio/ad4691.rst new file mode 100644 index 000000000000..e45733341a4b --- /dev/null +++ b/Documentation/iio/ad4691.rst @@ -0,0 +1,227 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============= +AD4691 driver +============= + +ADC driver for Analog Devices Inc. AD4691 family of multichannel SAR ADCs. +The module name is ``ad4691``. + + +Supported devices +================= + +The following chips are supported by this driver: + +* `AD4691 `_ — 16-channel, 500 kSPS +* `AD4692 `_ — 16-channel, 1 MSPS +* `AD4693 `_ — 8-channel, 500 kSPS +* `AD4694 `_ — 8-channel, 1 MSPS + + +IIO channels +============ + +Each physical ADC input maps to one IIO voltage channel. The AD4691 and AD4692 +expose 16 channels (``voltage0`` through ``voltage15``); the AD4693 and AD4694 +expose 8 channels (``voltage0`` through ``voltage7``). + +All channels share a common scale (``in_voltage_scale``), derived from the +reference voltage. Each channel exposes: + +* ``in_voltageN_raw`` — single-shot ADC result + +The following attributes are shared across all channels: + +* ``in_voltage_sampling_frequency`` — effective output rate, defined as the + internal oscillator frequency divided by the oversampling ratio. Writing this + attribute selects the nearest achievable rate for the current OSR; the value + read back reflects the actual rate after snapping to the closest valid + oscillator entry. +* ``in_voltage_sampling_frequency_available`` — list of achievable effective + rates for the current oversampling ratio. The list updates dynamically when + the oversampling ratio changes. + +The following attributes are shared across all channels and only available in +CNV Burst Mode: + +* ``in_voltage_oversampling_ratio`` — hardware oversampling depth applied to + all channels; see `Oversampling`_ below. +* ``in_voltage_oversampling_ratio_available`` — valid ratios: 1, 2, 4, 8, 16, + 32. + + +Operating modes +=============== + +The driver supports two operating modes, selected automatically from the +device tree at probe time. + +Manual Mode +----------- + +Selected when no ``pwms`` property is present in the device tree. The CNV pin +is tied to the SPI chip-select: every CS assertion triggers a conversion and +returns the previous result. A user-defined IIO trigger (e.g. hrtimer trigger) +drives the buffer. + +Oversampling is not supported in Manual Mode. + +CNV Burst Mode +-------------- + +Selected when a ``pwms`` property is present in the device tree. A PWM drives +the CNV pin at the configured conversion rate. A GP pin wired to the SoC and +declared in the device tree signals DATA_READY at the end of each burst, +triggering a readout of all active channel results into the IIO buffer. + +The buffer output rate is controlled by the ``sampling_frequency`` attribute +on the IIO buffer. In practice the PWM rate should be set low enough to allow +the SPI readout to complete before the next conversion burst begins. + +Autonomous Mode (idle / single-shot) +------------------------------------- + +When the IIO buffer is disabled, ``in_voltageN_raw`` reads perform a single +conversion on the requested channel using the internal oscillator. The +oscillator is started and stopped around each read to save power. + + +Oversampling +============ + +In CNV Burst Mode a shared hardware accumulator averages a configurable number +of successive conversions across all active channels. The result is always a +16-bit mean, so the buffer data type (shown in ``buffer0/in_voltageN_type``) +is unaffected by the oversampling ratio. Valid ratios are 1, 2, 4, 8, 16 and +32; the default is 1 (no averaging). Oversampling is not supported in Manual +Mode. + +.. code-block:: bash + + # Set oversampling ratio to 16 (shared across all channels) + echo 16 > /sys/bus/iio/devices/iio:device0/in_voltage_oversampling_ratio + + # Read the resulting effective sampling frequency + cat /sys/bus/iio/devices/iio:device0/in_voltage_sampling_frequency + +Writing ``in_voltage_oversampling_ratio`` stores the new shared depth and snaps +the internal oscillator to the largest valid table entry that is both less than +or equal to ``old_effective_rate × new_osr`` and evenly divisible by +``new_osr``. This preserves an integer read-back of +``in_voltage_sampling_frequency`` after the change and keeps the oscillator as +close as possible to the previous effective rate. + + +Reference voltage +================= + +The driver supports two reference configurations, mutually exclusive: + +* **External reference** (``ref-supply``): a voltage between 2.4 V and 5.25 V + supplied externally. +* **Buffered internal reference** (``refin-supply``): an internal reference + buffer is enabled by the driver. + +Exactly one of ``ref-supply`` or ``refin-supply`` must be present in the +device tree. The reference voltage determines the full-scale range reported +via ``in_voltage_scale``. + + +LDO supply +========== + +The chip contains an internal LDO that powers part of the analog front-end. +The supply configuration is mutually exclusive: + +* **External VDD** (``vdd-supply``): an external 1.8 V supply is used directly; + the internal LDO is disabled. +* **Internal LDO** (``ldo-in-supply``): the internal LDO is enabled and fed + from the ``ldo-in`` regulator. Use this when no external 1.8 V VDD is present. + +Exactly one of ``vdd-supply`` or ``ldo-in-supply`` must be provided. + + +Reset +===== + +The driver supports two reset mechanisms: + +* **Hardware reset** (``reset-gpios`` in device tree): the GPIO line is + asserted then deasserted at probe; the driver waits 300 µs for the chip + to complete its internal reset sequence before accepting SPI commands. +* **Software reset** (fallback when ``reset-gpios`` is absent): written + automatically at probe. + + +GP pins and interrupts +====================== + +The chip exposes up to four general-purpose (GP) pins. In CNV Burst Mode +(non-offload), one GP pin must be wired to an interrupt-capable SoC input and +declared in the device tree using the ``interrupts`` and ``interrupt-names`` +properties. The ``interrupt-names`` value identifies which GP pin is used +(``"gp0"`` through ``"gp3"``). + +Example device tree fragment:: + + adc@0 { + compatible = "adi,ad4692"; + ... + interrupt-parent = <&gpio0>; + interrupts = <17 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "gp0"; + }; + + +SPI offload support +=================== + +When a SPI offload engine (e.g. the AXI SPI Engine) is present, the driver +uses DMA-backed transfers for CPU-independent, high-throughput data capture. +SPI offload is detected automatically at probe; if no offload hardware is +available the driver falls back to the software triggered-buffer path. + +Two SPI offload sub-modes exist: + +CNV Burst offload +----------------- + +Used when a ``pwms`` property is present and SPI offload is available. The PWM +drives CNV at the configured rate; on DATA_READY the offload engine reads all +active channel results and streams them directly to the IIO DMA buffer with no +CPU involvement. The GP pin used as DATA_READY trigger is supplied by the +trigger-source consumer at buffer enable time; no ``interrupt-names`` entry is +required. + +Manual offload +-------------- + +Used when no ``pwms`` property is present and SPI offload is available. A +periodic SPI offload trigger controls the conversion rate and the offload engine +streams results directly to the IIO DMA buffer. + +The ``sampling_frequency`` attribute on the IIO buffer controls the trigger +rate (in Hz). The initial rate is 100 kHz. + +Oversampling is not supported in Manual Mode. + + +Buffer data format +================== + +The sample format in the IIO buffer depends on whether SPI offload is in use. + +Software triggered-buffer path (no SPI offload) +------------------------------------------------ + +Each active channel occupies one 16-bit big-endian slot (``storagebits=16``, +``endianness=be``). Active channels are packed densely in scan-index order, +followed by a 64-bit software timestamp appended by the IIO core. + +SPI offload path +---------------- + +Each active channel occupies one 16-bit CPU-native slot (``storagebits=16``, +``endianness=cpu``). The SPI offload engine streams 16-bit words directly from +the SPI Engine into the DMA buffer; no software timestamp is appended. diff --git a/Documentation/iio/adf41513.rst b/Documentation/iio/adf41513.rst new file mode 100644 index 000000000000..244453cce6f6 --- /dev/null +++ b/Documentation/iio/adf41513.rst @@ -0,0 +1,199 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ADF41513 driver +=============== + +This driver supports Analog Devices' ADF41513 and similar SPI PLL frequency +synthesizers. + +1. Supported devices +==================== + +* `ADF41510 `_ +* `ADF41513 `_ + +The ADF41513 is an ultralow noise frequency synthesizer that can be used to +implement local oscillators (LOs) as high as 26.5 GHz in the upconversion and +downconversion sections of wireless receivers and transmitters. The ADF41510 +is a similar device that supports frequencies up to 10 GHz. + +Both devices support integer-N and fractional-N operation modes, providing +excellent phase noise performance and flexible frequency generation +capabilities. + +Key Features: + +- **ADF41510**: 1 GHz to 10 GHz frequency range +- **ADF41513**: 1 GHz to 26.5 GHz frequency range +- Integer-N and fractional-N operation modes +- Ultra-low phase noise (-235 dBc/Hz integer-N, -231 dBc/Hz fractional-N) +- High maximum PFD frequency (250 MHz integer-N, 125 MHz fractional-N) +- 25-bit fixed modulus or 49-bit variable modulus fractional modes +- Programmable charge pump currents with 16x range +- Digital lock detect functionality +- Phase resync capability for consistent output phase + +2. Device attributes +==================== + +The ADF41513 driver provides the following IIO extended attributes for +frequency control and monitoring: + +Each IIO device has a device folder under ``/sys/bus/iio/devices/iio:deviceX``, +where X is the IIO index of the device. Under these folders reside a set of +device files that provide access to the synthesizer's functionality. + +The following table shows the ADF41513 related device files: + ++--------------------------------------+-------------------------------------------------------+ +| Device file | Description | ++======================================+=======================================================+ +| out_altvoltage0_frequency | RF output frequency control and readback (Hz) | ++--------------------------------------+-------------------------------------------------------+ +| out_altvoltage0_frequency_resolution | Target frequency resolution control (Hz) | ++--------------------------------------+-------------------------------------------------------+ +| out_altvoltage0_powerdown | Power management control (0=active, 1=power down) | ++--------------------------------------+-------------------------------------------------------+ +| out_altvoltage0_phase | RF output phase adjustment and readback (radians) | ++--------------------------------------+-------------------------------------------------------+ + +2.1 Frequency Control +---------------------- + +The ``out_altvoltage0_frequency`` attribute controls the RF output frequency +with sub-Hz precision. The driver automatically selects between integer-N and +fractional-N modes to achieve the requested frequency with the best possible +phase noise performance. + +**Supported ranges:** + +- **ADF41510**: 1,000,000,000 Hz to 10,000,000,000 Hz (1 GHz to 10 GHz) +- **ADF41513**: 1,000,000,000 Hz to 26,500,000,000 Hz (1 GHz to 26.5 GHz) + +The frequency is specified in Hz, for sub-Hz precision use decimal notation. +For example, 12.102 GHz would be written as "12102000000.000000". + +2.2 Frequency Resolution Control +-------------------------------- + +The ``out_altvoltage0_frequency_resolution`` attribute controls the target +frequency resolution that the driver attempts to achieve. This affects the +choice between integer-N and fractional-N modes, including fixed modulus +(25-bit) and variable modulus (49-bit) fractional-N modes: + +- **Integer-N**: Resolution = :math:`f_{PFD}` (same as PFD frequency) +- **Fixed modulus**: Resolution = :math:`f_{PFD} / 2^{25}` (~3 Hz with 100 MHz PFD) +- **Variable modulus**: Resolution = :math:`f_{PFD} / 2^{49}` (µHz resolution possible) + +Default resolution is 1 Hz (1,000,000 µHz). + +2.3 Phase adjustment +-------------------- + +The ``out_altvoltage0_phase`` attribute allows adjustment of the output phase +in radians. Setting this attribute enables phase adjustment. It can be set +from 0 to :math:`2\pi` radians. Reading this attribute returns the current +phase offset of the output signal. To create a consistent phase relationship +with the reference signal, the phase resync feature needs to be enabled by +setting a non-zero value to the ``adi,phase-resync-period-ns`` device property, +which triggers a phase resynchronization after locking is achieved. + +3. Operating modes +================== + +3.1 Integer-N Mode +------------------ + +When the requested frequency can be achieved as an integer multiple of the PFD +frequency (within the specified resolution tolerance), the driver automatically +selects integer-N mode for optimal phase noise performance. + +In integer-N mode: + +- Phase noise: -235 dBc/Hz normalized floor +- Frequency resolution: :math:`f_{PFD}` (same as PFD frequency) +- Maximum PFD frequency: 250 MHz +- Bleed current: Disabled + +3.2 Fractional-N Mode +--------------------- + +When sub-integer frequency steps are required, the driver automatically selects +fractional-N mode using either fixed or variable modulus. + +**Fixed Modulus (25-bit)**: + +- Used when variable modulus is not required +- Resolution: :math:`f_{PFD} / 2^{25}` +- Simpler implementation, faster settling + +**Variable Modulus (49-bit)**: + +- Used for maximum resolution requirements +- Resolution: :math:`f_{PFD} / 2^{49}` (theoretical) +- Exact frequency synthesis capability + +In fractional-N mode: + +- Phase noise: -231 dBc/Hz normalized floor +- Maximum PFD frequency: 125 MHz +- Bleed current: Automatically enabled and optimized +- Dithering: Enabled to reduce fractional spurs + +3.3 Automatic Mode Selection +---------------------------- + +The driver automatically selects the optimal operating mode based on: + +1. **Frequency accuracy requirements**: Determined by ``frequency_resolution`` setting +2. **Phase noise optimization**: Integer-N preferred when possible +3. **PFD frequency constraints**: Different limits for integer vs fractional modes +4. **Prescaler selection**: Automatic 4/5 vs 8/9 prescaler selection based on frequency + +4. Usage examples +================= + +4.1 Basic Frequency Setting +---------------------------- + +Set output frequency to 12.102 GHz: + +.. code-block:: bash + + root:/sys/bus/iio/devices/iio:device0> echo 12102000000 > out_altvoltage0_frequency + +Read current frequency: + +.. code-block:: bash + + root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency + 12101999999.582767 + +4.2 High Resolution Frequency Control +------------------------------------- + +Configure for sub-Hz resolution and set a precise frequency: + +.. code-block:: bash + + # Set resolution to 0.1 Hz (100,000 µHz) + root:/sys/bus/iio/devices/iio:device0> echo 0.1 > out_altvoltage0_frequency_resolution + + # Set frequency to 12.102 GHz (1 µHz precision) + root:/sys/bus/iio/devices/iio:device0> echo 12102000000 > out_altvoltage0_frequency + root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency + 12101999999.980131 + +4.3 Monitor Lock Status +----------------------- + +When lock detect GPIO is configured, check if PLL is locked: + +.. code-block:: bash + + # Read frequency - will return error if not locked + root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency + +If the PLL is not locked, the frequency read will return ``-EBUSY`` (Device or +resource busy). diff --git a/Documentation/iio/adxl313.rst b/Documentation/iio/adxl313.rst index 966e72c0109a..3641193bf647 100644 --- a/Documentation/iio/adxl313.rst +++ b/Documentation/iio/adxl313.rst @@ -11,7 +11,7 @@ This driver supports Analog Device's ADXL313 on SPI/I2C bus. * `ADXL313 `_ -The ADXL313is a low noise density, low power, 3-axis accelerometer with +The ADXL313 is a low noise density, low power, 3-axis accelerometer with selectable measurement ranges. The ADXL313 supports the ±0.5 g, ±1 g, ±2 g and ±4 g ranges. @@ -38,7 +38,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``. +---------------------------------------------------+----------------------------------------------------------+ | in_accel_x_raw | Raw X-axis accelerometer channel value. | +---------------------------------------------------+----------------------------------------------------------+ -| in_accel_y_calibbias | y-axis acceleration offset correction | +| in_accel_y_calibbias | Calibration offset for the Y-axis accelerometer channel. | +---------------------------------------------------+----------------------------------------------------------+ | in_accel_y_raw | Raw Y-axis accelerometer channel value. | +---------------------------------------------------+----------------------------------------------------------+ @@ -112,9 +112,9 @@ apply the following formula: Where _offset and _scale are device attributes. If no _offset attribute is present, simply assume its value is 0. -The ADXL313 driver offers data for a single types of channels, the table below -shows the measurement units for the processed value, which are defined by the -IIO framework: +The ADXL313 driver offers data for multiple channels of a single type. +The table below shows the measurement units for the processed value, +which are defined by the IIO framework: +-------------------------------------+---------------------------+ | Channel type | Measurement unit | diff --git a/Documentation/iio/adxl345.rst b/Documentation/iio/adxl345.rst index 978f746a8198..0aa33a852412 100644 --- a/Documentation/iio/adxl345.rst +++ b/Documentation/iio/adxl345.rst @@ -47,7 +47,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``. +-------------------------------------------+----------------------------------------------------------+ | in_accel_x_raw | Raw X-axis accelerometer channel value. | +-------------------------------------------+----------------------------------------------------------+ -| in_accel_y_calibbias | Y-axis acceleration offset correction | +| in_accel_y_calibbias | Calibration offset for the Y-axis accelerometer channel. | +-------------------------------------------+----------------------------------------------------------+ | in_accel_y_raw | Raw Y-axis accelerometer channel value. | +-------------------------------------------+----------------------------------------------------------+ diff --git a/Documentation/iio/adxl380.rst b/Documentation/iio/adxl380.rst index 61cafa2f98bf..654d4c0e84a1 100644 --- a/Documentation/iio/adxl380.rst +++ b/Documentation/iio/adxl380.rst @@ -51,7 +51,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``. +---------------------------------------------------+----------------------------------------------------------+ | in_accel_x_raw | Raw X-axis accelerometer channel value. | +---------------------------------------------------+----------------------------------------------------------+ -| in_accel_y_calibbias | y-axis acceleration offset correction | +| in_accel_y_calibbias | Calibration offset for the Y-axis accelerometer channel. | +---------------------------------------------------+----------------------------------------------------------+ | in_accel_y_raw | Raw Y-axis accelerometer channel value. | +---------------------------------------------------+----------------------------------------------------------+ diff --git a/Documentation/iio/iio_devbuf.rst b/Documentation/iio/iio_devbuf.rst index dca1f0200b0d..e91730fa3cea 100644 --- a/Documentation/iio/iio_devbuf.rst +++ b/Documentation/iio/iio_devbuf.rst @@ -83,9 +83,10 @@ and the relevant _type attributes to establish the data storage format. Read-only attribute containing the description of the scan element data storage within the buffer and hence the form in which it is read from userspace. Format -is [be|le]:[s|u]bits/storagebits[Xrepeat][>>shift], where: +is [be|le]:[f|s|u]bits/storagebits[Xrepeat][>>shift], where: - **be** or **le** specifies big or little-endian. +- **f** specifies if floating-point. - **s** or **u** specifies if signed (2's complement) or unsigned. - **bits** is the number of valid data bits. - **storagebits** is the number of bits (after padding) that it occupies in the diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst index ba3e609c6a13..b02b879b053a 100644 --- a/Documentation/iio/index.rst +++ b/Documentation/iio/index.rst @@ -23,6 +23,7 @@ Industrial I/O Kernel Drivers ad4000 ad4030 ad4062 + ad4691 ad4695 ad7191 ad7380 @@ -30,6 +31,7 @@ Industrial I/O Kernel Drivers ad7625 ad7944 ade9000 + adf41513 adis16475 adis16480 adis16550 diff --git a/Documentation/infiniband/user_mad.rst b/Documentation/infiniband/user_mad.rst index d88abfc0e370..cd66e7623d66 100644 --- a/Documentation/infiniband/user_mad.rst +++ b/Documentation/infiniband/user_mad.rst @@ -62,7 +62,7 @@ Receiving MADs struct ib_user_mad *mad; mad = malloc(sizeof *mad + 256); ret = read(fd, mad, sizeof *mad + 256); - if (ret == -ENOSPC)) { + if (ret == -ENOSPC) { length = mad.length; free(mad); mad = malloc(sizeof *mad + length); diff --git a/Documentation/input/userio.rst b/Documentation/input/userio.rst index f780c77931fe..7aaaa629bde0 100644 --- a/Documentation/input/userio.rst +++ b/Documentation/input/userio.rst @@ -5,7 +5,7 @@ The userio Protocol =================== -:Copyright: |copy| 2015 Stephen Chandler Paul +:Copyright: |copy| 2015 Lyude Paul Sponsored by Red Hat @@ -66,8 +66,27 @@ USERIO_CMD_SET_PORT_TYPE ~~~~~~~~~~~~~~~~~~~~~~~~ Sets the type of port we're emulating, where ``data`` is the port type being -set. Can be any of the macros from . For example: SERIO_8042 -would set the port type to be a normal PS/2 port. +set. Can be any of the serio type macros from . For example: +SERIO_8042 would set the port type to be a normal PS/2 port. + +USERIO_CMD_SET_PORT_PROTO +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sets the protocol of port we're emulating, where ``data`` is the protocol being +set. Can be any of the serio proto macros from . For example: +SERIO_IFORCE would set the port type to be an I-Force serial joystick. + +USERIO_CMD_SET_PORT_ID +~~~~~~~~~~~~~~~~~~~~~~ + +Sets the ``id`` value on the identification of port we're emulating, where +``data`` is the value being set. + +USERIO_CMD_SET_PORT_EXTRA +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sets the ``extra`` value on the identification of port we're emulating, where +``data`` is the value being set. USERIO_CMD_SEND_INTERRUPT ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/kbuild/kconfig.rst b/Documentation/kbuild/kconfig.rst index d213c4f599a4..e35dd1d5f9d3 100644 --- a/Documentation/kbuild/kconfig.rst +++ b/Documentation/kbuild/kconfig.rst @@ -59,6 +59,11 @@ Environment variables for ``*config``: This environment variable makes Kconfig warn about all unrecognized symbols in the config input. +``KCONFIG_WARN_CHANGED_INPUT`` + If set to a non-blank value, Kconfig prints optional warnings for + user-provided values that change after Kconfig resolves dependencies + or applies other constraints such as ranges. + ``KCONFIG_WERROR`` If set, Kconfig treats warnings as errors. diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst index c969c76ef7cb..23fd393c8193 100644 --- a/Documentation/kernel-hacking/locking.rst +++ b/Documentation/kernel-hacking/locking.rst @@ -471,7 +471,7 @@ to protect the cache and all the objects within it. Here's the code:: obj = __cache_find(id); if (obj) { ret = 0; - strcpy(name, obj->name); + strscpy(name, obj->name); } mutex_unlock(&cache_lock); return ret; @@ -553,7 +553,7 @@ which are taken away, and the ``+`` are lines which are added. obj = __cache_find(id); if (obj) { ret = 0; - strcpy(name, obj->name); + strscpy(name, obj->name); } - mutex_unlock(&cache_lock); + spin_unlock_irqrestore(&cache_lock, flags); @@ -676,7 +676,7 @@ Here is the code:: obj = __cache_find(id); - if (obj) { - ret = 0; - - strcpy(name, obj->name); + - strscpy(name, obj->name); - } + if (obj) + __object_get(obj); @@ -1317,7 +1317,7 @@ from user context, and can sleep. - put_user() -- kmalloc(GP_KERNEL) ` +- kmalloc(GFP_KERNEL) - mutex_lock_interruptible() and mutex_lock() diff --git a/Documentation/misc-devices/apds990x.rst b/Documentation/misc-devices/apds990x.rst deleted file mode 100644 index e2f75577f731..000000000000 --- a/Documentation/misc-devices/apds990x.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -====================== -Kernel driver apds990x -====================== - -Supported chips: -Avago APDS990X - -Data sheet: -Not freely available - -Author: -Samu Onkalo - -Description ------------ - -APDS990x is a combined ambient light and proximity sensor. ALS and proximity -functionality are highly connected. ALS measurement path must be running -while the proximity functionality is enabled. - -ALS produces raw measurement values for two channels: Clear channel -(infrared + visible light) and IR only. However, threshold comparisons happen -using clear channel only. Lux value and the threshold level on the HW -might vary quite much depending the spectrum of the light source. - -Driver makes necessary conversions to both directions so that user handles -only lux values. Lux value is calculated using information from the both -channels. HW threshold level is calculated from the given lux value to match -with current type of the lightning. Sometimes inaccuracy of the estimations -lead to false interrupt, but that doesn't harm. - -ALS contains 4 different gain steps. Driver automatically -selects suitable gain step. After each measurement, reliability of the results -is estimated and new measurement is triggered if necessary. - -Platform data can provide tuned values to the conversion formulas if -values are known. Otherwise plain sensor default values are used. - -Proximity side is little bit simpler. There is no need for complex conversions. -It produces directly usable values. - -Driver controls chip operational state using pm_runtime framework. -Voltage regulators are controlled based on chip operational state. - -SYSFS ------ - - -chip_id - RO - shows detected chip type and version - -power_state - RW - enable / disable chip. Uses counting logic - - 1 enables the chip - 0 disables the chip -lux0_input - RO - measured lux value - - sysfs_notify called when threshold interrupt occurs - -lux0_sensor_range - RO - lux0_input max value. - - Actually never reaches since sensor tends - to saturate much before that. Real max value varies depending - on the light spectrum etc. - -lux0_rate - RW - measurement rate in Hz - -lux0_rate_avail - RO - supported measurement rates - -lux0_calibscale - RW - calibration value. - - Set to neutral value by default. - Output results are multiplied with calibscale / calibscale_default - value. - -lux0_calibscale_default - RO - neutral calibration value - -lux0_thresh_above_value - RW - HI level threshold value. - - All results above the value - trigs an interrupt. 65535 (i.e. sensor_range) disables the above - interrupt. - -lux0_thresh_below_value - RW - LO level threshold value. - - All results below the value - trigs an interrupt. 0 disables the below interrupt. - -prox0_raw - RO - measured proximity value - - sysfs_notify called when threshold interrupt occurs - -prox0_sensor_range - RO - prox0_raw max value (1023) - -prox0_raw_en - RW - enable / disable proximity - uses counting logic - - - 1 enables the proximity - - 0 disables the proximity - -prox0_reporting_mode - RW - trigger / periodic. - - In "trigger" mode the driver tells two possible - values: 0 or prox0_sensor_range value. 0 means no proximity, - 1023 means proximity. This causes minimal number of interrupts. - In "periodic" mode the driver reports all values above - prox0_thresh_above. This causes more interrupts, but it can give - _rough_ estimate about the distance. - -prox0_reporting_mode_avail - RO - accepted values to prox0_reporting_mode (trigger, periodic) - -prox0_thresh_above_value - RW - threshold level which trigs proximity events. diff --git a/Documentation/misc-devices/index.rst b/Documentation/misc-devices/index.rst index 081e79415e38..f911edaecbfa 100644 --- a/Documentation/misc-devices/index.rst +++ b/Documentation/misc-devices/index.rst @@ -13,7 +13,6 @@ fit into other categories. ad525x_dpot amd-sbi - apds990x bh1770glc c2port dw-xdata-pcie diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst index 2da7ca0d3d17..457d6e8bc787 100644 --- a/Documentation/mm/damon/design.rst +++ b/Documentation/mm/damon/design.rst @@ -86,7 +86,7 @@ To know how user-space can do the configuration via :ref:`DAMON sysfs interface documentation. - .. _damon_design_vaddr_target_regions_construction: +.. _damon_design_vaddr_target_regions_construction: VMA-based Target Address Range Construction ------------------------------------------- @@ -686,9 +686,11 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to (1/10,000). - ``inactive_mem_bp``: Inactive to active + inactive (LRU) memory size ratio in bp (1/10,000). +- ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio + of a node in bp (1/10,000). -``nid`` is optionally required for only ``node_mem_used_bp``, -``node_mem_free_bp``, ``node_memcg_used_bp`` and ``node_memcg_free_bp`` to +``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``, +``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to point the specific NUMA node. ``path`` is optionally required for only ``node_memcg_used_bp`` and @@ -930,11 +932,11 @@ control parameters for the usage would also need to be optimized for the purpose. To support such cases, yet more DAMON API user kernel modules that provide more -simple and optimized user space interfaces are available. Currently, two -modules for proactive reclamation and LRU lists manipulation are provided. For -more detail, please read the usage documents for those -(:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` and -:doc:`/admin-guide/mm/damon/lru_sort`). +simple and optimized user space interfaces are available. Currently, three +modules for access monitoring statistics, proactive reclamation, and LRU lists +manipulation are provided. For more detail, please read the usage documents for +those (:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` +and :doc:`/admin-guide/mm/damon/lru_sort`). .. _damon_design_special_purpose_modules_exclusivity: diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst index 7d61b7a8b65b..54c461e7a143 100644 --- a/Documentation/mm/hmm.rst +++ b/Documentation/mm/hmm.rst @@ -191,7 +191,7 @@ like a CPU page fault. The usage pattern is:: mmap_read_unlock(mm); take_lock(driver->update); - if (mmu_interval_read_retry(&ni, range.notifier_seq) { + if (mmu_interval_read_retry(&ni, range.notifier_seq)) { release_lock(driver->update); goto again; } @@ -316,7 +316,7 @@ between device driver specific code and shared common code: system memory and device private memory. One of the first steps migrate_vma_setup() does is to invalidate other - device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and + device's MMUs with the ``mmu_notifier_invalidate_range_start()`` and ``mmu_notifier_invalidate_range_end()`` calls around the page table walks to fill in the ``args->src`` array with PFNs to be migrated. The ``invalidate_range_start()`` callback is passed a diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 7aa2a8886908..13a79f5d092c 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -7,6 +7,19 @@ of Linux. If you are looking for advice on simply allocating memory, see the :ref:`memory_allocation`. For controlling and tuning guides, see the :doc:`admin guide <../admin-guide/mm/index>`. +.. note:: + + Unfortunately, parts of this guide are still incomplete or missing. + While we appreciate contributions, documentation in this area is hard + to get right and requires a lot of attention to detail. New contributors + should reach out to the relevant maintainers early. + + This guide is expected to reflect reality, which requires contributors + to have a detailed understanding. Documentation generated with LLMs + by contributors unfamiliar with these details shifts the real work onto + reviewers, which is why such contributions will be rejected without + further comment. + .. toctree:: :maxdepth: 1 diff --git a/Documentation/mm/memfd_preservation.rst b/Documentation/mm/memfd_preservation.rst index a8a5b476afd3..c908a12dffa7 100644 --- a/Documentation/mm/memfd_preservation.rst +++ b/Documentation/mm/memfd_preservation.rst @@ -11,7 +11,7 @@ Memfd Preservation ABI ====================== .. kernel-doc:: include/linux/kho/abi/memfd.h - :doc: DOC: memfd Live Update ABI + :doc: memfd Live Update ABI .. kernel-doc:: include/linux/kho/abi/memfd.h :internal: diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index 6b12f3b007ec..383e59c42743 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -65,7 +65,14 @@ un-tracking state. Usage ===== -1) Build user-space helper:: +1) Build user-space helpers:: + +To filter page_owner output: + + cd tools/mm + make page_owner_filter + +To sort and analyze page_owner output: cd tools/mm make page_owner_sort @@ -74,7 +81,11 @@ Usage 3) Do the job that you want to debug. -4) Analyze information from page owner:: +4) (Optional) Filter page_owner output:: + + ./page_owner_filter -m handle -n 0,1,2 > filtered_page_owner.txt + +5) Analyze information from page owner:: cat /sys/kernel/debug/page_owner_stacks/show_stacks > stacks.txt cat stacks.txt @@ -263,3 +274,65 @@ STANDARD FORMAT SPECIFIERS f free whether the page has been released or not st stacktrace stack trace of the page allocation ator allocator memory allocator for pages + +Filtering page_owner output +============================ + +page_owner supports filtering output at the kernel level before reading, +which reduces the amount of data that needs to be processed in userspace. + +The page_owner_filter tool provides a convenient interface for this filtering +capability. It supports two types of filters: + +1. **print_mode filter**: Control what information is printed for each page + - ``stack``: Print full stack traces (default, compatible with existing usage) + - ``handle``: Print only stack handle numbers (much faster, smaller output) + - ``stack_handle``: Print both stack traces and handle numbers + + The ``handle`` mode uses numeric identifiers instead of full stack traces. + The mapping from handles to actual stack traces can be obtained via the + show_stacks_handles interface. + +2. **NUMA node filter**: Filter pages by NUMA node ID + - Supports single node: ``-n 0`` + - Multiple nodes: ``-n 0,1,2`` + - Ranges: ``-n 0-3`` + - Mixed format: ``-n 0,2-3,5`` + +Usage examples:: + + # Filter by print mode + ./page_owner_filter -m handle + ./page_owner_filter -m stack_handle + + # Filter by NUMA node + ./page_owner_filter -n 0 + ./page_owner_filter -n 0-3 + + # Combined filters + ./page_owner_filter -m stack -n 0,1,2 + ./page_owner_filter -m handle -n 0,2-3 + + # Save to file + ./page_owner_filter -m handle -o filtered_output.txt + +The handle mode is particularly useful for monitoring and performance-critical +scenarios as it dramatically reduces output size. Testing shows handle mode can +reduce output size by ~66% (84MB vs 244MB) and improve read performance by ~4.4x +compared to full stack output. + +The NUMA node filter is useful for NUMA-aware memory allocation analysis and debugging. + +Behind the scenes, page_owner_filter opens /sys/kernel/debug/page_owner and +writes filter commands before reading the filtered output. The filtering uses +per-file-descriptor state, allowing each open() to have independent filter settings. + +Each file descriptor maintains its own filter state, so you can have multiple +independent filtering operations running concurrently. For example, in different +terminals you can run different filters simultaneously:: + + # Terminal 1: Filter node 0 + ./page_owner_filter -n 0 > node0_output.txt + + # Terminal 2: Filter node 1 (runs concurrently) + ./page_owner_filter -n 1 > node1_output.txt diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index 042d64d72421..a7296f251799 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -724,7 +724,7 @@ the zap and the invocation of :c:func:`!free_pgtables`. Since it is assumed that all such steps have been taken, page table entries are cleared without page table locks (in the :c:func:`!pgd_clear`, :c:func:`!p4d_clear`, -:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions. +:c:func:`!pud_clear`, and :c:func:`!pmd_clear` functions). .. note:: It is possible for leaf page tables to be torn down independent of the page tables above it as is done by diff --git a/Documentation/netlink/specs/rt-addr.yaml b/Documentation/netlink/specs/rt-addr.yaml index 163a106c41bb..0ecbd24c890c 100644 --- a/Documentation/netlink/specs/rt-addr.yaml +++ b/Documentation/netlink/specs/rt-addr.yaml @@ -123,6 +123,9 @@ attribute-sets: - name: proto type: u8 + - + name: mc-users + type: u32 operations: @@ -176,6 +179,7 @@ operations: value: 58 attributes: &mcaddr-attrs - multicast + - mc-users - cacheinfo dump: request: diff --git a/Documentation/networking/netdevices.rst b/Documentation/networking/netdevices.rst index fde601acd1d2..d2a238f8cc8b 100644 --- a/Documentation/networking/netdevices.rst +++ b/Documentation/networking/netdevices.rst @@ -433,6 +433,8 @@ exceptions) notifiers run under the instance lock. Please extend this documentation whenever you make explicit assumption about lock being held from a notifier. +Drivers **must not** generate nested notifications of the ops-locked types. + NETDEV_INTERNAL symbol namespace ================================ diff --git a/Documentation/process/debugging/kgdb.rst b/Documentation/process/debugging/kgdb.rst index c4d0a9121d52..316b1d74e9c8 100644 --- a/Documentation/process/debugging/kgdb.rst +++ b/Documentation/process/debugging/kgdb.rst @@ -513,7 +513,7 @@ unregister all the kernel hook points. All kgdb I/O drivers can be reconfigured at run time, if ``CONFIG_SYSFS`` and ``CONFIG_MODULES`` are enabled, by echo'ing a new -config string to ``/sys/module//parameter/

::$DATA with FILE_DIRECTORY_FILE must fail with + * STATUS_NOT_A_DIRECTORY. + */ + if (stream_name || s_type == DATA_STREAM) { if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { if (s_type == DATA_STREAM) { rc = -EIO; @@ -3482,14 +3668,16 @@ int smb2_open(struct ksmbd_work *work) if (posix_acl_rc) ksmbd_debug(SMB, "inherit posix acl failed : %d\n", posix_acl_rc); - if (test_share_config_flag(work->tcon->share_conf, - KSMBD_SHARE_FLAG_ACL_XATTR)) { - rc = smb_inherit_dacl(conn, &path, sess->user->uid, - sess->user->gid); - } + rc = smb2_create_sd_buffer(work, req, &path); + if (rc && rc != -ENOENT) + goto err_out; - if (rc) { - rc = smb2_create_sd_buffer(work, req, &path); + if (rc == -ENOENT) { + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) { + rc = smb_inherit_dacl(conn, &path, sess->user->uid, + sess->user->gid); + } if (rc) { if (posix_acl_rc) ksmbd_vfs_set_init_posix_acl(idmap, @@ -3580,6 +3768,12 @@ int smb2_open(struct ksmbd_work *work) goto err_out; } + if (!stream_name && daccess & FILE_DELETE_LE && + ksmbd_has_stream_without_delete_share(fp)) { + rc = -EPERM; + goto err_out; + } + if (file_present || created) path_put(&path); @@ -3615,7 +3809,7 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "lease req for(%s) req oplock state 0x%x, lease state 0x%x\n", name, req_op_level, lc->req_state); - rc = find_same_lease_key(sess, fp->f_ci, lc); + rc = find_same_lease_key(conn, fp->f_ci, lc); if (rc) goto err_out1; } else if (open_flags == O_RDONLY && @@ -3631,8 +3825,10 @@ int smb2_open(struct ksmbd_work *work) goto err_out1; } - if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { + smb_break_all_levII_oplock_for_delete(work, fp); ksmbd_fd_set_delete_on_close(fp, file_info); + } if (need_truncate) { rc = smb2_create_truncate(&fp->filp->f_path); @@ -3649,7 +3845,6 @@ int smb2_open(struct ksmbd_work *work) rc = PTR_ERR(az_req); goto err_out1; } else if (az_req) { - loff_t alloc_size; int err; if (le16_to_cpu(az_req->ccontext.DataOffset) + @@ -3698,6 +3893,9 @@ int smb2_open(struct ksmbd_work *work) fp->create_time = ksmbd_UnixTimeToNT(stat.btime); else fp->create_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->change_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->allocation_size = S_ISDIR(stat.mode) ? 0 : + (alloc_size ?: stat.blocks << 9); if (req->FileAttributes || fp->f_ci->m_fattr == 0) fp->f_ci->m_fattr = cpu_to_le32(smb2_get_dos_mode(&stat, le32_to_cpu(req->FileAttributes))); @@ -3723,6 +3921,10 @@ int smb2_open(struct ksmbd_work *work) if (dh_info.type == DURABLE_REQ_V2) { memcpy(fp->create_guid, dh_info.CreateGuid, SMB2_CREATE_GUID_SIZE); + if (dh_info.app_instance_id) + memcpy(fp->app_instance_id, + dh_info.AppInstanceId, + SMB2_CREATE_GUID_SIZE); if (dh_info.timeout) fp->durable_timeout = min_t(unsigned int, dh_info.timeout, @@ -3743,10 +3945,20 @@ int smb2_open(struct ksmbd_work *work) rsp->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); rsp->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - rsp->ChangeTime = cpu_to_le64(time); - rsp->AllocationSize = S_ISDIR(stat.mode) ? 0 : - cpu_to_le64(stat.blocks << 9); + rsp->ChangeTime = cpu_to_le64(fp->change_time); + /* + * The cached allocation size hides filesystem rounding for the + * requested allocation, but it can go stale when the file grows past + * it via writes (e.g. across a durable reconnect). Refresh it once the + * file exceeds the cached value, rounding the end of file up to the + * volume allocation unit (the filesystem block size, matching the + * SectorsPerAllocationUnit/BytesPerSector ksmbd advertises) rather than + * using the raw on-disk block count, which can include filesystem + * preallocation and metadata rounding. + */ + if (!S_ISDIR(stat.mode) && stat.size > fp->allocation_size) + fp->allocation_size = round_up(stat.size, stat.blksize); + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); rsp->EndofFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); rsp->FileAttributes = fp->f_ci->m_fattr; @@ -4519,11 +4731,28 @@ int smb2_query_dir(struct ksmbd_work *work) unsigned char srch_flag; int buffer_sz; struct smb2_query_dir_private query_dir_private = {NULL, }; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 query directory request\n"); WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + if (ksmbd_override_fsids(work)) { rsp->hdr.Status = STATUS_NO_MEMORY; smb2_set_err_rsp(work); @@ -4536,7 +4765,7 @@ int smb2_query_dir(struct ksmbd_work *work) goto err_out2; } - dir_fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + dir_fp = ksmbd_lookup_fd_slow(work, id, pid); if (!dir_fp) { rc = -EBADF; goto err_out2; @@ -5013,8 +5242,7 @@ static int get_file_basic_info(struct smb2_query_info_rsp *rsp, basic_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); basic_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - basic_info->ChangeTime = cpu_to_le64(time); + basic_info->ChangeTime = cpu_to_le64(fp->change_time); basic_info->Attributes = fp->f_ci->m_fattr; basic_info->Pad = 0; rsp->OutputBufferLength = @@ -5039,7 +5267,7 @@ static int get_file_standard_info(struct smb2_query_info_rsp *rsp, delete_pending = ksmbd_inode_pending_delete(fp); if (ksmbd_stream_fd(fp) == false) { - sinfo->AllocationSize = cpu_to_le64(stat.blocks << 9); + sinfo->AllocationSize = cpu_to_le64(fp->allocation_size); sinfo->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { sinfo->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5116,13 +5344,11 @@ static int get_file_all_info(struct ksmbd_work *work, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->Attributes = fp->f_ci->m_fattr; file_info->Pad1 = 0; if (ksmbd_stream_fd(fp) == false) { - file_info->AllocationSize = - cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5326,11 +5552,10 @@ static int get_file_network_open_info(struct smb2_query_info_rsp *rsp, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->Attributes = fp->f_ci->m_fattr; if (ksmbd_stream_fd(fp) == false) { - file_info->AllocationSize = cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); } else { file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5458,13 +5683,12 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, file_info->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); file_info->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - file_info->ChangeTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); file_info->DosAttributes = fp->f_ci->m_fattr; file_info->Inode = cpu_to_le64(stat.ino); if (ksmbd_stream_fd(fp) == false) { file_info->EndOfFile = cpu_to_le64(stat.size); - file_info->AllocationSize = cpu_to_le64(stat.blocks << 9); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); } else { file_info->EndOfFile = cpu_to_le64(fp->stream.size); file_info->AllocationSize = cpu_to_le64(fp->stream.size); @@ -5645,7 +5869,8 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work, if (!share->path) return -EIO; - rc = kern_path(share->path, LOOKUP_NO_SYMLINKS, &path); + scoped_with_init_fs() + rc = kern_path(share->path, LOOKUP_NO_SYMLINKS, &path); if (rc) { pr_err("cannot create vfs path\n"); return -EIO; @@ -6003,6 +6228,9 @@ int smb2_query_info(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (ksmbd_override_fsids(work)) { rc = -ENOMEM; goto err_out; @@ -6107,6 +6335,9 @@ int smb2_close(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { ksmbd_debug(SMB, "IPC pipe close request\n"); @@ -6173,8 +6404,7 @@ int smb2_close(struct ksmbd_work *work) } rsp->Flags = SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB; - rsp->AllocationSize = S_ISDIR(stat.mode) ? 0 : - cpu_to_le64(stat.blocks << 9); + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); rsp->EndOfFile = cpu_to_le64(stat.size); rsp->Attributes = fp->f_ci->m_fattr; rsp->CreationTime = cpu_to_le64(fp->create_time); @@ -6182,8 +6412,7 @@ int smb2_close(struct ksmbd_work *work) rsp->LastAccessTime = cpu_to_le64(time); time = ksmbd_UnixTimeToNT(stat.mtime); rsp->LastWriteTime = cpu_to_le64(time); - time = ksmbd_UnixTimeToNT(stat.ctime); - rsp->ChangeTime = cpu_to_le64(time); + rsp->ChangeTime = cpu_to_le64(fp->change_time); ksmbd_fd_put(work, fp); } else { rsp->Flags = 0; @@ -6396,9 +6625,11 @@ static int set_file_basic_info(struct ksmbd_file *fp, attrs.ia_valid |= (ATTR_ATIME | ATTR_ATIME_SET); } - if (file_info->ChangeTime) + if (file_info->ChangeTime) { + fp->change_time = le64_to_cpu(file_info->ChangeTime); inode_set_ctime_to_ts(inode, ksmbd_NTtimeToUnix(file_info->ChangeTime)); + } if (file_info->LastWriteTime) { attrs.ia_mtime = ksmbd_NTtimeToUnix(file_info->LastWriteTime); @@ -6506,6 +6737,8 @@ static int set_file_allocation_info(struct ksmbd_work *work, if (size < alloc_blks * 512) i_size_write(inode, size); } + + fp->allocation_size = le64_to_cpu(file_alloc_info->AllocationSize); return 0; } @@ -6562,7 +6795,8 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, return smb2_rename(work, fp, rename_info, work->conn->local_nls); } -static int set_file_disposition_info(struct ksmbd_file *fp, +static int set_file_disposition_info(struct ksmbd_work *work, + struct ksmbd_file *fp, struct smb2_file_disposition_info *file_info) { struct inode *inode; @@ -6574,9 +6808,13 @@ static int set_file_disposition_info(struct ksmbd_file *fp, inode = file_inode(fp->filp); if (file_info->DeletePending) { + if (ksmbd_has_stream_without_delete_share(fp)) + return -ESHARE; + if (S_ISDIR(inode->i_mode) && ksmbd_vfs_empty_dir(fp) == -ENOTEMPTY) return -EBUSY; + smb_break_all_levII_oplock_for_delete(work, fp); ksmbd_set_inode_pending_delete(fp); } else { ksmbd_clear_inode_pending_delete(fp); @@ -6685,16 +6923,18 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, } case FILE_LINK_INFORMATION: { - if (!(fp->daccess & FILE_DELETE_LE)) { - pr_err("no right to delete : 0x%x\n", fp->daccess); - return -EACCES; - } + struct smb2_file_link_info *file_info; if (buf_len < sizeof(struct smb2_file_link_info)) return -EMSGSIZE; - return smb2_create_link(work, work->tcon->share_conf, - (struct smb2_file_link_info *)buffer, + file_info = (struct smb2_file_link_info *)buffer; + if (file_info->ReplaceIfExists && !(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + return smb2_create_link(work, work->tcon->share_conf, file_info, buf_len, fp->filp, work->conn->local_nls); } @@ -6703,7 +6943,7 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, if (buf_len < sizeof(struct smb2_file_disposition_info)) return -EMSGSIZE; - return set_file_disposition_info(fp, + return set_file_disposition_info(work, fp, (struct smb2_file_disposition_info *)buffer); } case FILE_FULL_EA_INFORMATION: @@ -6774,6 +7014,8 @@ int smb2_set_info(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -6989,7 +7231,7 @@ int smb2_read(struct ksmbd_work *work) size_t length, mincount; ssize_t nbytes = 0, remain_bytes = 0; int err = 0; - bool is_rdma_channel = false; + bool is_rdma_channel = false, async_interim = false; unsigned int max_read_size = conn->vals->max_read_size; unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; void *aux_payload_buf; @@ -7005,6 +7247,8 @@ int smb2_read(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -7059,6 +7303,14 @@ int smb2_read(struct ksmbd_work *work) goto out; } + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + offset = le64_to_cpu(req->Offset); if (offset < 0) { err = -EINVAL; @@ -7077,7 +7329,7 @@ int smb2_read(struct ksmbd_work *work) ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", fp->filp, offset, length); - aux_payload_buf = kvzalloc(ALIGN(length, 8), KSMBD_DEFAULT_GFP); + aux_payload_buf = kvmalloc(ALIGN(length, 8), KSMBD_DEFAULT_GFP); if (!aux_payload_buf) { err = -ENOMEM; goto out; @@ -7094,6 +7346,8 @@ int smb2_read(struct ksmbd_work *work) kvfree(aux_payload_buf); rsp->hdr.Status = STATUS_END_OF_FILE; smb2_set_err_rsp(work); + if (async_interim) + release_async_work(work); ksmbd_fd_put(work, fp); return -ENODATA; } @@ -7128,6 +7382,8 @@ int smb2_read(struct ksmbd_work *work) kvfree(aux_payload_buf); goto out; } + if (async_interim) + release_async_work(work); /* * RDMA responses are transferred through channel buffers and encrypted * responses use the encryption transform, so only normal SMB transport @@ -7141,6 +7397,8 @@ int smb2_read(struct ksmbd_work *work) return 0; out: + if (async_interim) + release_async_work(work); if (err) { if (err == -EISDIR) rsp->hdr.Status = STATUS_INVALID_DEVICE_REQUEST; @@ -7276,13 +7534,31 @@ int smb2_write(struct ksmbd_work *work) ssize_t nbytes; char *data_buf; bool writethrough = false, is_rdma_channel = false; + bool async_interim = false; int err = 0; unsigned int max_write_size = work->conn->vals->max_write_size; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; ksmbd_debug(SMB, "Received smb2 write request\n"); WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { ksmbd_debug(SMB, "IPC pipe write request\n"); return smb2_write_pipe(work); @@ -7327,7 +7603,7 @@ int smb2_write(struct ksmbd_work *work) goto out; } - fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + fp = ksmbd_lookup_fd_slow(work, id, pid); if (!fp) { err = -ENOENT; goto out; @@ -7339,6 +7615,14 @@ int smb2_write(struct ksmbd_work *work) goto out; } + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + if (length > max_write_size) { ksmbd_debug(SMB, "limiting write size to max size(%u)\n", max_write_size); @@ -7387,10 +7671,15 @@ int smb2_write(struct ksmbd_work *work) err = ksmbd_iov_pin_rsp(work, rsp, offsetof(struct smb2_write_rsp, Buffer)); if (err) goto out; + if (async_interim) + release_async_work(work); ksmbd_fd_put(work, fp); return 0; out: + if (async_interim) + release_async_work(work); + if (err == -EAGAIN) rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; else if (err == -ENOSPC || err == -EFBIG) @@ -7421,13 +7710,30 @@ int smb2_flush(struct ksmbd_work *work) { struct smb2_flush_req *req; struct smb2_flush_rsp *rsp; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; int err; WORK_BUFFERS(work, req, rsp); ksmbd_debug(SMB, "Received smb2 flush request(fid : %llu)\n", req->VolatileFileId); - err = ksmbd_vfs_fsync(work, req->VolatileFileId, req->PersistentFileId); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + err = ksmbd_vfs_fsync(work, id, pid); if (err) goto out; @@ -7459,7 +7765,8 @@ int smb2_cancel(struct ksmbd_work *work) hdr = ksmbd_resp_buf_next(work); ksmbd_debug(SMB, "smb2 cancel called on mid %llu, async flags 0x%x\n", - hdr->MessageId, hdr->Flags); + le64_to_cpu(hdr->MessageId), + le32_to_cpu(hdr->Flags)); if (hdr->Flags & SMB2_FLAGS_ASYNC_COMMAND) { command_list = &conn->async_requests; @@ -7645,11 +7952,29 @@ int smb2_lock(struct ksmbd_work *work) LIST_HEAD(lock_list); LIST_HEAD(rollback_list); int prior_lock = 0, bkt; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; WORK_BUFFERS(work, req, rsp); ksmbd_debug(SMB, "Received smb2 lock request\n"); - fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); if (!fp) { ksmbd_debug(SMB, "Invalid file id for lock : %llu\n", req->VolatileFileId); err = -ENOENT; @@ -7774,9 +8099,11 @@ int smb2_lock(struct ksmbd_work *work) nolock = 0; list_del(&cmp_lock->flist); list_del(&cmp_lock->clist); + cmp_lock->conn = NULL; spin_unlock(&conn->llist_lock); up_read(&conn_list_lock); + ksmbd_conn_put(conn); locks_free_lock(cmp_lock->fl); kfree(cmp_lock); goto out_check_cl; @@ -7911,6 +8238,7 @@ int smb2_lock(struct ksmbd_work *work) goto out2; } else if (!rc) { list_add(&smb_lock->llist, &rollback_list); + smb_lock->conn = ksmbd_conn_get(work->conn); spin_lock(&work->conn->llist_lock); list_add_tail(&smb_lock->clist, &work->conn->lock_list); @@ -7965,11 +8293,14 @@ int smb2_lock(struct ksmbd_work *work) } list_del(&smb_lock->llist); - spin_lock(&work->conn->llist_lock); + conn = smb_lock->conn; + spin_lock(&conn->llist_lock); if (!list_empty(&smb_lock->flist)) list_del(&smb_lock->flist); list_del(&smb_lock->clist); - spin_unlock(&work->conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); locks_free_lock(smb_lock->fl); if (rlock) @@ -8449,6 +8780,8 @@ int smb2_ioctl(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) { req = ksmbd_req_buf_next(work); rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; if (!has_file_id(req->VolatileFileId)) { ksmbd_debug(SMB, "Compound request set FID = %llu\n", work->compound_fid); @@ -8546,6 +8879,15 @@ int smb2_ioctl(struct ksmbd_work *work) case FSCTL_CREATE_OR_GET_OBJECT_ID: { struct file_object_buf_type1_ioctl_rsp *obj_buf; + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -EBADF; + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + ksmbd_fd_put(work, fp); nbytes = sizeof(struct file_object_buf_type1_ioctl_rsp); obj_buf = (struct file_object_buf_type1_ioctl_rsp *) @@ -8871,11 +9213,10 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) struct smb2_oplock_break *rsp; struct ksmbd_file *fp; struct oplock_info *opinfo = NULL; - __le32 err = 0; - int ret = 0; + __le32 status = STATUS_SUCCESS; + int ret; u64 volatile_id, persistent_id; char req_oplevel = 0, rsp_oplevel = 0; - unsigned int oplock_change_type; WORK_BUFFERS(work, req, rsp); @@ -8901,71 +9242,58 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) return; } - if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { - rsp->hdr.Status = STATUS_INVALID_OPLOCK_PROTOCOL; + if (opinfo->op_state != OPLOCK_ACK_WAIT) { + ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", + opinfo->op_state); + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) + status = STATUS_INVALID_OPLOCK_PROTOCOL; + else + status = STATUS_INVALID_DEVICE_STATE; goto err_out; } - if (opinfo->op_state == OPLOCK_STATE_NONE) { - ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", opinfo->op_state); - rsp->hdr.Status = STATUS_UNSUCCESSFUL; + if (req_oplevel == SMB2_OPLOCK_LEVEL_LEASE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_PARAMETER; goto err_out; } - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - (req_oplevel != SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE)) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_READ_TO_NONE; - } else if (req_oplevel == SMB2_OPLOCK_LEVEL_II || - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_DEVICE_STATE; - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_II) { - oplock_change_type = OPLOCK_WRITE_TO_READ; - } else if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_READ_TO_NONE; - } else { - oplock_change_type = 0; - } - } else { - oplock_change_type = 0; + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - switch (oplock_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_II; - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - default: - pr_err("unknown oplock change 0x%x -> 0x%x\n", - opinfo->level, rsp_oplevel); + if (opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - if (ret < 0) { - rsp->hdr.Status = err; + if (opinfo->level == SMB2_OPLOCK_LEVEL_BATCH && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE && + req_oplevel != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; goto err_out; } + if (opinfo->level == SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (req_oplevel == SMB2_OPLOCK_LEVEL_EXCLUSIVE) + rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; + else + rsp_oplevel = req_oplevel; + + opinfo->level = rsp_oplevel; + rsp->StructureSize = cpu_to_le16(24); rsp->OplockLevel = rsp_oplevel; rsp->Reserved = 0; @@ -8973,27 +9301,33 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) rsp->VolatileFid = volatile_id; rsp->PersistentFid = persistent_id; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_oplock_break)); - if (ret) { + if (ret) + ksmbd_debug(SMB, "failed to pin oplock break response: %d\n", + ret); + goto out; + err_out: - smb2_set_err_rsp(work); - } + rsp->hdr.Status = status; + smb2_set_err_rsp(work); +out: opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); opinfo_put(opinfo); ksmbd_fd_put(work, fp); } -static int check_lease_state(struct lease *lease, __le32 req_state) +static bool smb2_lease_state_valid(__le32 state) { - if ((lease->new_state == - (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) && - !(req_state & SMB2_LEASE_WRITE_CACHING_LE)) { - lease->new_state = req_state; - return 0; - } + return !(state & ~(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE)); +} - if (lease->new_state == req_state) +static int check_lease_state(struct lease *lease, __le32 req_state) +{ + if (smb2_lease_state_valid(req_state) && + !(req_state & ~lease->new_state)) return 0; return 1; @@ -9011,9 +9345,7 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) struct smb2_lease_ack *req; struct smb2_lease_ack *rsp; struct oplock_info *opinfo; - __le32 err = 0; int ret = 0; - unsigned int lease_change_type; __le32 lease_state; struct lease *lease; @@ -9037,80 +9369,23 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) goto err_out; } - if (check_lease_state(lease, req->LeaseState)) { - rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; - ksmbd_debug(OPLOCK, - "req lease state: 0x%x, expected state: 0x%x\n", - req->LeaseState, lease->new_state); - goto err_out; - } - if (!atomic_read(&opinfo->breaking_cnt)) { rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } - /* check for bad lease state */ - if (req->LeaseState & - (~(SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE))) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else if (lease->state == SMB2_LEASE_READ_CACHING_LE && - req->LeaseState != SMB2_LEASE_NONE_LE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else { - /* valid lease state changes */ - err = STATUS_INVALID_DEVICE_STATE; - if (req->LeaseState == SMB2_LEASE_NONE_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - } else if (req->LeaseState & SMB2_LEASE_READ_CACHING_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_READ; - else - lease_change_type = OPLOCK_READ_HANDLE_TO_READ; - } else { - lease_change_type = 0; - } - } - - switch (lease_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - break; - case OPLOCK_READ_HANDLE_TO_READ: - ret = opinfo_read_handle_to_read(opinfo); - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - break; - default: - ksmbd_debug(OPLOCK, "unknown lease change 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } - - if (ret < 0) { - rsp->hdr.Status = err; + if (check_lease_state(lease, req->LeaseState)) { + rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; + ksmbd_debug(OPLOCK, + "req lease state: 0x%x, expected state: 0x%x\n", + req->LeaseState, lease->new_state); goto err_out; } - lease_state = lease->state; + lease_state = req->LeaseState; + lease->state = lease_state; + lease->new_state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(lease); rsp->StructureSize = cpu_to_le16(36); rsp->Reserved = 0; @@ -9119,16 +9394,20 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) rsp->LeaseState = lease_state; rsp->LeaseDuration = 0; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); - if (ret) { -err_out: - smb2_set_err_rsp(work); - } + if (ret) + goto err_out; opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); atomic_dec(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); opinfo_put(opinfo); + return; + +err_out: + smb2_set_err_rsp(work); + opinfo_put(opinfo); + return; } /** @@ -9179,6 +9458,9 @@ int smb2_notify(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (work->next_smb2_rcv_hdr_off && req->hdr.NextCommand) { rsp->hdr.Status = STATUS_INTERNAL_ERROR; smb2_set_err_rsp(work); @@ -9401,10 +9683,13 @@ void smb3_preauth_hash_rsp(struct ksmbd_work *work) WORK_BUFFERS(work, req, rsp); - if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE && - conn->preauth_info) - ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, - conn->preauth_info->Preauth_HashValue); + if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE) { + ksmbd_conn_lock(conn); + if (conn->preauth_info) + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + conn->preauth_info->Preauth_HashValue); + ksmbd_conn_unlock(conn); + } if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && sess) { __u8 *hash_value; diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index 3bed676bb5ad..c2512dbcdec8 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -23,7 +23,7 @@ #define MAX_SMB2_HDR_SIZE 0x78 /* 4 len + 64 hdr + (2*24 wct) + 2 bct + 2 pad */ #define SMB21_DEFAULT_IOSIZE (1024 * 1024) -#define SMB3_DEFAULT_TRANS_SIZE (1024 * 1024) +#define SMB3_DEFAULT_TRANS_SIZE (4 * 1024 * 1024) #define SMB3_MIN_IOSIZE (64 * 1024) #define SMB3_MAX_IOSIZE (8 * 1024 * 1024) #define SMB3_MAX_MSGSIZE (4 * 4096) diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 340ea98fa494..9c59c8f73b66 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -374,6 +374,7 @@ static void parse_dacl(struct mnt_idmap *idmap, { int i, ret; u16 num_aces = 0; + u16 dacl_size; unsigned int acl_size; char *acl_base; struct smb_ace **ppace; @@ -403,7 +404,11 @@ static void parse_dacl(struct mnt_idmap *idmap, if (num_aces <= 0) return; - if (num_aces > (le16_to_cpu(pdacl->size) - sizeof(struct smb_acl)) / + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl)) + return; + + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / (offsetof(struct smb_ace, sid) + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) return; @@ -740,12 +745,18 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, if (nt_ace_size > aces_size) break; + if (ntace->sid.num_subauth == 0 || + ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + goto next_ace; + memcpy((char *)pndace + size, ntace, nt_ace_size); if (check_add_overflow(size, nt_ace_size, &size)) break; + num_aces++; + +next_ace: aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); - num_aces++; } } diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index 74b0307cb100..2797782c0623 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -67,8 +68,9 @@ static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf, } CLASS(filename_kernel, filename)(pathname); - err = vfs_path_parent_lookup(filename, flags, path, &last, - root_share_path); + scoped_with_init_fs() + err = vfs_path_parent_lookup(filename, flags, path, &last, + root_share_path); if (err) return err; @@ -623,7 +625,8 @@ int ksmbd_vfs_link(struct ksmbd_work *work, const char *oldname, if (ksmbd_override_fsids(work)) return -ENOMEM; - err = kern_path(oldname, LOOKUP_NO_SYMLINKS, &oldpath); + scoped_with_init_fs() + err = kern_path(oldname, LOOKUP_NO_SYMLINKS, &oldpath); if (err) { pr_err("cannot get linux path for %s, err = %d\n", oldname, err); @@ -700,10 +703,18 @@ int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, if (err) goto out_drop_write; + if (!work->tcon->posix_extensions && d_is_dir(old_child) && + ksmbd_has_open_files(old_child)) { + err = -EACCES; + goto out3; + } + parent_fp = ksmbd_lookup_fd_inode(old_child->d_parent); if (parent_fp) { - if (parent_fp->daccess & FILE_DELETE_LE) { - pr_err("parent dir is opened with delete access\n"); + if ((parent_fp->daccess & FILE_DELETE_LE) || + (!parent_fp->attrib_only && + !(parent_fp->saccess & FILE_SHARE_DELETE_LE))) { + pr_err("parent dir blocks delete sharing\n"); err = -ESHARE; ksmbd_fd_put(work, parent_fp); goto out3; @@ -1479,8 +1490,8 @@ int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn, if (rc < 0) pr_err("Failed to store XATTR ntacl :%d\n", rc); - kfree(sd_ndr.data); out: + kfree(sd_ndr.data); kfree(acl_ndr.data); kfree(smb_acl); kfree(def_smb_acl); @@ -1496,7 +1507,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, struct ndr n; struct inode *inode = d_inode(dentry); struct ndr acl_ndr = {0}; - struct xattr_ntacl acl; + struct xattr_ntacl acl = {0}; struct xattr_smb_acl *smb_acl = NULL, *def_smb_acl = NULL; __u8 cmp_hash[XATTR_SD_HASH_SIZE] = {0}; @@ -1507,7 +1518,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, n.length = rc; rc = ndr_decode_v4_ntacl(&n, &acl); if (rc) - goto free_n_data; + goto out_free; smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, ACL_TYPE_ACCESS); @@ -1533,6 +1544,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, *pntsd = acl.sd_buf; if (acl.sd_size < sizeof(struct smb_ntsd)) { pr_err("sd size is invalid\n"); + rc = -EINVAL; goto out_free; } @@ -1552,8 +1564,6 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, kfree(acl.sd_buf); *pntsd = NULL; } - -free_n_data: kfree(n.data); return rc; } @@ -1568,14 +1578,15 @@ int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap, err = ndr_encode_dos_attr(&n, da); if (err) - return err; + goto out; err = ksmbd_vfs_setxattr(idmap, path, XATTR_NAME_DOS_ATTRIBUTE, (void *)n.data, n.offset, 0, get_write); if (err) ksmbd_debug(SMB, "failed to store dos attribute in xattr\n"); - kfree(n.data); +out: + kfree(n.data); return err; } diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 8c556e46cc10..d95c405eab11 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "glob.h" #include "vfs_cache.h" @@ -37,6 +38,8 @@ static struct ksmbd_file_table global_ft; static atomic_long_t fd_limit; static struct kmem_cache *filp_cache; +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp); + #define OPLOCK_NONE 0 #define OPLOCK_EXCLUSIVE 1 #define OPLOCK_BATCH 2 @@ -251,6 +254,33 @@ void ksmbd_clear_inode_pending_delete(struct ksmbd_file *fp) up_write(&ci->m_lock); } +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp) +{ + struct ksmbd_file *prev_fp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + if (ksmbd_stream_fd(fp)) + return false; + + down_read(&ci->m_lock); + list_for_each_entry(prev_fp, &ci->m_fp_list, node) { + if (prev_fp == fp || !ksmbd_stream_fd(prev_fp)) + continue; + + if (file_inode(fp->filp) != file_inode(prev_fp->filp)) + continue; + + if (!(prev_fp->saccess & FILE_SHARE_DELETE_LE)) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + void ksmbd_fd_set_delete_on_close(struct ksmbd_file *fp, int file_info) { @@ -484,10 +514,14 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) * there are not accesses to fp->lock_list. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - if (!list_empty(&smb_lock->clist) && fp->conn) { - spin_lock(&fp->conn->llist_lock); - list_del(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + struct ksmbd_conn *conn = smb_lock->conn; + + if (conn) { + spin_lock(&conn->llist_lock); + list_del_init(&smb_lock->clist); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); } list_del(&smb_lock->flist); @@ -513,6 +547,78 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) kmem_cache_free(filp_cache, fp); } +/** + * ksmbd_close_disconnected_durable_delete_on_close() - drop a delete-on-close + * file kept present only by disconnected durable handles + * @dentry: dentry of the file being opened + * + * A durable handle opened with delete-on-close is preserved across a + * disconnect so it can be reclaimed by a durable reconnect. When a new + * (non-reconnect) open arrives for the same name instead, the disconnected + * handle has to give way. Close such handles so their delete-on-close is + * applied and the file is removed once the last handle is gone, letting the + * new open create a fresh file. + * + * The caller's inode reference is dropped before closing so that the final + * close can promote S_DEL_ON_CLS to S_DEL_PENDING and unlink the file. + * + * Return: true if a disconnected durable handle was closed. + */ +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry) +{ + struct ksmbd_inode *ci; + struct ksmbd_file *fp, *tmp; + LIST_HEAD(dispose); + bool closed = false; + + ci = ksmbd_inode_lookup_lock(dentry); + if (!ci) + return false; + + down_write(&ci->m_lock); + if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_ON_CLS_STREAM | S_DEL_PENDING)) { + list_for_each_entry_safe(fp, tmp, &ci->m_fp_list, node) { + if (fp->conn || !fp->is_durable || + fp->f_state != FP_INITED) + continue; + + /* + * Claim the close before unlinking fp from m_fp_list. + * refcount == 1 means only the durable lifetime ref is + * left. Add a transient ref so final close can drop both. + */ + write_lock(&global_ft.lock); + if (atomic_read(&fp->refcount) == 1) { + atomic_inc(&fp->refcount); + __ksmbd_remove_durable_fd(fp); + ksmbd_mark_fp_closed(fp); + list_move_tail(&fp->node, &dispose); + } + write_unlock(&global_ft.lock); + } + } + up_write(&ci->m_lock); + + /* + * Drop our lookup reference before closing so the last __ksmbd_close_fd() + * can drop m_count to zero and unlink the delete-on-close file. The + * collected handles still hold the transient reference taken above, so + * ci stays valid until they are closed below. + */ + ksmbd_inode_put(ci); + + while (!list_empty(&dispose)) { + fp = list_first_entry(&dispose, struct ksmbd_file, node); + list_del_init(&fp->node); + if (atomic_sub_and_test(2, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + closed = true; + } + } + + return closed; +} + static struct ksmbd_file *ksmbd_fp_get(struct ksmbd_file *fp) { if (fp->f_state != FP_INITED) @@ -576,6 +682,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) { struct ksmbd_file *fp; struct ksmbd_file_table *ft; + bool closed = false; if (!has_file_id(id)) return 0; @@ -590,6 +697,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) fp = NULL; else { fp->f_state = FP_CLOSED; + closed = true; if (!atomic_dec_and_test(&fp->refcount)) fp = NULL; } @@ -597,7 +705,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) write_unlock(&ft->lock); if (!fp) - return -EINVAL; + return closed ? 0 : -EINVAL; __put_fd_final(work, fp); return 0; @@ -670,7 +778,8 @@ struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id) struct ksmbd_file *fp; fp = __ksmbd_lookup_fd(&global_ft, id); - if (fp && (fp->conn || + if (fp && (fp->durable_reconnect_disabled || + fp->conn || (fp->durable_scavenger_timeout && (fp->durable_scavenger_timeout < jiffies_to_msecs(jiffies))))) { @@ -689,6 +798,127 @@ void ksmbd_put_durable_fd(struct ksmbd_file *fp) __ksmbd_close_fd(NULL, fp); } +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp) +{ + struct ksmbd_file *lfp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + down_read(&ci->m_lock); + list_for_each_entry(lfp, &ci->m_fp_list, node) { + if (lfp == fp) + continue; + + if (lfp->f_state == FP_INITED && + (READ_ONCE(lfp->conn) || READ_ONCE(lfp->tcon))) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + +static struct ksmbd_file *ksmbd_lookup_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + + if (!memchr_inv(app_instance_id, 0, SMB2_CREATE_GUID_SIZE)) + return NULL; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + if (!memcmp(fp->app_instance_id, app_instance_id, + SMB2_CREATE_GUID_SIZE)) { + fp = ksmbd_fp_get(fp); + break; + } + } + read_unlock(&global_ft.lock); + + return fp; +} + +int ksmbd_close_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file_table *ft; + struct ksmbd_file *fp; + struct oplock_info *opinfo; + int n_to_drop = 0; + + fp = ksmbd_lookup_fd_app_instance_id(app_instance_id); + if (!fp) + return 0; + + opinfo = opinfo_get(fp); + if (!opinfo) + goto out; + + down_read(&fp->f_ci->m_lock); + if (!opinfo->conn) { + up_read(&fp->f_ci->m_lock); + goto out; + } + + ft = &opinfo->sess->file_table; + write_lock(&ft->lock); + if (fp->f_state == FP_INITED && has_file_id(fp->volatile_id)) { + idr_remove(ft->idr, fp->volatile_id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + } + write_unlock(&ft->lock); + up_read(&fp->f_ci->m_lock); + opinfo_put(opinfo); + opinfo = NULL; + + if (!n_to_drop) + goto out; + + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + if (fp->conn) + atomic_dec(&fp->conn->stats.open_files_count); + __ksmbd_close_fd(NULL, fp); + } + return 0; + +out: + if (opinfo) + opinfo_put(opinfo); + ksmbd_put_durable_fd(fp); + return 0; +} + +int ksmbd_invalidate_durable_fd(unsigned long long id) +{ + struct ksmbd_file *fp; + + fp = ksmbd_lookup_global_fd(id); + if (!fp) + return -ENOENT; + + fp->durable_reconnect_disabled = true; + + if (fp->conn) { + ksmbd_put_durable_fd(fp); + return -ENOENT; + } + + fp->durable_timeout = 1; + fp->durable_scavenger_timeout = jiffies_to_msecs(jiffies); + ksmbd_put_durable_fd(fp); + if (waitqueue_active(&dh_wq)) + wake_up(&dh_wq); + + return -ENOENT; +} + struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid) { struct ksmbd_file *fp = NULL; @@ -734,6 +964,30 @@ struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry) return NULL; } +bool ksmbd_has_open_files(struct dentry *dentry) +{ + struct ksmbd_file *fp; + unsigned int id; + bool ret = false; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + struct dentry *fp_dentry = fp->filp->f_path.dentry; + + if (fp->f_state != FP_INITED) + continue; + if (fp_dentry == dentry) + continue; + if (is_subdir(fp_dentry, dentry)) { + ret = true; + break; + } + } + read_unlock(&global_ft.lock); + + return ret; +} + #define OPEN_ID_TYPE_VOLATILE_ID (0) #define OPEN_ID_TYPE_PERSISTENT_ID (1) @@ -758,7 +1012,8 @@ static int __open_id(struct ksmbd_file_table *ft, struct ksmbd_file *fp, idr_preload(KSMBD_DEFAULT_GFP); write_lock(&ft->lock); - ret = idr_alloc_cyclic(ft->idr, fp, 0, INT_MAX - 1, GFP_NOWAIT); + ret = idr_alloc_cyclic(ft->idr, fp, KSMBD_START_FID, INT_MAX - 1, + GFP_NOWAIT); if (ret >= 0) { id = ret; ret = 0; @@ -929,6 +1184,7 @@ __close_file_table_ids(struct ksmbd_session *sess, * global_ft. */ idr_remove(ft->idr, id); + fp->durable_volatile_id = fp->volatile_id; fp->volatile_id = KSMBD_NO_FID; write_unlock(&ft->lock); @@ -1108,10 +1364,10 @@ static int ksmbd_durable_scavenger(void *dummy) if (try_to_freeze()) continue; - remaining_jiffies = wait_event_timeout(dh_wq, + remaining_jiffies = wait_event_interruptible_timeout(dh_wq, ksmbd_durable_scavenger_alive() == false, __msecs_to_jiffies(min_timeout)); - if (remaining_jiffies) + if ((long)remaining_jiffies > 0) min_timeout = jiffies_to_msecs(remaining_jiffies); else min_timeout = DURABLE_HANDLE_MAX_TIMEOUT; @@ -1225,16 +1481,21 @@ void ksmbd_stop_durable_scavenger(void) static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { + char *name; + if (!user) return -EINVAL; /* Duplicate the user name to ensure identity persistence */ - fp->owner.name = kstrdup(user->name, GFP_KERNEL); - if (!fp->owner.name) + name = kstrdup(user->name, GFP_KERNEL); + if (!name) return -ENOMEM; + spin_lock(&fp->f_lock); fp->owner.uid = user->uid; fp->owner.gid = user->gid; + fp->owner.name = name; + spin_unlock(&fp->f_lock); return 0; } @@ -1252,18 +1513,24 @@ static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user) { - if (!user || !fp->owner.name) + bool ret = false; + + if (!user) return false; + spin_lock(&fp->f_lock); + if (!fp->owner.name) + goto out; + /* Check if the UID and GID match first (fast path) */ if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) - return false; + goto out; /* Validate the account name to ensure the same SecurityContext */ - if (strcmp(fp->owner.name, user->name)) - return false; - - return true; + ret = (strcmp(fp->owner.name, user->name) == 0); +out: + spin_unlock(&fp->f_lock); + return ret; } static bool session_fd_check(struct ksmbd_tree_connect *tcon, @@ -1294,18 +1561,26 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, conn = fp->conn; ci = fp->f_ci; down_write(&ci->m_lock); - list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { if (op->conn != conn) continue; ksmbd_conn_put(op->conn); op->conn = NULL; + op->sess = NULL; } up_write(&ci->m_lock); list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - spin_lock(&conn->llist_lock); + struct ksmbd_conn *lock_conn = smb_lock->conn; + + if (!lock_conn) + continue; + spin_lock(&lock_conn->llist_lock); list_del_init(&smb_lock->clist); - spin_unlock(&conn->llist_lock); + smb_lock->conn = NULL; + spin_unlock(&lock_conn->llist_lock); + ksmbd_conn_put(lock_conn); } fp->conn = NULL; @@ -1435,6 +1710,7 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) } list_for_each_entry(smb_lock, &fp->lock_list, flist) { + smb_lock->conn = ksmbd_conn_get(conn); spin_lock(&conn->llist_lock); list_add_tail(&smb_lock->clist, &conn->lock_list); spin_unlock(&conn->llist_lock); @@ -1442,16 +1718,20 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) ci = fp->f_ci; down_write(&ci->m_lock); - list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { if (op->conn) continue; op->conn = ksmbd_conn_get(fp->conn); + op->sess = work->sess; } up_write(&ci->m_lock); + spin_lock(&fp->f_lock); fp->owner.uid = fp->owner.gid = 0; kfree(fp->owner.name); fp->owner.name = NULL; + spin_unlock(&fp->f_lock); return 0; } diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index 7d547e1a74f7..287f3e675cd3 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -23,7 +23,12 @@ #define FILE_GENERIC_WRITE 0x120116 #define FILE_GENERIC_EXECUTE 0X1200a0 -#define KSMBD_START_FID 0 +/* + * Start volatile/persistent file id allocation at 1. A file id of 0 yields an + * SMB2 FileId of {0, 0}, which clients (e.g. Windows, Samba) treat as a null + * handle and never close, leaking the open on the server. + */ +#define KSMBD_START_FID 1 #define KSMBD_NO_FID (INT_MAX) #define SMB2_NO_FID (0xFFFFFFFFFFFFFFFFULL) @@ -32,6 +37,7 @@ struct ksmbd_session; struct ksmbd_lock { struct file_lock *fl; + struct ksmbd_conn *conn; struct list_head clist; struct list_head flist; struct list_head llist; @@ -80,6 +86,7 @@ struct ksmbd_file { struct file *filp; u64 persistent_id; u64 volatile_id; + u64 durable_volatile_id; spinlock_t f_lock; @@ -95,6 +102,8 @@ struct ksmbd_file { __le32 coption; __le32 cdoption; __u64 create_time; + __u64 change_time; + __u64 allocation_size; __u64 itime; bool is_nt_open; @@ -121,6 +130,7 @@ struct ksmbd_file { bool is_durable; bool is_persistent; bool is_resilient; + bool durable_reconnect_disabled; bool is_posix_ctxt; struct durable_owner owner; @@ -159,11 +169,17 @@ struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp); struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d); void ksmbd_inode_put(struct ksmbd_inode *ci); +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry); struct ksmbd_file *ksmbd_lookup_global_fd(unsigned long long id); struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id); void ksmbd_put_durable_fd(struct ksmbd_file *fp); +int ksmbd_invalidate_durable_fd(unsigned long long id); +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp); +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp); +int ksmbd_close_fd_app_instance_id(char *app_instance_id); struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid); struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry); +bool ksmbd_has_open_files(struct dentry *dentry); unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp); struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp); void ksmbd_launch_ksmbd_durable_scavenger(void); diff --git a/fs/super.c b/fs/super.c index a8fd61136aaf..9f82e6b38f07 100644 --- a/fs/super.c +++ b/fs/super.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include #include /* for the emergency remount stuff */ @@ -102,7 +104,7 @@ static bool super_flags(const struct super_block *sb, unsigned int flags) * creation will succeed and SB_BORN is set by vfs_get_tree() or we're * woken and we'll see SB_DYING. * - * The caller must have acquired a temporary reference on @sb->s_count. + * The caller must have acquired a temporary reference on @sb->s_passive. * * Return: The function returns true if SB_BORN was set and with * s_umount held. The function returns false if SB_DYING was @@ -169,6 +171,19 @@ static void super_wake(struct super_block *sb, unsigned int flag) wake_up_var(&sb->s_flags); } +/* + * The s_op->nr_cached_objects hooks (used for example by btrfs and xfs) + * operate on filesystem-global state and ignore sc->memcg. Driving them + * from per-memcg shrink_slab_memcg() invocations only burns CPU walking + * per-cpu counters and queueing duplicate work: the actual reclaim happens on + * the global path (kswapd or root direct reclaim) regardless. Restrict them + * to that path. + */ +static inline bool super_fs_objects_eligible(struct shrink_control *sc) +{ + return !sc->memcg || mem_cgroup_is_root(sc->memcg); +} + /* * One thing we have to be careful of with a per-sb shrinker is that we don't * drop the last active reference to the superblock from within the shrinker. @@ -198,7 +213,7 @@ static unsigned long super_cache_scan(struct shrinker *shrink, if (!super_trylock_shared(sb)) return SHRINK_STOP; - if (sb->s_op->nr_cached_objects) + if (sb->s_op->nr_cached_objects && super_fs_objects_eligible(sc)) fs_objects = sb->s_op->nr_cached_objects(sb, sc); inodes = list_lru_shrink_count(&sb->s_inode_lru, sc); @@ -259,7 +274,8 @@ static unsigned long super_cache_count(struct shrinker *shrink, return 0; smp_rmb(); - if (sb->s_op && sb->s_op->nr_cached_objects) + if (sb->s_op && sb->s_op->nr_cached_objects && + super_fs_objects_eligible(sc)) total_objects = sb->s_op->nr_cached_objects(sb, sc); total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc); @@ -272,6 +288,8 @@ static unsigned long super_cache_count(struct shrinker *shrink, return total_objects; } +static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb); + static void destroy_super_work(struct work_struct *work) { struct super_block *s = container_of(work, struct super_block, @@ -279,6 +297,8 @@ static void destroy_super_work(struct work_struct *work) fsnotify_sb_free(s); security_sb_free(s); put_user_ns(s->s_user_ns); + /* Only an unregistered entry is still owned by the superblock. */ + kfree(s->s_super_dev); kfree(s->s_subtype); for (int i = 0; i < SB_FREEZE_LEVELS; i++) percpu_free_rwsem(&s->s_writers.rw_sem[i]); @@ -367,7 +387,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, spin_lock_init(&s->s_inode_wblist_lock); fserror_mount(s); - s->s_count = 1; + refcount_set(&s->s_passive, 1); atomic_set(&s->s_active, 1); mutex_init(&s->s_vfs_rename_mutex); lockdep_set_class(&s->s_vfs_rename_mutex, &type->s_vfs_rename_key); @@ -392,6 +412,10 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, goto fail; if (list_lru_init_memcg(&s->s_inode_lru, s->s_shrink)) goto fail; + s->s_super_dev = super_dev_alloc(0, s); + if (!s->s_super_dev) + goto fail; + s->s_min_writeback_pages = MIN_WRITEBACK_PAGES; return s; @@ -403,12 +427,17 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, /* Superblock refcounting */ /* - * Drop a superblock's refcount. The caller must hold sb_lock. + * Drop a superblock's passive reference. Must be called WITHOUT sb_lock held; + * put_super() acquires sb_lock itself when the final reference is dropped. */ -static void __put_super(struct super_block *s) +void put_super(struct super_block *s) { - if (!--s->s_count) { + if (refcount_dec_and_test(&s->s_passive)) { + + spin_lock(&sb_lock); list_del_init(&s->s_list); + spin_unlock(&sb_lock); + WARN_ON(s->s_dentry_lru.node); WARN_ON(s->s_inode_lru.node); WARN_ON(s->s_mounts); @@ -416,18 +445,109 @@ static void __put_super(struct super_block *s) } } -/** - * put_super - drop a temporary reference to superblock - * @sb: superblock in question - * - * Drops a temporary reference, frees superblock if there's no - * references left. - */ -void put_super(struct super_block *sb) +struct super_dev { + dev_t sd_dev; + struct super_block *sd_sb; + refcount_t sd_ref; + struct rhlist_head sd_node; + struct rcu_head sd_rcu; +}; + +static struct rhltable super_dev_table; +static const struct rhashtable_params super_dev_params = { + .key_len = sizeof(dev_t), + .key_offset = offsetof(struct super_dev, sd_dev), + .head_offset = offsetof(struct super_dev, sd_node), +}; + +static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb) { - spin_lock(&sb_lock); - __put_super(sb); - spin_unlock(&sb_lock); + struct super_dev *fsd; + + fsd = kzalloc_obj(*fsd); + if (!fsd) + return NULL; + fsd->sd_dev = dev; + fsd->sd_sb = sb; + refcount_set(&fsd->sd_ref, 1); + return fsd; +} + +static void super_dev_put(struct super_dev *fsd) +{ + /* Unlink only once unpinned, so a cursor never resumes from a removed node. */ + if (fsd && refcount_dec_and_test(&fsd->sd_ref)) { + rhltable_remove(&super_dev_table, &fsd->sd_node, super_dev_params); + put_super(fsd->sd_sb); + kfree_rcu(fsd, sd_rcu); + } +} + +void __init super_dev_init(void) +{ + if (rhltable_init(&super_dev_table, &super_dev_params)) + panic("VFS: Cannot initialise super_dev_table\n"); +} + +static int super_dev_insert(struct super_dev *fsd) +{ + int err; + + err = rhltable_insert(&super_dev_table, &fsd->sd_node, super_dev_params); + if (!err) + refcount_inc(&fsd->sd_sb->s_passive); + return err; +} + +/* Register @sb under @sb->s_dev as the final fallible act of a set callback. */ +static int super_dev_register(struct super_block *sb) +{ + struct super_dev *fsd = sb->s_super_dev; + int err; + + lockdep_assert_held(&sb_lock); + VFS_WARN_ON_ONCE(!sb->s_dev); + VFS_WARN_ON_ONCE(!fsd || fsd->sd_dev); + + fsd->sd_dev = sb->s_dev; + err = super_dev_insert(fsd); + if (err) + fsd->sd_dev = 0; + return err; +} + +static struct super_dev *super_dev_get(struct rhlist_head *pos) +{ + struct super_dev *sb_dev; + + for (; pos; pos = rcu_dereference_all(pos->next)) { + sb_dev = container_of(pos, struct super_dev, sd_node); + if (refcount_inc_not_zero(&sb_dev->sd_ref)) + return sb_dev; + } + return NULL; +} + +static struct super_dev *super_dev_first(dev_t dev) +{ + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_get(rhltable_lookup(&super_dev_table, &dev, super_dev_params)); + rcu_read_unlock(); + return sb_dev; +} + +static struct super_dev *super_dev_next(struct super_dev *prev) +{ + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_get(rcu_dereference_all(prev->sd_node.next)); + rcu_read_unlock(); + + super_dev_put(prev); + return sb_dev; } static void kill_super_notify(struct super_block *sb) @@ -449,6 +569,12 @@ static void kill_super_notify(struct super_block *sb) hlist_del_init(&sb->s_instances); spin_unlock(&sb_lock); + /* Drop sget_fc()'s claim; a never-registered entry stays with the sb. */ + if (sb->s_super_dev->sd_dev) { + super_dev_put(sb->s_super_dev); + sb->s_super_dev = NULL; + } + /* * Let concurrent mounts know that this thing is really dead. * We don't need @sb->s_umount here as every concurrent caller @@ -478,11 +604,7 @@ void deactivate_locked_super(struct super_block *s) kill_super_notify(s); - /* - * Since list_lru_destroy() may sleep, we cannot call it from - * put_super(), where we hold the sb_lock. Therefore we destroy - * the lru lists right now. - */ + /* list_lru_destroy() may sleep; put_super() callers may not. */ list_lru_destroy(&s->s_dentry_lru); list_lru_destroy(&s->s_inode_lru); @@ -529,7 +651,7 @@ static bool grab_super(struct super_block *sb) { bool locked; - sb->s_count++; + refcount_inc(&sb->s_passive); spin_unlock(&sb_lock); locked = super_lock_excl(sb); if (locked) { @@ -556,7 +678,7 @@ static bool grab_super(struct super_block *sb) * lock held in read mode in case of success. On successful return, * the caller must drop the s_umount lock when done. * - * Note that unlike get_super() et.al. this one does *not* bump ->s_count. + * Note that unlike get_super() et.al. this one does *not* bump ->s_passive. * The reason why it's safe is that we are OK with doing trylock instead * of down_read(). There's a couple of places that are OK with that, but * it's very much not a general-purpose interface. @@ -763,6 +885,7 @@ struct super_block *sget_fc(struct fs_context *fc, } if (!s) { spin_unlock(&sb_lock); + s = alloc_super(fc->fs_type, fc->sb_flags, user_ns); if (!s) return ERR_PTR(-ENOMEM); @@ -772,11 +895,13 @@ struct super_block *sget_fc(struct fs_context *fc, s->s_fs_info = fc->s_fs_info; err = set(s, fc); if (err) { + VFS_WARN_ON_ONCE(s->s_super_dev->sd_dev); s->s_fs_info = NULL; spin_unlock(&sb_lock); destroy_unused_super(s); return ERR_PTR(err); } + VFS_WARN_ON_ONCE(!s->s_super_dev->sd_dev); fc->s_fs_info = NULL; s->s_type = fc->fs_type; s->s_iflags |= fc->s_iflags; @@ -851,14 +976,17 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg, struct super_block *sb, *p = NULL; bool excl = flags & SUPER_ITER_EXCL; - guard(spinlock)(&sb_lock); + spin_lock(&sb_lock); for (sb = first_super(flags); !list_entry_is_head(sb, &super_blocks, s_list); sb = next_super(sb, flags)) { if (super_flags(sb, SB_DYING)) continue; - sb->s_count++; + + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + spin_unlock(&sb_lock); if (flags & SUPER_ITER_UNLOCKED) { @@ -868,13 +996,14 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg, super_unlock(sb, excl); } - spin_lock(&sb_lock); if (p) - __put_super(p); + put_super(p); p = sb; + spin_lock(&sb_lock); } + spin_unlock(&sb_lock); if (p) - __put_super(p); + put_super(p); } void iterate_supers(void (*f)(struct super_block *, void *), void *arg) @@ -903,7 +1032,9 @@ void iterate_supers_type(struct file_system_type *type, if (super_flags(sb, SB_DYING)) continue; - sb->s_count++; + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + spin_unlock(&sb_lock); locked = super_lock_shared(sb); @@ -912,41 +1043,33 @@ void iterate_supers_type(struct file_system_type *type, super_unlock_shared(sb); } - spin_lock(&sb_lock); if (p) - __put_super(p); + put_super(p); p = sb; + spin_lock(&sb_lock); } - if (p) - __put_super(p); spin_unlock(&sb_lock); + if (p) + put_super(p); } EXPORT_SYMBOL(iterate_supers_type); struct super_block *user_get_super(dev_t dev, bool excl) { - struct super_block *sb; + struct super_dev *sb_dev; - spin_lock(&sb_lock); - list_for_each_entry(sb, &super_blocks, s_list) { - bool locked; + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; - if (sb->s_dev != dev) + if (!super_lock(sb, excl)) continue; - sb->s_count++; - spin_unlock(&sb_lock); - - locked = super_lock(sb, excl); - if (locked) - return sb; - - spin_lock(&sb_lock); - __put_super(sb); - break; + /* The pinned entry holds a passive reference, take our own. */ + refcount_inc(&sb->s_passive); + super_dev_put(sb_dev); + return sb; } - spin_unlock(&sb_lock); return NULL; } @@ -1222,7 +1345,16 @@ EXPORT_SYMBOL(free_anon_bdev); int set_anon_super(struct super_block *s, void *data) { - return get_anon_bdev(&s->s_dev); + int error; + + error = get_anon_bdev(&s->s_dev); + if (error) + return error; + + error = super_dev_register(s); + if (error) + free_anon_bdev(s->s_dev); + return error; } EXPORT_SYMBOL(set_anon_super); @@ -1308,7 +1440,7 @@ EXPORT_SYMBOL(get_tree_keyed); static int set_bdev_super(struct super_block *s, void *data) { s->s_dev = *(dev_t *)data; - return 0; + return super_dev_register(s); } static int super_s_dev_set(struct super_block *s, struct fs_context *fc) @@ -1350,197 +1482,313 @@ struct super_block *sget_dev(struct fs_context *fc, dev_t dev) EXPORT_SYMBOL(sget_dev); #ifdef CONFIG_BLOCK -/* - * Lock the superblock that is holder of the bdev. Returns the superblock - * pointer if we successfully locked the superblock and it is alive. Otherwise - * we return NULL and just unlock bdev->bd_holder_lock. - * - * The function must be called with bdev->bd_holder_lock and releases it. - */ -static struct super_block *bdev_super_lock(struct block_device *bdev, bool excl) - __releases(&bdev->bd_holder_lock) +static int fs_super_freeze(struct super_block *sb) { - struct super_block *sb = bdev->bd_holder; - bool locked; - - lockdep_assert_held(&bdev->bd_holder_lock); - lockdep_assert_not_held(&sb->s_umount); - lockdep_assert_not_held(&bdev->bd_disk->open_mutex); - - /* Make sure sb doesn't go away from under us */ - spin_lock(&sb_lock); - sb->s_count++; - spin_unlock(&sb_lock); - - mutex_unlock(&bdev->bd_holder_lock); - - locked = super_lock(sb, excl); - - /* - * If the superblock wasn't already SB_DYING then we hold - * s_umount and can safely drop our temporary reference. - */ - put_super(sb); - - if (!locked) - return NULL; - - if (!sb->s_root || !(sb->s_flags & SB_ACTIVE)) { - super_unlock(sb, excl); - return NULL; - } + if (sb->s_op->freeze_super) + return sb->s_op->freeze_super(sb, + FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); + return freeze_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); +} - return sb; +static int fs_super_thaw(struct super_block *sb) +{ + if (sb->s_op->thaw_super) + return sb->s_op->thaw_super(sb, + FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); + return thaw_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); } static void fs_bdev_mark_dead(struct block_device *bdev, bool surprise) { - struct super_block *sb; + struct super_dev *sb_dev; + dev_t dev = bdev->bd_dev; - sb = bdev_super_lock(bdev, false); - if (!sb) - return; + mutex_unlock(&bdev->bd_holder_lock); - if (sb->s_op->remove_bdev) { - int ret; + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; - ret = sb->s_op->remove_bdev(sb, bdev); - if (!ret) { - super_unlock_shared(sb); - return; + if (!super_lock_shared(sb)) + continue; + if (sb->s_root && (sb->s_flags & SB_ACTIVE)) { + if (!sb->s_op->remove_bdev || + sb->s_op->remove_bdev(sb, bdev)) { + if (!surprise) + sync_filesystem(sb); + shrink_dcache_sb(sb); + evict_inodes(sb); + if (sb->s_op->shutdown) + sb->s_op->shutdown(sb); + } } - /* Fallback to shutdown. */ + super_unlock_shared(sb); } - - if (!surprise) - sync_filesystem(sb); - shrink_dcache_sb(sb); - evict_inodes(sb); - if (sb->s_op->shutdown) - sb->s_op->shutdown(sb); - - super_unlock_shared(sb); } static void fs_bdev_sync(struct block_device *bdev) { - struct super_block *sb; - - sb = bdev_super_lock(bdev, false); - if (!sb) - return; + struct super_dev *sb_dev; + dev_t dev = bdev->bd_dev; - sync_filesystem(sb); - super_unlock_shared(sb); -} + mutex_unlock(&bdev->bd_holder_lock); -static struct super_block *get_bdev_super(struct block_device *bdev) -{ - bool active = false; - struct super_block *sb; + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; - sb = bdev_super_lock(bdev, true); - if (sb) { - active = atomic_inc_not_zero(&sb->s_active); - super_unlock_excl(sb); + if (!super_lock_shared(sb)) + continue; + if (sb->s_root && (sb->s_flags & SB_ACTIVE)) + sync_filesystem(sb); + super_unlock_shared(sb); } - if (!active) - return NULL; - return sb; } /** - * fs_bdev_freeze - freeze owning filesystem of block device + * fs_bdev_freeze - freeze every superblock using a block device * @bdev: block device * - * Freeze the filesystem that owns this block device if it is still - * active. - * - * A filesystem that owns multiple block devices may be frozen from each - * block device and won't be unfrozen until all block devices are - * unfrozen. Each block device can only freeze the filesystem once as we - * nest freezes for block devices in the block layer. + * Freeze each live superblock using @bdev. A superblock owning several block + * devices is frozen once per device and stays frozen until all are thawed; the + * block layer nests these freezes so the count stays balanced. * - * Return: If the freeze was successful zero is returned. If the freeze - * failed a negative error code is returned. + * Return: 0, or the error from the one superblock on a single-fs device. When + * several superblocks share @bdev a per-superblock failure is swallowed + * (see below), but a sync_blockdev() failure is always reported. */ static int fs_bdev_freeze(struct block_device *bdev) { - struct super_block *sb; - int error = 0; + dev_t dev = bdev->bd_dev; + struct super_dev *sb_dev; + unsigned int count = 0; + int error = 0, err; lockdep_assert_held(&bdev->bd_fsfreeze_mutex); - sb = get_bdev_super(bdev); - if (!sb) - return -EINVAL; + mutex_unlock(&bdev->bd_holder_lock); - if (sb->s_op->freeze_super) - error = sb->s_op->freeze_super(sb, - FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); - else - error = freeze_super(sb, - FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + if (!get_active_super(sb_dev->sd_sb)) + continue; + err = fs_super_freeze(sb_dev->sd_sb); + if (err && !error) + error = err; + deactivate_super(sb_dev->sd_sb); + count++; + } + + /* + * When several superblocks share the device, keep it frozen even if some + * of them failed to freeze and swallow the error: rolling the rest back + * via thaw_super() can fail too, so neither is a clear win. A single + * filesystem (count == 1) still reports its error. + */ + if (error && count > 1) + error = 0; if (!error) error = sync_blockdev(bdev); - deactivate_super(sb); return error; } /** - * fs_bdev_thaw - thaw owning filesystem of block device + * fs_bdev_thaw - thaw every superblock using a block device * @bdev: block device * - * Thaw the filesystem that owns this block device. - * - * A filesystem that owns multiple block devices may be frozen from each - * block device and won't be unfrozen until all block devices are - * unfrozen. Each block device can only freeze the filesystem once as we - * nest freezes for block devices in the block layer. + * The counterpart to fs_bdev_freeze(): thaw each live superblock using @bdev. + * A zero return does not imply a superblock is fully unfrozen; it may have been + * frozen more than once (by the kernel or via another device). * - * Return: If the thaw was successful zero is returned. If the thaw - * failed a negative error code is returned. If this function - * returns zero it doesn't mean that the filesystem is unfrozen - * as it may have been frozen multiple times (kernel may hold a - * freeze or might be frozen from other block devices). + * Return: 0, or the first error on a single-fs device; a shared device swallows + * per-superblock errors, as fs_bdev_freeze() does. */ static int fs_bdev_thaw(struct block_device *bdev) { - struct super_block *sb; - int error; + dev_t dev = bdev->bd_dev; + struct super_dev *sb_dev; + unsigned int count = 0; + int error = 0, err; lockdep_assert_held(&bdev->bd_fsfreeze_mutex); - /* - * The block device may have been frozen before it was claimed by a - * filesystem. Concurrently another process might try to mount that - * frozen block device and has temporarily claimed the block device for - * that purpose causing a concurrent fs_bdev_thaw() to end up here. The - * mounter is already about to abort mounting because they still saw an - * elevanted bdev->bd_fsfreeze_count so get_bdev_super() will return - * NULL in that case. - */ - sb = get_bdev_super(bdev); - if (!sb) - return -EINVAL; + mutex_unlock(&bdev->bd_holder_lock); - if (sb->s_op->thaw_super) - error = sb->s_op->thaw_super(sb, - FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); - else - error = thaw_super(sb, - FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); - deactivate_super(sb); + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + if (!get_active_super(sb_dev->sd_sb)) + continue; + err = fs_super_thaw(sb_dev->sd_sb); + if (err && !error) + error = err; + deactivate_super(sb_dev->sd_sb); + count++; + } + + /* Shared device: swallow per-superblock errors, like fs_bdev_freeze(). */ + if (error && count > 1) + error = 0; return error; } -const struct blk_holder_ops fs_holder_ops = { +static const struct blk_holder_ops fs_holder_ops = { .mark_dead = fs_bdev_mark_dead, .sync = fs_bdev_sync, .freeze = fs_bdev_freeze, .thaw = fs_bdev_thaw, }; -EXPORT_SYMBOL_GPL(fs_holder_ops); + +static struct super_dev *super_dev_lookup(dev_t dev, struct super_block *sb) +{ + struct super_dev *it; + struct rhlist_head *list, *pos; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "suspicious super_dev_lookup() usage"); + VFS_WARN_ON_ONCE(!dev); + VFS_WARN_ON_ONCE(!sb); + + list = rhltable_lookup(&super_dev_table, &dev, super_dev_params); + rhl_for_each_entry_rcu(it, pos, list, sd_node) { + if (it->sd_sb == sb) + return it; + } + + return NULL; +} + +static int fs_bdev_register(struct file *bdev_file, struct super_block *sb) +{ + struct super_dev *sb_dev __free(kfree) = NULL; + dev_t dev = file_bdev(bdev_file)->bd_dev; + int err; + + scoped_guard(rcu) { + sb_dev = super_dev_lookup(dev, sb); + if (sb_dev && refcount_inc_not_zero(&sb_dev->sd_ref)) { + retain_and_null_ptr(sb_dev); + return 0; + } + } + + sb_dev = super_dev_alloc(dev, sb); + if (!sb_dev) + return -ENOMEM; + + err = super_dev_insert(sb_dev); + if (err) + return err; + + /* Publish the entry before reading the count; pairs with bdev_freeze(). */ + smp_mb(); + if (atomic_read(&file_bdev(bdev_file)->bd_fsfreeze_count) > 0) { + err = -EBUSY; + super_dev_put(sb_dev); + } + + retain_and_null_ptr(sb_dev); + return err; +} + +/** + * fs_bdev_file_open_by_dev - claim a block device on behalf of a superblock + * @dev: block device number + * @mode: open mode + * @holder: block-layer exclusivity token (a superblock, or the file_system_type + * when the device may be shared by several superblocks of that type) + * @sb: superblock to drive fs_holder_ops events for + * + * Open @dev with &fs_holder_ops and register that @sb uses it, so device + * removal/sync/freeze/thaw are propagated to @sb (and any other superblock + * sharing @dev). Must be paired with fs_bdev_file_release(). + * + * Return: an opened block-device file or an ERR_PTR(). + */ +struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, + struct super_block *sb) +{ + struct file *bdev_file; + int err; + + bdev_file = bdev_file_open_by_dev(dev, mode, holder, &fs_holder_ops); + if (IS_ERR(bdev_file)) + return bdev_file; + + err = fs_bdev_register(bdev_file, sb); + if (err) { + bdev_fput(bdev_file); + return ERR_PTR(err); + } + return bdev_file; +} +EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_dev); + +/** + * fs_bdev_file_open_by_path - claim a block device on behalf of a superblock + * @path: path to the block device + * @mode: open mode + * @holder: block-layer exclusivity token (a superblock, or the file_system_type + * when the device may be shared by several superblocks of that type) + * @sb: superblock to drive fs_holder_ops events for + * + * Open the block device at @path with &fs_holder_ops and register that @sb + * uses it, so device removal/sync/freeze/thaw are propagated to @sb (and any + * other superblock sharing the device). Must be paired with + * fs_bdev_file_release(). + * + * Return: an opened block-device file or an ERR_PTR(). + */ +struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode, + void *holder, struct super_block *sb) +{ + struct file *bdev_file; + int err; + + bdev_file = bdev_file_open_by_path(path, mode, holder, &fs_holder_ops); + if (IS_ERR(bdev_file)) + return bdev_file; + + err = fs_bdev_register(bdev_file, sb); + if (err) { + bdev_fput(bdev_file); + return ERR_PTR(err); + } + return bdev_file; +} +EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_path); + +/** + * fs_bdev_unregister - drop a superblock's claim on a block device + * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}() + * @sb: superblock the device was claimed for + * + * The inverse of fs_bdev_register(): drop one claim on the {dev, @sb} entry + * (the last claim unregisters it; a pinning cursor defers the actual unlink) + * without closing the device. A caller that must act on the still-open device + * between unregistering and closing - e.g. re-allow freezing one denied for a + * membership change - pairs this with bdev_fput(). fs_bdev_file_release() is + * the common unregister-and-close. + */ +void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb) +{ + dev_t dev = file_bdev(bdev_file)->bd_dev; + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_lookup(dev, sb); + rcu_read_unlock(); + super_dev_put(sb_dev); +} +EXPORT_SYMBOL_GPL(fs_bdev_unregister); + +/** + * fs_bdev_file_release - release a block device claimed for a superblock + * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}() + * @sb: superblock the device was claimed for + * + * Unregister the {dev, @sb} entry, then close the block device. + */ +void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb) +{ + fs_bdev_unregister(bdev_file, sb); + bdev_fput(bdev_file); +} +EXPORT_SYMBOL_GPL(fs_bdev_file_release); int setup_bdev_super(struct super_block *sb, int sb_flags, struct fs_context *fc) @@ -1549,7 +1797,7 @@ int setup_bdev_super(struct super_block *sb, int sb_flags, struct file *bdev_file; struct block_device *bdev; - bdev_file = bdev_file_open_by_dev(sb->s_dev, mode, sb, &fs_holder_ops); + bdev_file = fs_bdev_file_open_by_dev(sb->s_dev, mode, sb, sb); if (IS_ERR(bdev_file)) { if (fc) errorf(fc, "%s: Can't open blockdev", fc->source); @@ -1563,20 +1811,19 @@ int setup_bdev_super(struct super_block *sb, int sb_flags, * writable from userspace even for a read-only block device. */ if ((mode & BLK_OPEN_WRITE) && bdev_read_only(bdev)) { - bdev_fput(bdev_file); + fs_bdev_file_release(bdev_file, sb); return -EACCES; } - /* - * It is enough to check bdev was not frozen before we set - * s_bdev as freezing will wait until SB_BORN is set. - */ + /* The sget_fc() entry is already published; pairs with bdev_freeze(). */ + smp_mb(); if (atomic_read(&bdev->bd_fsfreeze_count) > 0) { if (fc) warnf(fc, "%pg: Can't mount, blockdev is frozen", bdev); - bdev_fput(bdev_file); + fs_bdev_file_release(bdev_file, sb); return -EBUSY; } + spin_lock(&sb_lock); sb->s_bdev_file = bdev_file; sb->s_bdev = bdev; @@ -1665,7 +1912,7 @@ void kill_block_super(struct super_block *sb) generic_shutdown_super(sb); if (bdev) { sync_blockdev(bdev); - bdev_fput(sb->s_bdev_file); + fs_bdev_file_release(sb->s_bdev_file, sb); } } diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index 86d41e077e4d..23ec924162d6 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -303,7 +303,7 @@ static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry, } static int ubifs_create(struct mnt_idmap *idmap, struct inode *dir, - struct dentry *dentry, umode_t mode, bool excl) + struct dentry *dentry, umode_t mode) { struct inode *inode; struct ubifs_info *c = dir->i_sb->s_fs_info; @@ -1031,7 +1031,7 @@ static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir, sz_change = CALC_DENT_SIZE(fname_len(&nm)); - inode = ubifs_new_inode(c, dir, S_IFDIR | mode, false); + inode = ubifs_new_inode(c, dir, mode, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_fname; diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 9a3b7cef3606..b90841ac0a40 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -371,7 +371,7 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode) } static int udf_create(struct mnt_idmap *idmap, struct inode *dir, - struct dentry *dentry, umode_t mode, bool excl) + struct dentry *dentry, umode_t mode) { struct inode *inode = udf_new_inode(dir, mode); @@ -428,7 +428,7 @@ static struct dentry *udf_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct udf_inode_info *dinfo = UDF_I(dir); struct udf_inode_info *iinfo; - inode = udf_new_inode(dir, S_IFDIR | mode); + inode = udf_new_inode(dir, mode); if (IS_ERR(inode)) return ERR_CAST(inode); diff --git a/fs/ufs/namei.c b/fs/ufs/namei.c index 5b3c85c93242..6703f3bcf76f 100644 --- a/fs/ufs/namei.c +++ b/fs/ufs/namei.c @@ -70,8 +70,7 @@ static struct dentry *ufs_lookup(struct inode * dir, struct dentry *dentry, unsi * with d_instantiate(). */ static int ufs_create (struct mnt_idmap * idmap, - struct inode * dir, struct dentry * dentry, umode_t mode, - bool excl) + struct inode * dir, struct dentry * dentry, umode_t mode) { struct inode *inode; @@ -174,7 +173,7 @@ static struct dentry *ufs_mkdir(struct mnt_idmap * idmap, struct inode * dir, inode_inc_link_count(dir); - inode = ufs_new_inode(dir, S_IFDIR|mode); + inode = ufs_new_inode(dir, mode); err = PTR_ERR(inode); if (IS_ERR(inode)) goto out_dir; diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c index 543c60c12461..8c4b8e27e956 100644 --- a/fs/unicode/utf8-core.c +++ b/fs/unicode/utf8-core.c @@ -138,28 +138,6 @@ int utf8_casefold_hash(const struct unicode_map *um, const void *salt, } EXPORT_SYMBOL(utf8_casefold_hash); -int utf8_normalize(const struct unicode_map *um, const struct qstr *str, - unsigned char *dest, size_t dlen) -{ - struct utf8cursor cur; - ssize_t nlen = 0; - - if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0) - return -EINVAL; - - for (nlen = 0; nlen < dlen; nlen++) { - int c = utf8byte(&cur); - - dest[nlen] = c; - if (!c) - return nlen; - if (c == -1) - break; - } - return -EINVAL; -} -EXPORT_SYMBOL(utf8_normalize); - static const struct utf8data *find_table_version(const struct utf8data *table, size_t nr_entries, unsigned int version) { diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c index c5bd3271aa96..0b9eab157432 100644 --- a/fs/vboxsf/dir.c +++ b/fs/vboxsf/dir.c @@ -298,9 +298,9 @@ static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry, static int vboxsf_dir_mkfile(struct mnt_idmap *idmap, struct inode *parent, struct dentry *dentry, - umode_t mode, bool excl) + umode_t mode) { - return vboxsf_dir_create(parent, dentry, mode, false, excl, NULL); + return vboxsf_dir_create(parent, dentry, mode, false, true, NULL); } static struct dentry *vboxsf_dir_mkdir(struct mnt_idmap *idmap, diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c index ce767b40482f..bbada0d3cc08 100644 --- a/fs/xfs/libxfs/xfs_dquot_buf.c +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -436,17 +436,27 @@ xfs_dqinode_metadir_create( error = xfs_metadir_create(&upd, S_IFREG); if (error) - return error; + goto out_cancel; xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE); error = xfs_metadir_commit(&upd); if (error) - return error; + goto out_irele; xfs_finish_inode_setup(upd.ip); *ipp = upd.ip; return 0; + +out_cancel: + xfs_metadir_cancel(&upd, error); +out_irele: + /* Have to finish setting up the inode to ensure it's deleted. */ + if (upd.ip) { + xfs_finish_inode_setup(upd.ip); + xfs_irele(upd.ip); + } + return error; } #ifndef __KERNEL__ diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index 7114c9a44182..2554494847ff 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -652,7 +652,7 @@ xrep_agfl_fill( while (agbno < start + len && af->fl_off < af->flcount) af->agfl_bno[af->fl_off++] = cpu_to_be32(agbno++); - error = xagb_bitmap_set(&af->used_extents, start, agbno - 1); + error = xagb_bitmap_set(&af->used_extents, start, agbno - start); if (error) return error; diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 2a0c54256e93..51293b6f331f 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -764,8 +764,7 @@ xfs_bio_submit_read( /* defer read completions to the ioend workqueue */ iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0); - bio->bi_end_io = xfs_end_bio; - submit_bio(bio); + iomap_bio_submit_read_endio(iter, ctx, xfs_end_bio); } static const struct iomap_read_ops xfs_iomap_read_ops = { diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 3ce12fe1c307..8c210ebb66bc 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -120,6 +120,22 @@ xfs_buf_free( call_rcu(&bp->b_rcu, xfs_buf_free_callback); } +static int +xfs_buf_alloc_folio( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + struct folio *folio; + + folio = folio_alloc(gfp_mask, get_order(size)); + if (!folio) + return -ENOMEM; + bp->b_addr = folio_address(folio); + trace_xfs_buf_backing_folio(bp, _RET_IP_); + return 0; +} + static int xfs_buf_alloc_kmem( struct xfs_buf *bp, @@ -129,7 +145,7 @@ xfs_buf_alloc_kmem( ASSERT(is_power_of_2(size)); ASSERT(size < PAGE_SIZE); - bp->b_addr = kmalloc(size, gfp_mask | __GFP_NOFAIL); + bp->b_addr = kmalloc(size, gfp_mask); if (!bp->b_addr) return -ENOMEM; @@ -148,6 +164,26 @@ xfs_buf_alloc_kmem( return 0; } +static int +xfs_buf_alloc_vmalloc( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + for (;;) { + bp->b_addr = __vmalloc(size, gfp_mask); + if (bp->b_addr) + break; + if (gfp_mask & __GFP_NORETRY) + return -ENOMEM; + XFS_STATS_INC(bp->b_mount, xb_page_retries); + memalloc_retry_wait(gfp_mask); + } + + trace_xfs_buf_backing_vmalloc(bp, _RET_IP_); + return 0; +} + /* * Allocate backing memory for a buffer. * @@ -175,7 +211,6 @@ xfs_buf_alloc_backing_mem( { size_t size = BBTOB(bp->b_length); gfp_t gfp_mask = GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOWARN; - struct folio *folio; if (xfs_buftarg_is_mem(bp->b_target)) return xmbuf_map_backing_mem(bp); @@ -187,22 +222,6 @@ xfs_buf_alloc_backing_mem( if (flags & XBF_READ_AHEAD) gfp_mask |= __GFP_NORETRY; - /* - * For buffers smaller than PAGE_SIZE use a kmalloc allocation if that - * is properly aligned. The slab allocator now guarantees an aligned - * allocation for all power of two sizes, which matches most of the - * smaller than PAGE_SIZE buffers used by XFS. - */ - if (size < PAGE_SIZE && is_power_of_2(size)) - return xfs_buf_alloc_kmem(bp, size, gfp_mask); - - /* - * Don't bother with the retry loop for single PAGE allocations: vmalloc - * won't do any better. - */ - if (size <= PAGE_SIZE) - gfp_mask |= __GFP_NOFAIL; - /* * Optimistically attempt a single high order folio allocation for * larger than PAGE_SIZE buffers. @@ -215,35 +234,31 @@ xfs_buf_alloc_backing_mem( * path for them instead of wasting memory here. */ if (size > PAGE_SIZE) { - if (!is_power_of_2(size)) - goto fallback; - gfp_mask &= ~__GFP_DIRECT_RECLAIM; - gfp_mask |= __GFP_NORETRY; - } - folio = folio_alloc(gfp_mask, get_order(size)); - if (!folio) { - if (size <= PAGE_SIZE) - return -ENOMEM; - trace_xfs_buf_backing_fallback(bp, _RET_IP_); - goto fallback; - } - bp->b_addr = folio_address(folio); - trace_xfs_buf_backing_folio(bp, _RET_IP_); - return 0; - -fallback: - for (;;) { - bp->b_addr = __vmalloc(size, gfp_mask); - if (bp->b_addr) - break; - if (flags & XBF_READ_AHEAD) - return -ENOMEM; - XFS_STATS_INC(bp->b_mount, xb_page_retries); - memalloc_retry_wait(gfp_mask); + if (is_power_of_2(size)) { + gfp_t folio_gfp = gfp_mask; + + folio_gfp &= ~__GFP_DIRECT_RECLAIM; + folio_gfp |= __GFP_NORETRY; + if (xfs_buf_alloc_folio(bp, size, folio_gfp) == 0) + return 0; + trace_xfs_buf_backing_fallback(bp, _RET_IP_); + } + return xfs_buf_alloc_vmalloc(bp, size, gfp_mask); } - trace_xfs_buf_backing_vmalloc(bp, _RET_IP_); - return 0; + /* + * The slab allocator now guarantees aligned allocations for all power + * of two sizes. This covers most smaller XFS buffers, so just use + * kmalloc in this case. + * + * Don't bother with the vmalloc fallback for allocations of page size + * or less: vmalloc won't do any better. + */ + if (!(gfp_mask & __GFP_NORETRY)) + gfp_mask |= __GFP_NOFAIL; + if (size < PAGE_SIZE && is_power_of_2(size)) + return xfs_buf_alloc_kmem(bp, size, gfp_mask); + return xfs_buf_alloc_folio(bp, size, gfp_mask); } static int @@ -1083,11 +1098,17 @@ xfs_buf_ioend_handle_error( return false; } -/* returns false if the caller needs to resubmit the I/O, else true */ -static bool -__xfs_buf_ioend( +/* + * Complete a buffer read or write. + * + * Releases the buffer if the I/O was asynchronous. + */ +static void +xfs_buf_ioend( struct xfs_buf *bp) { + bool async = bp->b_flags & XBF_ASYNC; + trace_xfs_buf_iodone(bp, _RET_IP_); if (bp->b_flags & XBF_READ) { @@ -1101,14 +1122,16 @@ __xfs_buf_ioend( if (bp->b_flags & XBF_READ_AHEAD) percpu_counter_dec(&bp->b_target->bt_readahead_count); } else { - if (!bp->b_error) { + if (unlikely(bp->b_error)) { + if (xfs_buf_ioend_handle_error(bp)) { + ASSERT(async); + return; + } + } else { bp->b_flags &= ~XBF_WRITE_FAIL; bp->b_flags |= XBF_DONE; } - if (unlikely(bp->b_error) && xfs_buf_ioend_handle_error(bp)) - return false; - /* clear the retry state */ bp->b_last_error = 0; bp->b_retries = 0; @@ -1128,30 +1151,15 @@ __xfs_buf_ioend( bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD | _XBF_LOGRECOVERY); - return true; -} - -static void -xfs_buf_ioend( - struct xfs_buf *bp) -{ - if (!__xfs_buf_ioend(bp)) - return; - if (bp->b_flags & XBF_ASYNC) + if (async) xfs_buf_relse(bp); - else - complete(&bp->b_iowait); } static void xfs_buf_ioend_work( struct work_struct *work) { - struct xfs_buf *bp = - container_of(work, struct xfs_buf, b_ioend_work); - - if (__xfs_buf_ioend(bp)) - xfs_buf_relse(bp); + xfs_buf_ioend(container_of(work, struct xfs_buf, b_ioend_work)); } void @@ -1177,17 +1185,18 @@ xfs_buf_ioerror_alert( } /* - * To simulate an I/O failure, the buffer must be locked and held with at least - * two references. + * Fail a locked and referenced buffer outside the I/O path. * - * The buf item reference is dropped via ioend processing. The second reference - * is owned by the caller and is dropped on I/O completion if the buffer is - * XBF_ASYNC. + * The caller transfers a reference which will be released after processing the + * error. */ void -xfs_buf_ioend_fail( +xfs_buf_fail( struct xfs_buf *bp) { + ASSERT(xfs_buf_islocked(bp)); + + bp->b_flags |= XBF_ASYNC; bp->b_flags &= ~XBF_DONE; xfs_buf_stale(bp); xfs_buf_ioerror(bp, -EIO); @@ -1300,12 +1309,11 @@ xfs_buf_iowait( { ASSERT(!(bp->b_flags & XBF_ASYNC)); - do { - trace_xfs_buf_iowait(bp, _RET_IP_); - wait_for_completion(&bp->b_iowait); - trace_xfs_buf_iowait_done(bp, _RET_IP_); - } while (!__xfs_buf_ioend(bp)); + trace_xfs_buf_iowait(bp, _RET_IP_); + wait_for_completion(&bp->b_iowait); + trace_xfs_buf_iowait_done(bp, _RET_IP_); + xfs_buf_ioend(bp); return bp->b_error; } @@ -1369,8 +1377,8 @@ xfs_buf_submit( * on shutdown. */ if (bp->b_mount->m_log && xlog_is_shutdown(bp->b_mount->m_log)) { - xfs_buf_ioend_fail(bp); - return; + xfs_buf_ioerror(bp, -EIO); + goto ioerror; } if (bp->b_flags & XBF_WRITE) @@ -1383,18 +1391,26 @@ xfs_buf_submit( bp->b_error = 0; if ((bp->b_flags & XBF_WRITE) && !xfs_buf_verify_write(bp)) { + /* ->verify_write should have set b_error already */ xfs_force_shutdown(bp->b_mount, SHUTDOWN_CORRUPT_INCORE); - xfs_buf_ioend(bp); - return; + goto ioerror; } /* In-memory targets are directly mapped, no I/O required. */ - if (xfs_buftarg_is_mem(bp->b_target)) { - xfs_buf_ioend(bp); - return; - } + if (xfs_buftarg_is_mem(bp->b_target)) + goto end_io; xfs_buf_submit_bio(bp); + return; + +ioerror: + bp->b_flags &= ~XBF_DONE; + xfs_buf_stale(bp); +end_io: + if (bp->b_flags & XBF_ASYNC) + xfs_buf_ioend(bp); + else + complete(&bp->b_iowait); } /* @@ -1615,7 +1631,7 @@ xfs_free_buftarg( fs_put_dax(btp->bt_daxdev, btp->bt_mount); /* the main block device is closed by kill_block_super */ if (btp->bt_bdev != btp->bt_mount->m_super->s_bdev) - bdev_fput(btp->bt_file); + fs_bdev_file_release(btp->bt_file, btp->bt_mount->m_super); kfree(btp); } diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index b3cd1c7029f1..79cc9c3f0254 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -290,7 +290,7 @@ extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error, xfs_failaddr_t failaddr); #define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address) extern void xfs_buf_ioerror_alert(struct xfs_buf *bp, xfs_failaddr_t fa); -void xfs_buf_ioend_fail(struct xfs_buf *); +void xfs_buf_fail(struct xfs_buf *bp); void __xfs_buf_mark_corrupt(struct xfs_buf *bp, xfs_failaddr_t fa); #define xfs_buf_mark_corrupt(bp) __xfs_buf_mark_corrupt((bp), __this_address) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 8487635579e5..1f055cd6732e 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -549,8 +549,7 @@ xfs_buf_item_unpin( * wait for the lock and then run the IO failure completion. */ xfs_buf_lock(bp); - bp->b_flags |= XBF_ASYNC; - xfs_buf_ioend_fail(bp); + xfs_buf_fail(bp); return; } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 317eb57f989f..15279d22a894 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2646,8 +2646,7 @@ xfs_iflush_cluster( * inode cluster buffers. */ xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); - bp->b_flags |= XBF_ASYNC; - xfs_buf_ioend_fail(bp); + xfs_buf_fail(bp); return error; } diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 6339f4956ecb..4a3299abf774 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -293,8 +293,7 @@ xfs_vn_create( struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, - umode_t mode, - bool flags) + umode_t mode) { return xfs_generic_create(idmap, dir, dentry, mode, 0, NULL); } @@ -306,7 +305,7 @@ xfs_vn_mkdir( struct dentry *dentry, umode_t mode) { - return ERR_PTR(xfs_generic_create(idmap, dir, dentry, mode | S_IFDIR, 0, NULL)); + return ERR_PTR(xfs_generic_create(idmap, dir, dentry, mode, 0, NULL)); } STATIC struct dentry * @@ -338,7 +337,7 @@ STATIC struct dentry * xfs_vn_ci_lookup( struct inode *dir, struct dentry *dentry, - unsigned int flags) + unsigned int flags) { struct xfs_inode *ip; struct xfs_name xname; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 09e6678ca487..5f984bf5698a 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1028,7 +1028,7 @@ xlog_verify_head( { struct xlog_rec_header *tmp_rhead; char *tmp_buffer; - xfs_daddr_t first_bad; + xfs_daddr_t first_bad = XFS_BUF_DADDR_NULL; xfs_daddr_t tmp_rhead_blk; int found; int error; @@ -1057,7 +1057,8 @@ xlog_verify_head( */ error = xlog_do_recovery_pass(log, *head_blk, tmp_rhead_blk, XLOG_RECOVER_CRCPASS, &first_bad); - if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && first_bad) { + if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && + first_bad != XFS_BUF_DADDR_NULL) { /* * We've hit a potential torn write. Reset the error and warn * about it. @@ -3575,4 +3576,3 @@ xlog_recover_cancel( if (xlog_recovery_needed(log)) xlog_recover_cancel_intents(log); } - diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index aa0d2976f1c3..896b24f87ac9 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -166,10 +166,9 @@ xfs_qm_dqpurge( * does it on success. */ error = xfs_qm_dqflush(dqp, bp); - if (!error) { + if (!error) error = xfs_bwrite(bp); - xfs_buf_relse(bp); - } + xfs_buf_relse(bp); xfs_dqflock(dqp); } xfs_dquot_detach_buf(dqp); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index eac7f9503805..d1c622f0a957 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -400,8 +400,8 @@ xfs_blkdev_get( blk_mode_t mode; mode = sb_open_mode(mp->m_super->s_flags); - *bdev_filep = bdev_file_open_by_path(name, mode, - mp->m_super, &fs_holder_ops); + *bdev_filep = fs_bdev_file_open_by_path(name, mode, + mp->m_super, mp->m_super); if (IS_ERR(*bdev_filep)) { error = PTR_ERR(*bdev_filep); *bdev_filep = NULL; @@ -526,7 +526,7 @@ xfs_open_devices( mp->m_logdev_targp = mp->m_ddev_targp; /* Handle won't be used, drop it */ if (logdev_file) - bdev_fput(logdev_file); + fs_bdev_file_release(logdev_file, mp->m_super); } return 0; @@ -534,14 +534,17 @@ xfs_open_devices( out_free_rtdev_targ: if (mp->m_rtdev_targp) xfs_free_buftarg(mp->m_rtdev_targp); + mp->m_rtdev_targp = NULL; + rtdev_file = NULL; /* released by xfs_free_buftarg() */ out_free_ddev_targ: xfs_free_buftarg(mp->m_ddev_targp); + mp->m_ddev_targp = NULL; out_close_rtdev: if (rtdev_file) - bdev_fput(rtdev_file); + fs_bdev_file_release(rtdev_file, mp->m_super); out_close_logdev: if (logdev_file) - bdev_fput(logdev_file); + fs_bdev_file_release(logdev_file, mp->m_super); return error; } diff --git a/include/acpi/platform/aclinuxex.h b/include/acpi/platform/aclinuxex.h index aeb74e2f9d4f..760e1ded325c 100644 --- a/include/acpi/platform/aclinuxex.h +++ b/include/acpi/platform/aclinuxex.h @@ -134,6 +134,7 @@ static inline void acpi_os_terminate_debugger(void) /* * OSL interfaces added by Linux */ +#define acpi_ut_safe_strncpy strscpy_pad #endif /* __KERNEL__ */ diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h index ddd90bbe40df..94388da3490c 100644 --- a/include/asm-generic/audit_change_attr.h +++ b/include/asm-generic/audit_change_attr.h @@ -40,3 +40,6 @@ __NR_link, #ifdef __NR_linkat __NR_linkat, #endif +#ifdef __NR_file_setattr +__NR_file_setattr, +#endif diff --git a/include/asm-generic/audit_read.h b/include/asm-generic/audit_read.h index fb9991f53fb6..d8dc3dd6bf63 100644 --- a/include/asm-generic/audit_read.h +++ b/include/asm-generic/audit_read.h @@ -3,6 +3,9 @@ __NR_readlink, #endif __NR_quotactl, +#ifdef __NR_quotactl_fd +__NR_quotactl_fd, +#endif __NR_listxattr, #ifdef __NR_listxattrat __NR_listxattrat, @@ -18,3 +21,31 @@ __NR_fgetxattr, #ifdef __NR_readlinkat __NR_readlinkat, #endif +#ifdef __NR_file_getattr +__NR_file_getattr, +#endif +#ifdef __NR_stat +__NR_stat, +#endif +#ifdef __NR_stat64 +__NR_stat64, +#endif +#ifdef __NR_lstat +__NR_lstat, +#endif +#ifdef __NR_lstat64 +__NR_lstat64, +#endif +#ifdef __NR_fstat +__NR_fstat, +#endif +#ifdef __NR_fstat64 +__NR_fstat64, +#endif +#ifdef __NR_newfstatat +__NR_newfstatat, +#endif +#ifdef __NR_fstatat64 +__NR_fstatat64, +#endif +__NR_statx, diff --git a/include/asm-generic/audit_write.h b/include/asm-generic/audit_write.h index f9f1d0ae11d9..378128dc31e3 100644 --- a/include/asm-generic/audit_write.h +++ b/include/asm-generic/audit_write.h @@ -5,6 +5,9 @@ __NR_acct, __NR_swapon, #endif __NR_quotactl, +#ifdef __NR_quotactl_fd +__NR_quotactl_fd, +#endif #ifdef __NR_truncate __NR_truncate, #endif diff --git a/include/asm-generic/bitops/le.h b/include/asm-generic/bitops/le.h index d51beff60375..e3b0da9a25f1 100644 --- a/include/asm-generic/bitops/le.h +++ b/include/asm-generic/bitops/le.h @@ -16,47 +16,47 @@ #endif -static inline int test_bit_le(int nr, const void *addr) +static inline int test_bit_le(unsigned long nr, const void *addr) { return test_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline void set_bit_le(int nr, void *addr) +static inline void set_bit_le(unsigned long nr, void *addr) { set_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline void clear_bit_le(int nr, void *addr) +static inline void clear_bit_le(unsigned long nr, void *addr) { clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline void __set_bit_le(int nr, void *addr) +static inline void __set_bit_le(unsigned long nr, void *addr) { __set_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline void __clear_bit_le(int nr, void *addr) +static inline void __clear_bit_le(unsigned long nr, void *addr) { __clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline int test_and_set_bit_le(int nr, void *addr) +static inline int test_and_set_bit_le(unsigned long nr, void *addr) { return test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline int test_and_clear_bit_le(int nr, void *addr) +static inline int test_and_clear_bit_le(unsigned long nr, void *addr) { return test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline int __test_and_set_bit_le(int nr, void *addr) +static inline int __test_and_set_bit_le(unsigned long nr, void *addr) { return __test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr); } -static inline int __test_and_clear_bit_le(int nr, void *addr) +static inline int __test_and_clear_bit_le(unsigned long nr, void *addr) { return __test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr); } diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h index 29cab7947980..3ff832ebcea5 100644 --- a/include/asm-generic/fixmap.h +++ b/include/asm-generic/fixmap.h @@ -21,7 +21,7 @@ #define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) #define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * 'index to address' translation. If anyone tries to use the idx * directly without translation, we catch the bug with a NULL-deference @@ -97,5 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #define set_fixmap_io(idx, phys) \ __set_fixmap(idx, phys, FIXMAP_PAGE_IO) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_FIXMAP_H */ diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h index f2979e3a96b6..875ccae19683 100644 --- a/include/asm-generic/getorder.h +++ b/include/asm-generic/getorder.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_GETORDER_H #define __ASM_GENERIC_GETORDER_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -47,6 +47,6 @@ static __always_inline __attribute_const__ int get_order(unsigned long size) #endif } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_GENERIC_GETORDER_H */ diff --git a/include/asm-generic/kprobes.h b/include/asm-generic/kprobes.h index 5290a2b2e15a..16f16963d503 100644 --- a/include/asm-generic/kprobes.h +++ b/include/asm-generic/kprobes.h @@ -2,7 +2,7 @@ #ifndef _ASM_GENERIC_KPROBES_H #define _ASM_GENERIC_KPROBES_H -#if defined(__KERNEL__) && !defined(__ASSEMBLY__) +#if defined(__KERNEL__) && !defined(__ASSEMBLER__) #ifdef CONFIG_KPROBES /* * Blacklist ganerating macro. Specify functions which is not probed @@ -21,6 +21,6 @@ static unsigned long __used \ # define __kprobes # define nokprobe_inline inline #endif -#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ +#endif /* defined(__KERNEL__) && !defined(__ASSEMBLER__) */ #endif /* _ASM_GENERIC_KPROBES_H */ diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h index efa6610acbc7..c6b4eafaf4cb 100644 --- a/include/asm-generic/memory_model.h +++ b/include/asm-generic/memory_model.h @@ -4,7 +4,7 @@ #include -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * supports 3 memory models. @@ -53,7 +53,7 @@ static inline int pfn_valid(unsigned long pfn) */ #define __page_to_pfn(pg) \ ({ const struct page *__pg = (pg); \ - int __sec = memdesc_section(__pg->flags); \ + int __sec = memdesc_section(&__pg->flags); \ (unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec))); \ }) @@ -86,6 +86,6 @@ static inline int pfn_valid(unsigned long pfn) #endif /* CONFIG_DEBUG_VIRTUAL */ #define phys_to_page(phys) pfn_to_page(PHYS_PFN(phys)) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/asm-generic/mmu.h b/include/asm-generic/mmu.h index 061838037542..5f78971e3ac2 100644 --- a/include/asm-generic/mmu.h +++ b/include/asm-generic/mmu.h @@ -6,7 +6,7 @@ * This is the mmu.h header for nommu implementations. * Architectures with an MMU need something more complex. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef struct { unsigned long end_brk; diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h index 03b7dae47dd4..89c21f84cffb 100644 --- a/include/asm-generic/pgtable-nop4d.h +++ b/include/asm-generic/pgtable-nop4d.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOP4D_H #define _PGTABLE_NOP4D_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __PAGETABLE_P4D_FOLDED 1 @@ -54,5 +54,5 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) #undef p4d_addr_end #define p4d_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOP4D_H */ diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 8ffd64e7a24c..36b6490ed180 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPMD_H #define _PGTABLE_NOPMD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -68,6 +68,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #undef pmd_addr_end #define pmd_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPMD_H */ diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index eb70c6d7ceff..356cbfbaab24 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -2,7 +2,7 @@ #ifndef _PGTABLE_NOPUD_H #define _PGTABLE_NOPUD_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -62,5 +62,5 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) #undef pud_addr_end #define pud_addr_end(addr, end) (end) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _PGTABLE_NOPUD_H */ diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index b276f783494c..4569045e7139 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -91,8 +91,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) memcpy((void __force *)to, from, n); return 0; } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #endif /* CONFIG_UACCESS_MEMCPY */ /* diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index 5c6d9799f4e7..a6b03cfba0e2 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_VSYSCALL_H #define __ASM_GENERIC_VSYSCALL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __arch_get_vdso_u_time_data static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) @@ -30,6 +30,6 @@ static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vd } #endif /* __arch_sync_vdso_time_data */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __ASM_GENERIC_VSYSCALL_H */ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 7643ba954125..4e9ed8e73403 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -161,9 +161,17 @@ struct af_alg_ctx { unsigned int inflight; }; +struct af_alg_allowlist_entry { + const char *name; + bool privileged; +}; + int af_alg_register_type(const struct af_alg_type *type); int af_alg_unregister_type(const struct af_alg_type *type); +int af_alg_check_restriction(const char *name, + const struct af_alg_allowlist_entry allowlist[]); + int af_alg_release(struct socket *sock); void af_alg_release_parent(struct sock *sk); int af_alg_accept(struct sock *sk, struct socket *newsock, diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h index 016c74fb747c..802b143de83d 100644 --- a/include/cxl/cxl.h +++ b/include/cxl/cxl.h @@ -226,4 +226,6 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev, struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds, struct range *range); + +int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity); #endif /* __CXL_CXL_H__ */ diff --git a/include/cxl/pci.h b/include/cxl/pci.h new file mode 100644 index 000000000000..3e0000015871 --- /dev/null +++ b/include/cxl/pci.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ + +#ifndef __CXL_CXL_PCI_H__ +#define __CXL_CXL_PCI_H__ + +/* Register Block Identifier (RBI) */ +enum cxl_regloc_type { + CXL_REGLOC_RBI_EMPTY = 0, + CXL_REGLOC_RBI_COMPONENT, + CXL_REGLOC_RBI_VIRT, + CXL_REGLOC_RBI_MEMDEV, + CXL_REGLOC_RBI_PMU, + CXL_REGLOC_RBI_TYPES +}; + +struct cxl_register_map; +struct pci_dev; + +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, + struct cxl_register_map *map); +#endif diff --git a/include/drm/bridge/analogix_dp.h b/include/drm/bridge/analogix_dp.h index 854af692229b..0e0b87abee59 100644 --- a/include/drm/bridge/analogix_dp.h +++ b/include/drm/bridge/analogix_dp.h @@ -16,12 +16,21 @@ enum analogix_dp_devtype { EXYNOS_DP, RK3288_DP, RK3399_EDP, + RK3576_EDP, RK3588_EDP, }; -static inline bool is_rockchip(enum analogix_dp_devtype type) +static inline bool analogix_dp_is_rockchip(enum analogix_dp_devtype type) { - return type == RK3288_DP || type == RK3399_EDP || type == RK3588_EDP; + switch (type) { + case RK3288_DP: + case RK3399_EDP: + case RK3576_EDP: + case RK3588_EDP: + return true; + default: + return false; + } } struct analogix_dp_plat_data { diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h index 25363541e69d..22105c3e8e4d 100644 --- a/include/drm/bridge/dw_dp.h +++ b/include/drm/bridge/dw_dp.h @@ -24,4 +24,5 @@ struct dw_dp_plat_data { struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, const struct dw_dp_plat_data *plat_data); +void dw_dp_unbind(struct dw_dp *dp); #endif /* __DW_DP__ */ diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h index 829e4d98d61c..7154216e0aff 100644 --- a/include/drm/display/drm_dp.h +++ b/include/drm/display/drm_dp.h @@ -354,15 +354,24 @@ # define DP_DSC_20_PER_DP_DSC_SINK (1 << 1) # define DP_DSC_24_PER_DP_DSC_SINK (1 << 2) -#define DP_DSC_BITS_PER_PIXEL_INC 0x06F +#define DP_DSC_MAX_BPP_DELTA_VERSION_1 0x06E # define DP_DSC_RGB_YCbCr444_MAX_BPP_DELTA_MASK 0x1f -# define DP_DSC_RGB_YCbCr420_MAX_BPP_DELTA_MASK 0xe0 +# define DP_DSC_NATIVE_YCbCr420_MAX_BPP_DELTA_MASK 0xe0 + +# define DP_DSC_BPP_DELTA_444 16 +# define DP_DSC_BPP_DELTA_420 12 +# define DP_DSC_BPP_DELTA_SHIFT_420 5 + +#define DP_DSC_BITS_PER_PIXEL_INC 0x06F # define DP_DSC_BITS_PER_PIXEL_1_16 0x0 # define DP_DSC_BITS_PER_PIXEL_1_8 0x1 # define DP_DSC_BITS_PER_PIXEL_1_4 0x2 # define DP_DSC_BITS_PER_PIXEL_1_2 0x3 # define DP_DSC_BITS_PER_PIXEL_1_1 0x4 # define DP_DSC_BITS_PER_PIXEL_MASK 0x7 +# define DP_DSC_NATIVE_YCbCr422_MAX_BPP_DELTA_MASK 0x78 +# define DP_DSC_BPP_DELTA_NATIVE_SHIFT_422 3 +# define DP_DSC_BPP_DELTA_NATIVE_422 16 #define DP_PSR_SUPPORT 0x070 /* XXX 1.2? */ # define DP_PSR_IS_SUPPORTED 1 diff --git a/include/drm/display/drm_dp_helper.h b/include/drm/display/drm_dp_helper.h index 8c2d77a032f0..ab16c1be3900 100644 --- a/include/drm/display/drm_dp_helper.h +++ b/include/drm/display/drm_dp_helper.h @@ -115,6 +115,7 @@ struct drm_dp_vsc_sdp { * @duration_decr_ms: Successive frame duration decrease * @target_rr_divider: Target refresh rate divider * @mode: Adaptive Sync Operation Mode + * @coasting_vtotal: Coasting vtotal */ struct drm_dp_as_sdp { unsigned char sdp_type; diff --git a/include/drm/display/drm_hdmi_state_helper.h b/include/drm/display/drm_hdmi_state_helper.h index 0adc30c55ec9..13375bd0f4ae 100644 --- a/include/drm/display/drm_hdmi_state_helper.h +++ b/include/drm/display/drm_hdmi_state_helper.h @@ -11,8 +11,8 @@ struct hdmi_audio_infoframe; enum drm_connector_status; -void __drm_atomic_helper_connector_hdmi_reset(struct drm_connector *connector, - struct drm_connector_state *new_conn_state); +void __drm_atomic_helper_connector_hdmi_state_init(struct drm_connector *connector, + struct drm_connector_state *new_conn_state); int drm_atomic_helper_connector_hdmi_check(struct drm_connector *connector, struct drm_atomic_commit *state); diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h index 1a80a8cdf269..88087910ab1a 100644 --- a/include/drm/drm_atomic.h +++ b/include/drm/drm_atomic.h @@ -265,7 +265,10 @@ struct drm_private_state_funcs { * @atomic_create_state: * * Allocates a pristine, initialized, state for the private - * object and returns it. + * object and returns it. This callback must have no side + * effects: in particular, the returned state must not be + * assigned to the object's state pointer and it must not affect + * the hardware state. * * RETURNS: * diff --git a/include/drm/drm_atomic_helper.h b/include/drm/drm_atomic_helper.h index b84152810abb..4cfeec70d648 100644 --- a/include/drm/drm_atomic_helper.h +++ b/include/drm/drm_atomic_helper.h @@ -295,4 +295,11 @@ drm_atomic_helper_bridge_propagate_bus_fmt(struct drm_bridge *bridge, u32 output_fmt, unsigned int *num_input_fmts); +u32 * +drm_atomic_helper_bridge_get_hdmi_output_bus_fmts(struct drm_bridge *bridge, + struct drm_bridge_state *bridge_state, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state, + unsigned int *num_output_fmts); + #endif /* DRM_ATOMIC_HELPER_H_ */ diff --git a/include/drm/drm_atomic_state_helper.h b/include/drm/drm_atomic_state_helper.h index 61a3b38ad49f..34a599c3d86d 100644 --- a/include/drm/drm_atomic_state_helper.h +++ b/include/drm/drm_atomic_state_helper.h @@ -40,11 +40,13 @@ struct drm_private_state; struct drm_modeset_acquire_ctx; struct drm_device; -void __drm_atomic_helper_crtc_state_reset(struct drm_crtc_state *state, +void __drm_atomic_helper_crtc_state_init(struct drm_crtc_state *state, struct drm_crtc *crtc); void __drm_atomic_helper_crtc_reset(struct drm_crtc *crtc, struct drm_crtc_state *state); void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc); +struct drm_crtc_state * +drm_atomic_helper_crtc_create_state(struct drm_crtc *crtc); void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc, struct drm_crtc_state *state); struct drm_crtc_state * @@ -53,8 +55,10 @@ void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc_state *state); void drm_atomic_helper_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *state); -void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *state, +void __drm_atomic_helper_plane_state_init(struct drm_plane_state *state, struct drm_plane *plane); +struct drm_plane_state * +drm_atomic_helper_plane_create_state(struct drm_plane *plane); void __drm_atomic_helper_plane_reset(struct drm_plane *plane, struct drm_plane_state *state); void drm_atomic_helper_plane_reset(struct drm_plane *plane); @@ -66,11 +70,13 @@ void __drm_atomic_helper_plane_destroy_state(struct drm_plane_state *state); void drm_atomic_helper_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state); -void __drm_atomic_helper_connector_state_reset(struct drm_connector_state *conn_state, +void __drm_atomic_helper_connector_state_init(struct drm_connector_state *conn_state, struct drm_connector *connector); void __drm_atomic_helper_connector_reset(struct drm_connector *connector, struct drm_connector_state *conn_state); void drm_atomic_helper_connector_reset(struct drm_connector *connector); +struct drm_connector_state * +drm_atomic_helper_connector_create_state(struct drm_connector *connector); void drm_atomic_helper_connector_tv_reset(struct drm_connector *connector); int drm_atomic_helper_connector_tv_check(struct drm_connector *connector, struct drm_atomic_commit *state); @@ -96,7 +102,7 @@ struct drm_bridge_state * drm_atomic_helper_bridge_duplicate_state(struct drm_bridge *bridge); void drm_atomic_helper_bridge_destroy_state(struct drm_bridge *bridge, struct drm_bridge_state *state); -void __drm_atomic_helper_bridge_reset(struct drm_bridge *bridge, - struct drm_bridge_state *state); +void __drm_atomic_helper_bridge_state_init(struct drm_bridge_state *state, + struct drm_bridge *bridge); struct drm_bridge_state * -drm_atomic_helper_bridge_reset(struct drm_bridge *bridge); +drm_atomic_helper_bridge_create_state(struct drm_bridge *bridge); diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h index 4ba3a5deef9a..18f3db367dc1 100644 --- a/include/drm/drm_bridge.h +++ b/include/drm/drm_bridge.h @@ -504,31 +504,20 @@ struct drm_bridge_funcs { struct drm_connector_state *conn_state); /** - * @atomic_reset: + * @atomic_create_state: * - * Reset the bridge to a predefined state (or retrieve its current - * state) and return a &drm_bridge_state object matching this state. - * This function is called at attach time. - * - * The atomic_reset hook is mandatory if the bridge implements any of - * the atomic hooks, and should be left unassigned otherwise. For - * bridges that don't subclass &drm_bridge_state, the - * drm_atomic_helper_bridge_reset() helper function shall be used to - * implement this hook. - * - * Note that the atomic_reset() semantics is not exactly matching the - * reset() semantics found on other components (connector, plane, ...). - * - * 1. The reset operation happens when the bridge is attached, not when - * drm_mode_config_reset() is called - * 2. It's meant to be used exclusively on bridges that have been - * converted to the ATOMIC API + * Allocate a pristine, initialized, state for the bridge + * object and return it. This callback must have no side + * effects: in particular, the returned state must not be + * assigned to the object's state pointer and it must not affect + * the hardware state. * * RETURNS: - * A valid drm_bridge_state object in case of success, an ERR_PTR() - * giving the reason of the failure otherwise. + * + * A new, pristine, bridge state instance or an error pointer + * on failure. */ - struct drm_bridge_state *(*atomic_reset)(struct drm_bridge *bridge); + struct drm_bridge_state *(*atomic_create_state)(struct drm_bridge *bridge); /** * @detect: @@ -1256,6 +1245,10 @@ struct drm_bridge { * @hpd_mutex: Protects the @hpd_cb and @hpd_data fields. */ struct mutex hpd_mutex; + /** + * @hpd_state_mutex: Protects the HPD en/disablement state for the bridge. + */ + struct mutex hpd_state_mutex; /** * @hpd_cb: Hot plug detection callback, registered with * drm_bridge_hpd_enable(). @@ -1371,7 +1364,7 @@ drm_bridge_get_current_state(struct drm_bridge *bridge) * drm_atomic_private_obj_init(), so we need to make sure we're * working with one before we try to use the lock. */ - if (!bridge->funcs || !bridge->funcs->atomic_reset) + if (!bridge->funcs || !bridge->funcs->atomic_create_state) return NULL; drm_modeset_lock_assert_held(&bridge->base.lock); diff --git a/include/drm/drm_colorop.h b/include/drm/drm_colorop.h index d08a6a8a8392..224fae40ed2b 100644 --- a/include/drm/drm_colorop.h +++ b/include/drm/drm_colorop.h @@ -423,6 +423,8 @@ int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *col enum drm_colorop_lut3d_interpolation_type interpolation, uint32_t flags); +struct drm_colorop_state * +drm_atomic_helper_colorop_create_state(struct drm_colorop *colorop); struct drm_colorop_state * drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colorop); diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index 5ad62c207d00..af075c38f4db 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -571,12 +571,80 @@ enum drm_colorspace { * YCbCr 4:2:2 output format (ie. with horizontal subsampling) * @DRM_OUTPUT_COLOR_FORMAT_YCBCR420: * YCbCr 4:2:0 output format (ie. with horizontal and vertical subsampling) + * @DRM_OUTPUT_COLOR_FORMAT_COUNT: + * Number of valid output color format values in this enum */ enum drm_output_color_format { DRM_OUTPUT_COLOR_FORMAT_RGB444 = 0, DRM_OUTPUT_COLOR_FORMAT_YCBCR444, DRM_OUTPUT_COLOR_FORMAT_YCBCR422, DRM_OUTPUT_COLOR_FORMAT_YCBCR420, + DRM_OUTPUT_COLOR_FORMAT_COUNT, +}; + +/** + * enum drm_connector_color_format - Connector Color Format Request + * + * This enum, unlike &enum drm_output_color_format, is used to specify requests + * for a specific color format on a connector through the DRM "color format" + * property. The difference is that it has an "AUTO" value to specify that + * no specific choice has been made. + */ +enum drm_connector_color_format { + /** + * @DRM_CONNECTOR_COLOR_FORMAT_AUTO: The driver or display protocol + * helpers should pick a suitable color format. All implementations of a + * specific display protocol must behave the same way with "AUTO", but + * different display protocols do not necessarily have the same "AUTO" + * semantics. + * + * For HDMI, "AUTO" picks RGB, but falls back to YCbCr 4:2:0 if the + * bandwidth required for full-scale RGB is not available, or the mode + * is YCbCr 4:2:0-only, as long as the mode and output both support + * YCbCr 4:2:0. + * + * For display protocols other than HDMI, the recursive bridge chain + * format selection picks the first chain of bridge formats that works, + * as has already been the case before the introduction of the "color + * format" property. Non-HDMI bridges should therefore either sort their + * bus output formats by preference, or agree on a unified auto format + * selection logic that's implemented in a common state helper (like + * how HDMI does it). + */ + DRM_CONNECTOR_COLOR_FORMAT_AUTO = 0, + + /** + * @DRM_CONNECTOR_COLOR_FORMAT_RGB444: RGB output format. The + * quantization range depends on the value of the "Broadcast RGB" + * property if it is present on the connector. + */ + DRM_CONNECTOR_COLOR_FORMAT_RGB444, + + /** + * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR444: YCbCr 4:4:4 output format (ie. + * not subsampled). Quantization range is "Limited" by default. + */ + DRM_CONNECTOR_COLOR_FORMAT_YCBCR444, + + /** + * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR422: YCbCr 4:2:2 output format (ie. + * with horizontal subsampling). Quantization range is "Limited" by + * default. + */ + DRM_CONNECTOR_COLOR_FORMAT_YCBCR422, + + /** + * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR420: YCbCr 4:2:0 output format (ie. + * with horizontal and vertical subsampling). Quantization range is + * "Limited" by default. + */ + DRM_CONNECTOR_COLOR_FORMAT_YCBCR420, + + /** + * @DRM_CONNECTOR_COLOR_FORMAT_COUNT: Number of valid connector color + * format values in this enum + */ + DRM_CONNECTOR_COLOR_FORMAT_COUNT, }; const char * @@ -921,6 +989,12 @@ struct drm_display_info { * @amd_vsdb: AMD-specific VSDB information. */ struct drm_amd_vsdb_info amd_vsdb; + + /** + * @panel_type: Panel type from DisplayID Display Parameters + * Data Block (tag 0x21). Uses DRM_MODE_PANEL_TYPE_* constants. + */ + u8 panel_type; }; int drm_display_info_set_bus_formats(struct drm_display_info *info, @@ -1167,6 +1241,13 @@ struct drm_connector_state { */ enum drm_colorspace colorspace; + /** + * @color_format: State variable for Connector property to request + * color format change on Sink. This is most commonly used to switch + * between RGB to YUV and vice-versa. + */ + enum drm_connector_color_format color_format; + /** * @writeback_job: Writeback job for writeback connectors * @@ -1571,6 +1652,22 @@ struct drm_connector_funcs { */ void (*destroy)(struct drm_connector *connector); + /** + * @atomic_create_state: + * + * Allocate a pristine, initialized, state for the connector + * object and return it. This callback must have no side + * effects: in particular, the returned state must not be + * assigned to the object's state pointer and it must not affect + * the hardware state. + * + * RETURNS: + * + * A new, pristine, connector state instance or an error pointer + * on failure. + */ + struct drm_connector_state *(*atomic_create_state)(struct drm_connector *connector); + /** * @atomic_duplicate_state: * @@ -1712,6 +1809,16 @@ struct drm_connector_funcs { * Allows connectors to create connector-specific debugfs files. */ void (*debugfs_init)(struct drm_connector *connector, struct dentry *root); + + /** + * @color_format: + * + * Allows connectors to return a connector color format other than + * @conn_state.color_format for purposes of e.g. display protocol + * specific helper logic having already mapped it to an output format. + */ + enum drm_connector_color_format (*color_format)( + const struct drm_connector_state *conn_state); }; /** @@ -2165,6 +2272,12 @@ struct drm_connector { */ struct drm_property *colorspace_property; + /** + * @color_format_property: Connector property to set the suitable + * color format supported by the sink. + */ + struct drm_property *color_format_property; + /** * @path_blob_ptr: * @@ -2522,6 +2635,8 @@ drm_connector_is_unregistered(struct drm_connector *connector) void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode, enum drm_connector_status status); +enum drm_connector_color_format +drm_connector_get_color_format(const struct drm_connector_state *conn_state); const char *drm_get_connector_type_name(unsigned int connector_type); const char *drm_get_connector_status_name(enum drm_connector_status status); const char *drm_get_subpixel_order_name(enum subpixel_order order); @@ -2608,13 +2723,13 @@ struct drm_tile_group { struct kref refcount; struct drm_device *dev; int id; - u8 group_data[8]; + u8 group_data[9]; }; struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev, - const char topology[8]); + const char topology_id[9]); struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev, - const char topology[8]); + const char topology_id[9]); void drm_mode_put_tile_group(struct drm_device *dev, struct drm_tile_group *tg); @@ -2648,6 +2763,9 @@ bool drm_connector_has_possible_encoder(struct drm_connector *connector, struct drm_encoder *encoder); const char *drm_get_colorspace_name(enum drm_colorspace colorspace); +int drm_connector_attach_color_format_property(struct drm_connector *connector, + unsigned long supported_color_formats); + /** * drm_for_each_connector_iter - connector_list iterator macro * @connector: &struct drm_connector pointer used as cursor diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h index c6dbe8b7db9e..152349f973e3 100644 --- a/include/drm/drm_crtc.h +++ b/include/drm/drm_crtc.h @@ -638,6 +638,22 @@ struct drm_crtc_funcs { int (*set_property)(struct drm_crtc *crtc, struct drm_property *property, uint64_t val); + /** + * @atomic_create_state: + * + * Allocate a pristine, initialized, state for the CRTC object + * and return it. This callback must have no side effects: in + * particular, the returned state must not be assigned to the + * object's state pointer and it must not affect the hardware + * state. + * + * RETURNS: + * + * A new, pristine, CRTC state instance or an error pointer + * on failure. + */ + struct drm_crtc_state *(*atomic_create_state)(struct drm_crtc *crtc); + /** * @atomic_duplicate_state: * diff --git a/include/drm/drm_debugfs.h b/include/drm/drm_debugfs.h index ea8cba94208a..eb93512b0f23 100644 --- a/include/drm/drm_debugfs.h +++ b/include/drm/drm_debugfs.h @@ -48,7 +48,7 @@ * For each DRM GPU VA space drivers should call drm_debugfs_gpuva_info() from * their @show callback. */ -#define DRM_DEBUGFS_GPUVA_INFO(show, data) {"gpuvas", show, DRIVER_GEM_GPUVA, data} +#define DRM_DEBUGFS_GPUVA_INFO(show, data) {"gpuvas", show, 0, data} /** * struct drm_info_list - debugfs info list entry diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h index 42fc085f986d..e09559495c5b 100644 --- a/include/drm/drm_drv.h +++ b/include/drm/drm_drv.h @@ -107,12 +107,6 @@ enum drm_driver_feature { * acceleration should be handled by two drivers that are connected using auxiliary bus. */ DRIVER_COMPUTE_ACCEL = BIT(7), - /** - * @DRIVER_GEM_GPUVA: - * - * Driver supports user defined GPU VA bindings for GEM objects. - */ - DRIVER_GEM_GPUVA = BIT(8), /** * @DRIVER_CURSOR_HOTSPOT: * diff --git a/include/drm/drm_fixed.h b/include/drm/drm_fixed.h index 33de514a5221..21d822aeed55 100644 --- a/include/drm/drm_fixed.h +++ b/include/drm/drm_fixed.h @@ -79,7 +79,8 @@ static inline u32 dfixed_div(fixed20_12 A, fixed20_12 B) #define DRM_FIXED_ALMOST_ONE (DRM_FIXED_ONE - DRM_FIXED_EPSILON) /** - * @drm_sm2fixp + * drm_sm2fixp() - convert signed-magnitude to fixed point + * @a: 1.31.32 signed-magnitude fixed point * * Convert a 1.31.32 signed-magnitude fixed point to 32.32 * 2s-complement fixed point diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h index 8a704f6a65c1..885244e375d3 100644 --- a/include/drm/drm_gem.h +++ b/include/drm/drm_gem.h @@ -661,9 +661,6 @@ static inline bool drm_gem_is_imported(const struct drm_gem_object *obj) * * This initializes the &drm_gem_object's &drm_gpuvm_bo list. * - * Calling this function is only necessary for drivers intending to support the - * &drm_driver_feature DRIVER_GEM_GPUVA. - * * See also drm_gem_gpuva_set_lock(). */ static inline void drm_gem_gpuva_init(struct drm_gem_object *obj) diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index b2c23af628e1..bcbb92f3fe9b 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -142,6 +142,10 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); +int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle); + extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index 655bd9104ffb..38221d83285b 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -159,8 +159,6 @@ struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, u64 addr, u64 range); struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, u64 addr, u64 range); -struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start); -struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end); /** * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is @@ -930,6 +928,8 @@ struct drm_gpuva_op_unmap { * If either a new mapping's start address is aligned with the start address * of the old mapping or the new mapping's end address is aligned with the * end address of the old mapping, either @prev or @next is NULL. + * This will also be the case when the requested mapping begins before the + * old mapping's start address or stretches beyond its end address. * * Note, the reason for a dedicated remap operation, rather than arbitrary * unmap and map operations, is to give drivers the chance of extracting driver diff --git a/include/drm/drm_managed.h b/include/drm/drm_managed.h index 72bfac002c06..72d0d68be226 100644 --- a/include/drm/drm_managed.h +++ b/include/drm/drm_managed.h @@ -18,7 +18,7 @@ typedef void (*drmres_release_t)(struct drm_device *dev, void *res); * @action: function which should be called when @dev is released * @data: opaque pointer, passed to @action * - * This function adds the @release action with optional parameter @data to the + * This function adds the release @action with optional parameter @data to the * list of cleanup actions for @dev. The cleanup actions will be run in reverse * order in the final drm_dev_put() call for @dev. */ diff --git a/include/drm/drm_mipi_dsi.h b/include/drm/drm_mipi_dsi.h index b429acde4f71..7ff43967251e 100644 --- a/include/drm/drm_mipi_dsi.h +++ b/include/drm/drm_mipi_dsi.h @@ -138,6 +138,8 @@ struct mipi_dsi_host *of_find_mipi_dsi_host_by_node(struct device_node *node); #define MIPI_DSI_MODE_LPM BIT(11) /* transmit data ending at the same time for all lanes within one hsync */ #define MIPI_DSI_HS_PKT_END_ALIGNED BIT(12) +/* pack all DSC slices for a line into a single packet */ +#define MIPI_DSI_MODE_DSC_ALL_SLICES_IN_PKT BIT(13) enum mipi_dsi_pixel_format { MIPI_DSI_FMT_RGB888, diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h index e584652ddf67..d8f5b7e9673e 100644 --- a/include/drm/drm_mode_config.h +++ b/include/drm/drm_mode_config.h @@ -1007,6 +1007,7 @@ static inline int drm_mode_config_init(struct drm_device *dev) return drmm_mode_config_init(dev); } +int drm_mode_config_create_initial_state(struct drm_device *dev); void drm_mode_config_reset(struct drm_device *dev); void drm_mode_config_cleanup(struct drm_device *dev); diff --git a/include/drm/drm_modes.h b/include/drm/drm_modes.h index b9bb92e4b029..6e3eccc3c349 100644 --- a/include/drm/drm_modes.h +++ b/include/drm/drm_modes.h @@ -193,6 +193,7 @@ enum drm_mode_status { #define DRM_MODE_MATCH_FLAGS (1 << 2) #define DRM_MODE_MATCH_3D_FLAGS (1 << 3) #define DRM_MODE_MATCH_ASPECT_RATIO (1 << 4) +#define DRM_MODE_MATCH_TIMINGS_VRR (1 << 5) /** * struct drm_display_mode - DRM kernel-internal display mode structure diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h index 419c88c873a6..2c5a5a70a71b 100644 --- a/include/drm/drm_plane.h +++ b/include/drm/drm_plane.h @@ -388,6 +388,22 @@ struct drm_plane_funcs { int (*set_property)(struct drm_plane *plane, struct drm_property *property, uint64_t val); + /** + * @atomic_create_state: + * + * Allocate a pristine, initialized, state for the plane object + * and return it. This callback must have no side effects: in + * particular, the returned state must not be assigned to the + * object's state pointer and it must not affect the hardware + * state. + * + * RETURNS: + * + * A new, pristine, plane state instance or an error pointer + * on failure. + */ + struct drm_plane_state *(*atomic_create_state)(struct drm_plane *plane); + /** * @atomic_duplicate_state: * diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h index ab017b05e175..2adc5ac688e1 100644 --- a/include/drm/drm_print.h +++ b/include/drm/drm_print.h @@ -87,7 +87,7 @@ extern unsigned long __drm_debug; * - drm.debug=0x2 will enable DRIVER messages * - drm.debug=0x3 will enable CORE and DRIVER messages * - ... - * - drm.debug=0x1ff will enable all messages + * - drm.debug=0x3ff will enable all messages * * An interesting feature is that it's possible to enable verbose logging at * run-time by echoing the debug value in its sysfs node:: diff --git a/include/drm/drm_ras.h b/include/drm/drm_ras.h index f2a787bc4f64..0beede3ddc4e 100644 --- a/include/drm/drm_ras.h +++ b/include/drm/drm_ras.h @@ -6,6 +6,8 @@ #ifndef __DRM_RAS_H__ #define __DRM_RAS_H__ +#include + #include /** diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index d61c19e78182..363d13fc929f 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -217,8 +217,7 @@ struct drm_sched_entity { * @stopped: * * Marks the enity as removed from rq and destined for - * termination. This is set by calling drm_sched_entity_flush() and by - * drm_sched_fini(). + * termination. This is set by calling drm_sched_entity_flush(). */ bool stopped; diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h index 39991afeb173..de395df9ca30 100644 --- a/include/drm/intel/display_parent_interface.h +++ b/include/drm/intel/display_parent_interface.h @@ -167,7 +167,9 @@ struct intel_display_overlay_interface { struct intel_display_panic_interface { struct intel_panic *(*alloc)(void); - int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb); + int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb, + struct drm_gem_object *obj, + unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width)); void (*finish)(struct intel_panic *panic); }; diff --git a/include/drm/intel/mchbar_regs.h b/include/drm/intel/mchbar_regs.h index ca0d421be16c..66498ca5e40b 100644 --- a/include/drm/intel/mchbar_regs.h +++ b/include/drm/intel/mchbar_regs.h @@ -6,8 +6,6 @@ #ifndef __INTEL_MCHBAR_REGS__ #define __INTEL_MCHBAR_REGS__ -#include "i915_reg_defs.h" - /* * MCHBAR mirror. * diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h index e32ef763427c..dff389b56eb3 100644 --- a/include/drm/intel/pciids.h +++ b/include/drm/intel/pciids.h @@ -893,8 +893,9 @@ MACRO__(0xD741, ## __VA_ARGS__), \ MACRO__(0xD742, ## __VA_ARGS__), \ MACRO__(0xD743, ## __VA_ARGS__), \ - MACRO__(0xD744, ## __VA_ARGS__), \ - MACRO__(0xD745, ## __VA_ARGS__) + MACRO__(0xD745, ## __VA_ARGS__), \ + MACRO__(0xD74A, ## __VA_ARGS__), \ + MACRO__(0xD74B, ## __VA_ARGS__) /* CRI */ #define INTEL_CRI_IDS(MACRO__, ...) \ diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h index 1af73c0ad41c..2cf433f61d5e 100644 --- a/include/dt-bindings/arm/qcom,ids.h +++ b/include/dt-bindings/arm/qcom,ids.h @@ -184,6 +184,7 @@ #define QCOM_ID_IPQ8078 344 #define QCOM_ID_SDM636 345 #define QCOM_ID_SDA636 346 +#define QCOM_ID_SDM850 348 #define QCOM_ID_SDM632 349 #define QCOM_ID_SDA632 350 #define QCOM_ID_SDA450 351 diff --git a/include/dt-bindings/clock/canaan,k230-clk.h b/include/dt-bindings/clock/canaan,k230-clk.h new file mode 100644 index 000000000000..3b916678cc5b --- /dev/null +++ b/include/dt-bindings/clock/canaan,k230-clk.h @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Kendryte Canaan K230 Clock Drivers + * + * Author: Xukai Wang + */ + +#ifndef __DT_BINDINGS_CANAAN_K230_CLOCK_H__ +#define __DT_BINDINGS_CANAAN_K230_CLOCK_H__ + +#define K230_CPU0_SRC_GATE 0 +#define K230_CPU0_PLIC_GATE 1 +#define K230_CPU0_NOC_DDRCP4_GATE 2 +#define K230_CPU0_APB_GATE 3 +#define K230_CPU0_SRC_RATE 4 +#define K230_CPU0_AXI_RATE 5 +#define K230_CPU0_PLIC_RATE 6 +#define K230_CPU0_APB_RATE 7 +#define K230_HS_SSI0_MUX 8 +#define K230_HS_USB_REF_MUX 9 +#define K230_HS_HCLK_HIGH_GATE 10 +#define K230_HS_HCLK_GATE 11 +#define K230_HS_SD0_AHB_GATE 12 +#define K230_HS_SD1_AHB_GATE 13 +#define K230_HS_SSI1_AHB_GATE 14 +#define K230_HS_SSI2_AHB_GATE 15 +#define K230_HS_USB0_AHB_GATE 16 +#define K230_HS_USB1_AHB_GATE 17 +#define K230_HS_SSI0_AXI_GATE 18 +#define K230_HS_SSI1_GATE 19 +#define K230_HS_SSI2_GATE 20 +#define K230_HS_QSPI_AXI_SRC_GATE 21 +#define K230_HS_SSI1_AXI_GATE 22 +#define K230_HS_SSI2_AXI_GATE 23 +#define K230_HS_SD_CARD_SRC_GATE 24 +#define K230_HS_SD0_CARD_GATE 25 +#define K230_HS_SD1_CARD_GATE 26 +#define K230_HS_SD_AXI_SRC_GATE 27 +#define K230_HS_SD0_AXI_GATE 28 +#define K230_HS_SD1_AXI_GATE 29 +#define K230_HS_SD0_BASE_GATE 30 +#define K230_HS_SD1_BASE_GATE 31 +#define K230_HS_SSI0_GATE 32 +#define K230_HS_SD_TIMER_SRC_GATE 33 +#define K230_HS_SD0_TIMER_GATE 34 +#define K230_HS_SD1_TIMER_GATE 35 +#define K230_HS_USB0_REF_GATE 36 +#define K230_HS_USB1_REF_GATE 37 +#define K230_HS_HCLK_HIGH_RATE 38 +#define K230_HS_HCLK_RATE 39 +#define K230_HS_SSI0_AXI_RATE 40 +#define K230_HS_SSI1_RATE 41 +#define K230_HS_SSI2_RATE 42 +#define K230_HS_QSPI_AXI_SRC_RATE 43 +#define K230_HS_SD_CARD_SRC_RATE 44 +#define K230_HS_SD_AXI_SRC_RATE 45 +#define K230_HS_USB_REF_50M_RATE 46 +#define K230_HS_SD_TIMER_SRC_RATE 47 +#define K230_TIMER0_MUX 48 +#define K230_TIMER1_MUX 49 +#define K230_TIMER2_MUX 50 +#define K230_TIMER3_MUX 51 +#define K230_TIMER4_MUX 52 +#define K230_TIMER5_MUX 53 +#define K230_SHRM_SRAM_MUX 54 +#define K230_DDRC_SRC_MUX 55 +#define K230_AI_SRC_MUX 56 +#define K230_CAMERA0_MUX 57 +#define K230_CAMERA1_MUX 58 +#define K230_CAMERA2_MUX 59 +#define K230_CPU1_SRC_MUX 60 +#define K230_CPU1_SRC_GATE 61 +#define K230_CPU1_PLIC_GATE 62 +#define K230_CPU1_APB_GATE 63 +#define K230_CPU1_SRC_RATE 64 +#define K230_CPU1_AXI_RATE 65 +#define K230_CPU1_PLIC_RATE 66 +#define K230_PMU_APB_GATE 67 +#define K230_LS_APB_SRC_GATE 68 +#define K230_LS_UART0_APB_GATE 69 +#define K230_LS_UART1_APB_GATE 70 +#define K230_LS_UART2_APB_GATE 71 +#define K230_LS_UART3_APB_GATE 72 +#define K230_LS_UART4_APB_GATE 73 +#define K230_LS_I2C0_APB_GATE 74 +#define K230_LS_I2C1_APB_GATE 75 +#define K230_LS_I2C2_APB_GATE 76 +#define K230_LS_I2C3_APB_GATE 77 +#define K230_LS_I2C4_APB_GATE 78 +#define K230_LS_GPIO_APB_GATE 79 +#define K230_LS_PWM_APB_GATE 80 +#define K230_LS_JAMLINK0_APB_GATE 81 +#define K230_LS_JAMLINK1_APB_GATE 82 +#define K230_LS_JAMLINK2_APB_GATE 83 +#define K230_LS_JAMLINK3_APB_GATE 84 +#define K230_LS_AUDIO_APB_GATE 85 +#define K230_LS_ADC_APB_GATE 86 +#define K230_LS_CODEC_APB_GATE 87 +#define K230_LS_I2C0_GATE 88 +#define K230_LS_I2C1_GATE 89 +#define K230_LS_I2C2_GATE 90 +#define K230_LS_I2C3_GATE 91 +#define K230_LS_I2C4_GATE 92 +#define K230_LS_CODEC_ADC_GATE 93 +#define K230_LS_CODEC_DAC_GATE 94 +#define K230_LS_AUDIO_DEV_GATE 95 +#define K230_LS_PDM_GATE 96 +#define K230_LS_ADC_GATE 97 +#define K230_LS_UART0_GATE 98 +#define K230_LS_UART1_GATE 99 +#define K230_LS_UART2_GATE 100 +#define K230_LS_UART3_GATE 101 +#define K230_LS_UART4_GATE 102 +#define K230_LS_JAMLINK0CO_GATE 103 +#define K230_LS_JAMLINK1CO_GATE 104 +#define K230_LS_JAMLINK2CO_GATE 105 +#define K230_LS_JAMLINK3CO_GATE 106 +#define K230_LS_GPIO_DEBOUNCE_GATE 107 +#define K230_SYSCTL_WDT0_APB_GATE 108 +#define K230_SYSCTL_WDT1_APB_GATE 109 +#define K230_SYSCTL_TIMER_APB_GATE 110 +#define K230_SYSCTL_IOMUX_APB_GATE 111 +#define K230_SYSCTL_MAILBOX_APB_GATE 112 +#define K230_SYSCTL_HDI_GATE 113 +#define K230_SYSCTL_TIME_STAMP_GATE 114 +#define K230_SYSCTL_WDT0_GATE 115 +#define K230_SYSCTL_WDT1_GATE 116 +#define K230_TIMER0_GATE 117 +#define K230_TIMER1_GATE 118 +#define K230_TIMER2_GATE 119 +#define K230_TIMER3_GATE 120 +#define K230_TIMER4_GATE 121 +#define K230_TIMER5_GATE 122 +#define K230_SHRM_APB_GATE 123 +#define K230_SHRM_AXI_GATE 124 +#define K230_SHRM_AXI_SLAVE_GATE 125 +#define K230_SHRM_NONAI2D_AXI_GATE 126 +#define K230_SHRM_SRAM_GATE 127 +#define K230_SHRM_DECOMPRESS_AXI_GATE 128 +#define K230_SHRM_SDMA_AXI_GATE 129 +#define K230_SHRM_PDMA_AXI_GATE 130 +#define K230_DDRC_SRC_GATE 131 +#define K230_DDRC_BYPASS_GATE 132 +#define K230_DDRC_APB_GATE 133 +#define K230_DISPLAY_AHB_GATE 134 +#define K230_DISPLAY_AXI_GATE 135 +#define K230_DISPLAY_GPU_GATE 136 +#define K230_DISPLAY_DPIP_GATE 137 +#define K230_DISPLAY_CFG_GATE 138 +#define K230_DISPLAY_REF_GATE 139 +#define K230_USB_480M_GATE 140 +#define K230_USB_100M_GATE 141 +#define K230_DPHY_DFT_GATE 142 +#define K230_SPI2AXI_GATE 143 +#define K230_AI_SRC_GATE 144 +#define K230_AI_AXI_GATE 145 +#define K230_AI_SRC_RATE 146 +#define K230_CAMERA0_GATE 147 +#define K230_CAMERA1_GATE 148 +#define K230_CAMERA2_GATE 149 +#define K230_LS_APB_SRC_RATE 150 +#define K230_LS_I2C0_RATE 151 +#define K230_LS_I2C1_RATE 152 +#define K230_LS_I2C2_RATE 153 +#define K230_LS_I2C3_RATE 154 +#define K230_LS_I2C4_RATE 155 +#define K230_LS_CODEC_ADC_RATE 156 +#define K230_LS_CODEC_DAC_RATE 157 +#define K230_LS_AUDIO_DEV_RATE 158 +#define K230_LS_PDM_RATE 159 +#define K230_LS_ADC_RATE 160 +#define K230_LS_UART0_RATE 161 +#define K230_LS_UART1_RATE 162 +#define K230_LS_UART2_RATE 163 +#define K230_LS_UART3_RATE 164 +#define K230_LS_UART4_RATE 165 +#define K230_LS_JAMLINKCO_SRC_RATE 166 +#define K230_LS_GPIO_DEBOUNCE_RATE 167 +#define K230_SYSCTL_HDI_RATE 168 +#define K230_SYSCTL_TIME_STAMP_RATE 169 +#define K230_SYSCTL_TEMP_SENSOR_RATE 170 +#define K230_SYSCTL_WDT0_RATE 171 +#define K230_SYSCTL_WDT1_RATE 172 +#define K230_TIMER0_SRC_RATE 173 +#define K230_TIMER1_SRC_RATE 174 +#define K230_TIMER2_SRC_RATE 175 +#define K230_TIMER3_SRC_RATE 176 +#define K230_TIMER4_SRC_RATE 177 +#define K230_TIMER5_SRC_RATE 178 +#define K230_SHRM_APB_RATE 179 +#define K230_DDRC_SRC_RATE 180 +#define K230_DDRC_APB_RATE 181 +#define K230_DISPLAY_AHB_RATE 182 +#define K230_DISPLAY_CLKEXT_RATE 183 +#define K230_DISPLAY_GPU_RATE 184 +#define K230_DISPLAY_DPIP_RATE 185 +#define K230_DISPLAY_CFG_RATE 186 +#define K230_VPU_SRC_GATE 187 +#define K230_VPU_AXI_GATE 188 +#define K230_VPU_DDRCP2_GATE 189 +#define K230_VPU_CFG_GATE 190 +#define K230_VPU_SRC_RATE 191 +#define K230_VPU_AXI_SRC_RATE 192 +#define K230_VPU_CFG_RATE 193 +#define K230_SEC_APB_GATE 194 +#define K230_SEC_FIX_GATE 195 +#define K230_SEC_AXI_GATE 196 +#define K230_SEC_APB_RATE 197 +#define K230_SEC_FIX_RATE 198 +#define K230_SEC_AXI_RATE 199 +#define K230_USB_480M_RATE 200 +#define K230_USB_100M_RATE 201 +#define K230_DPHY_DFT_RATE 202 +#define K230_SPI2AXI_RATE 203 +#define K230_CAMERA0_RATE 204 +#define K230_CAMERA1_RATE 205 +#define K230_CAMERA2_RATE 206 +#define K230_SHRM_SRAM_DIV2 207 + +#endif /* __DT_BINDINGS_CANAAN_K230_CLOCK_H__ */ diff --git a/include/dt-bindings/clock/qcom,hawi-gcc.h b/include/dt-bindings/clock/qcom,hawi-gcc.h new file mode 100644 index 000000000000..6cd7fa0884f5 --- /dev/null +++ b/include/dt-bindings/clock/qcom,hawi-gcc.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GCC_HAWI_H +#define _DT_BINDINGS_CLK_QCOM_GCC_HAWI_H + +/* GCC clocks */ +#define GCC_AGGRE_NOC_PCIE_AXI_CLK 0 +#define GCC_AGGRE_STARDUSTNOC_USB3_PRIM_AXI_CLK 1 +#define GCC_AGGRE_UFS_PHY_AXI_CLK 2 +#define GCC_BOOT_ROM_AHB_CLK 3 +#define GCC_CAM_BIST_MCLK_AHB_CLK 4 +#define GCC_CAMERA_AHB_CLK 5 +#define GCC_CAMERA_HF_AXI_CLK 6 +#define GCC_CAMERA_RSC_CORE_CLK 7 +#define GCC_CAMERA_SF_AXI_CLK 8 +#define GCC_CAMERA_XO_CLK 9 +#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK 10 +#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK 11 +#define GCC_CNOC_PCIE_SF_AXI_CLK 12 +#define GCC_EVA_AHB_CLK 13 +#define GCC_EVA_AXI0_CLK 14 +#define GCC_EVA_AXI0C_CLK 15 +#define GCC_EVA_XO_CLK 16 +#define GCC_GP1_CLK 17 +#define GCC_GP1_CLK_SRC 18 +#define GCC_GP2_CLK 19 +#define GCC_GP2_CLK_SRC 20 +#define GCC_GP3_CLK 21 +#define GCC_GP3_CLK_SRC 22 +#define GCC_GPLL0 23 +#define GCC_GPLL0_OUT_EVEN 24 +#define GCC_GPLL4 25 +#define GCC_GPLL5 26 +#define GCC_GPLL7 27 +#define GCC_GPLL9 28 +#define GCC_GPU_CFG_AHB_CLK 29 +#define GCC_GPU_GEMNOC_GFX_CLK 30 +#define GCC_GPU_GPLL0_CLK_SRC 31 +#define GCC_GPU_GPLL0_DIV_CLK_SRC 32 +#define GCC_GPU_RSC_CORE_CLK 33 +#define GCC_GPU_SMMU_VOTE_CLK 34 +#define GCC_MMU_TCU_VOTE_CLK 35 +#define GCC_PCIE_0_AUX_CLK 36 +#define GCC_PCIE_0_AUX_CLK_SRC 37 +#define GCC_PCIE_0_CFG_AHB_CLK 38 +#define GCC_PCIE_0_MSTR_AXI_CLK 39 +#define GCC_PCIE_0_PHY_AUX_CLK 40 +#define GCC_PCIE_0_PHY_AUX_CLK_SRC 41 +#define GCC_PCIE_0_PHY_RCHNG_CLK 42 +#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC 43 +#define GCC_PCIE_0_PIPE_CLK 44 +#define GCC_PCIE_0_PIPE_CLK_SRC 45 +#define GCC_PCIE_0_PIPE_DIV2_CLK 46 +#define GCC_PCIE_0_PIPE_DIV_CLK_SRC 47 +#define GCC_PCIE_0_SLV_AXI_CLK 48 +#define GCC_PCIE_0_SLV_Q2A_AXI_CLK 49 +#define GCC_PCIE_1_AUX_CLK 50 +#define GCC_PCIE_1_AUX_CLK_SRC 51 +#define GCC_PCIE_1_CFG_AHB_CLK 52 +#define GCC_PCIE_1_MSTR_AXI_CLK 53 +#define GCC_PCIE_1_PHY_AUX_CLK 54 +#define GCC_PCIE_1_PHY_AUX_CLK_SRC 55 +#define GCC_PCIE_1_PHY_RCHNG_CLK 56 +#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC 57 +#define GCC_PCIE_1_PIPE_CLK 58 +#define GCC_PCIE_1_PIPE_CLK_SRC 59 +#define GCC_PCIE_1_PIPE_DIV2_CLK 60 +#define GCC_PCIE_1_PIPE_DIV_CLK_SRC 61 +#define GCC_PCIE_1_RSC_CORE_CLK 62 +#define GCC_PCIE_1_SLV_AXI_CLK 63 +#define GCC_PCIE_1_SLV_Q2A_AXI_CLK 64 +#define GCC_PCIE_RSC_CORE_CLK 65 +#define GCC_PCIE_RSCC_CFG_AHB_CLK 66 +#define GCC_PCIE_RSCC_XO_CLK 67 +#define GCC_PDM2_CLK 68 +#define GCC_PDM2_CLK_SRC 69 +#define GCC_PDM_AHB_CLK 70 +#define GCC_PDM_XO4_CLK 71 +#define GCC_QUPV3_I2C_CORE_CLK 72 +#define GCC_QUPV3_I2C_S0_CLK 73 +#define GCC_QUPV3_I2C_S0_CLK_SRC 74 +#define GCC_QUPV3_I2C_S1_CLK 75 +#define GCC_QUPV3_I2C_S1_CLK_SRC 76 +#define GCC_QUPV3_I2C_S2_CLK 77 +#define GCC_QUPV3_I2C_S2_CLK_SRC 78 +#define GCC_QUPV3_I2C_S3_CLK 79 +#define GCC_QUPV3_I2C_S3_CLK_SRC 80 +#define GCC_QUPV3_I2C_S4_CLK 81 +#define GCC_QUPV3_I2C_S4_CLK_SRC 82 +#define GCC_QUPV3_I2C_S_AHB_CLK 83 +#define GCC_QUPV3_WRAP1_CORE_2X_CLK 84 +#define GCC_QUPV3_WRAP1_CORE_CLK 85 +#define GCC_QUPV3_WRAP1_QSPI_REF_CLK 86 +#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC 87 +#define GCC_QUPV3_WRAP1_S0_CLK 88 +#define GCC_QUPV3_WRAP1_S0_CLK_SRC 89 +#define GCC_QUPV3_WRAP1_S1_CLK 90 +#define GCC_QUPV3_WRAP1_S1_CLK_SRC 91 +#define GCC_QUPV3_WRAP1_S2_CLK 92 +#define GCC_QUPV3_WRAP1_S2_CLK_SRC 93 +#define GCC_QUPV3_WRAP1_S3_CLK 94 +#define GCC_QUPV3_WRAP1_S3_CLK_SRC 95 +#define GCC_QUPV3_WRAP1_S4_CLK 96 +#define GCC_QUPV3_WRAP1_S4_CLK_SRC 97 +#define GCC_QUPV3_WRAP1_S5_CLK 98 +#define GCC_QUPV3_WRAP1_S5_CLK_SRC 99 +#define GCC_QUPV3_WRAP1_S6_CLK 100 +#define GCC_QUPV3_WRAP1_S6_CLK_SRC 101 +#define GCC_QUPV3_WRAP1_S7_CLK 102 +#define GCC_QUPV3_WRAP1_S7_CLK_SRC 103 +#define GCC_QUPV3_WRAP2_CORE_2X_CLK 104 +#define GCC_QUPV3_WRAP2_CORE_CLK 105 +#define GCC_QUPV3_WRAP2_S0_CLK 106 +#define GCC_QUPV3_WRAP2_S0_CLK_SRC 107 +#define GCC_QUPV3_WRAP2_S1_CLK 108 +#define GCC_QUPV3_WRAP2_S1_CLK_SRC 109 +#define GCC_QUPV3_WRAP2_S2_CLK 110 +#define GCC_QUPV3_WRAP2_S2_CLK_SRC 111 +#define GCC_QUPV3_WRAP2_S3_CLK 112 +#define GCC_QUPV3_WRAP2_S3_CLK_SRC 113 +#define GCC_QUPV3_WRAP2_S4_CLK 114 +#define GCC_QUPV3_WRAP2_S4_CLK_SRC 115 +#define GCC_QUPV3_WRAP3_CORE_2X_CLK 116 +#define GCC_QUPV3_WRAP3_CORE_CLK 117 +#define GCC_QUPV3_WRAP3_QSPI_REF_CLK 118 +#define GCC_QUPV3_WRAP3_QSPI_REF_CLK_SRC 119 +#define GCC_QUPV3_WRAP3_S0_CLK 120 +#define GCC_QUPV3_WRAP3_S0_CLK_SRC 121 +#define GCC_QUPV3_WRAP3_S1_CLK 122 +#define GCC_QUPV3_WRAP3_S1_CLK_SRC 123 +#define GCC_QUPV3_WRAP3_S2_CLK 124 +#define GCC_QUPV3_WRAP3_S2_CLK_SRC 125 +#define GCC_QUPV3_WRAP3_S3_CLK 126 +#define GCC_QUPV3_WRAP3_S3_CLK_SRC 127 +#define GCC_QUPV3_WRAP3_S4_CLK 128 +#define GCC_QUPV3_WRAP3_S4_CLK_SRC 129 +#define GCC_QUPV3_WRAP3_S5_CLK 130 +#define GCC_QUPV3_WRAP3_S5_CLK_SRC 131 +#define GCC_QUPV3_WRAP4_CORE_2X_CLK 132 +#define GCC_QUPV3_WRAP4_CORE_CLK 133 +#define GCC_QUPV3_WRAP4_S0_CLK 134 +#define GCC_QUPV3_WRAP4_S0_CLK_SRC 135 +#define GCC_QUPV3_WRAP4_S1_CLK 136 +#define GCC_QUPV3_WRAP4_S1_CLK_SRC 137 +#define GCC_QUPV3_WRAP4_S2_CLK 138 +#define GCC_QUPV3_WRAP4_S2_CLK_SRC 139 +#define GCC_QUPV3_WRAP4_S3_CLK 140 +#define GCC_QUPV3_WRAP4_S3_CLK_SRC 141 +#define GCC_QUPV3_WRAP4_S4_CLK 142 +#define GCC_QUPV3_WRAP4_S4_CLK_SRC 143 +#define GCC_QUPV3_WRAP_1_M_AXI_CLK 144 +#define GCC_QUPV3_WRAP_1_S_AHB_CLK 145 +#define GCC_QUPV3_WRAP_2_M_AHB_CLK 146 +#define GCC_QUPV3_WRAP_2_S_AHB_CLK 147 +#define GCC_QUPV3_WRAP_3_M_AHB_CLK 148 +#define GCC_QUPV3_WRAP_3_S_AHB_CLK 149 +#define GCC_QUPV3_WRAP_4_M_AHB_CLK 150 +#define GCC_QUPV3_WRAP_4_S_AHB_CLK 151 +#define GCC_SDCC2_AHB_CLK 152 +#define GCC_SDCC2_APPS_CLK 153 +#define GCC_SDCC2_APPS_CLK_SRC 154 +#define GCC_SDCC4_AHB_CLK 155 +#define GCC_SDCC4_APPS_CLK 156 +#define GCC_SDCC4_APPS_CLK_SRC 157 +#define GCC_UFS_PHY_AHB_CLK 158 +#define GCC_UFS_PHY_AXI_CLK 159 +#define GCC_UFS_PHY_AXI_CLK_SRC 160 +#define GCC_UFS_PHY_ICE_CORE_CLK 161 +#define GCC_UFS_PHY_ICE_CORE_CLK_SRC 162 +#define GCC_UFS_PHY_PHY_AUX_CLK 163 +#define GCC_UFS_PHY_PHY_AUX_CLK_SRC 164 +#define GCC_UFS_PHY_RX_SYMBOL_0_CLK 165 +#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC 166 +#define GCC_UFS_PHY_RX_SYMBOL_1_CLK 167 +#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC 168 +#define GCC_UFS_PHY_TX_SYMBOL_0_CLK 169 +#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC 170 +#define GCC_UFS_PHY_UNIPRO_5_CORE_CLK 171 +#define GCC_UFS_PHY_UNIPRO_5_CORE_CLK_SRC 172 +#define GCC_USB30_PRIM_MASTER_CLK 173 +#define GCC_USB30_PRIM_MASTER_CLK_SRC 174 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK 175 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC 176 +#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC 177 +#define GCC_USB30_PRIM_SLEEP_CLK 178 +#define GCC_USB3_PRIM_PHY_AUX_CLK 179 +#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC 180 +#define GCC_USB3_PRIM_PHY_COM_AUX_CLK 181 +#define GCC_USB3_PRIM_PHY_PIPE_CLK 182 +#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC 183 +#define GCC_VIDEO_AHB_CLK 184 +#define GCC_VIDEO_AXI0_CLK 185 +#define GCC_VIDEO_AXI0C_CLK 186 +#define GCC_VIDEO_XO_CLK 187 + +/* GCC power domains */ +#define GCC_PCIE_0_GDSC 0 +#define GCC_PCIE_0_PHY_GDSC 1 +#define GCC_PCIE_1_GDSC 2 +#define GCC_PCIE_1_PHY_GDSC 3 +#define GCC_UFS_MEM_PHY_GDSC 4 +#define GCC_UFS_PHY_GDSC 5 +#define GCC_USB30_PRIM_GDSC 6 +#define GCC_USB3_PHY_GDSC 7 + +/* GCC resets */ +#define GCC_CAMERA_BCR 0 +#define GCC_EVA_AXI0_CLK_ARES 1 +#define GCC_EVA_AXI0C_CLK_ARES 2 +#define GCC_EVA_BCR 3 +#define GCC_GPU_BCR 4 +#define GCC_PCIE_0_BCR 5 +#define GCC_PCIE_0_LINK_DOWN_BCR 6 +#define GCC_PCIE_0_NOCSR_COM_PHY_BCR 7 +#define GCC_PCIE_0_PHY_BCR 8 +#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR 9 +#define GCC_PCIE_1_BCR 10 +#define GCC_PCIE_1_LINK_DOWN_BCR 11 +#define GCC_PCIE_1_NOCSR_COM_PHY_BCR 12 +#define GCC_PCIE_1_PHY_BCR 13 +#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR 14 +#define GCC_PCIE_PHY_BCR 15 +#define GCC_PCIE_PHY_CFG_AHB_BCR 16 +#define GCC_PCIE_PHY_COM_BCR 17 +#define GCC_PCIE_RSCC_BCR 18 +#define GCC_PDM_BCR 19 +#define GCC_QUPV3_WRAPPER_1_BCR 20 +#define GCC_QUPV3_WRAPPER_2_BCR 21 +#define GCC_QUPV3_WRAPPER_3_BCR 22 +#define GCC_QUPV3_WRAPPER_4_BCR 23 +#define GCC_QUPV3_WRAPPER_I2C_BCR 24 +#define GCC_QUSB2PHY_PRIM_BCR 25 +#define GCC_QUSB2PHY_SEC_BCR 26 +#define GCC_SDCC2_BCR 27 +#define GCC_SDCC4_BCR 28 +#define GCC_TCSR_PCIE_BCR 29 +#define GCC_UFS_PHY_BCR 30 +#define GCC_USB30_PRIM_BCR 31 +#define GCC_USB3_DP_PHY_PRIM_BCR 32 +#define GCC_USB3_DP_PHY_SEC_BCR 33 +#define GCC_USB3_PHY_PRIM_BCR 34 +#define GCC_USB3_PHY_SEC_BCR 35 +#define GCC_USB3PHY_PHY_PRIM_BCR 36 +#define GCC_USB3PHY_PHY_SEC_BCR 37 +#define GCC_VIDEO_AXI0_CLK_ARES 38 +#define GCC_VIDEO_AXI0C_CLK_ARES 39 +#define GCC_VIDEO_BCR 40 +#define GCC_VIDEO_XO_CLK_ARES 41 + +#endif diff --git a/include/dt-bindings/clock/qcom,hawi-tcsrcc.h b/include/dt-bindings/clock/qcom,hawi-tcsrcc.h new file mode 100644 index 000000000000..957bc5f75bb7 --- /dev/null +++ b/include/dt-bindings/clock/qcom,hawi-tcsrcc.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_HAWI_H +#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_HAWI_H + +/* TCSR_CC clocks */ +#define TCSR_PCIE_0_CLKREF_EN 0 +#define TCSR_PCIE_1_CLKREF_EN 1 +#define TCSR_UFS_CLKREF_EN 2 +#define TCSR_USB2_CLKREF_EN 3 +#define TCSR_USB3_CLKREF_EN 4 + +#endif diff --git a/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h new file mode 100644 index 000000000000..172330e43669 --- /dev/null +++ b/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_IPQ5332_CMN_PLL_H +#define _DT_BINDINGS_CLK_QCOM_IPQ5332_CMN_PLL_H + +/* CMN PLL core clock. */ +#define IPQ5332_CMN_PLL_CLK 0 + +/* The output clocks from CMN PLL of IPQ5332. */ +#define IPQ5332_XO_24MHZ_CLK 1 +#define IPQ5332_SLEEP_32KHZ_CLK 2 +#define IPQ5332_PCS_31P25MHZ_CLK 3 +#define IPQ5332_NSS_300MHZ_CLK 4 +#define IPQ5332_PPE_200MHZ_CLK 5 +#define IPQ5332_ETH_50MHZ_CLK 6 +#endif diff --git a/include/dt-bindings/clock/qcom,nord-camcc.h b/include/dt-bindings/clock/qcom,nord-camcc.h new file mode 100644 index 000000000000..655fef2084a5 --- /dev/null +++ b/include/dt-bindings/clock/qcom,nord-camcc.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_NORD_H +#define _DT_BINDINGS_CLK_QCOM_CAM_CC_NORD_H + +/* CAM_CC clocks */ +#define CAM_CC_CAMNOC_DCD_XO_CLK 0 +#define CAM_CC_CAMNOC_NRT_AXI_CLK 1 +#define CAM_CC_CAMNOC_RT_AXI_CLK 2 +#define CAM_CC_CAMNOC_RT_AXI_CLK_SRC 3 +#define CAM_CC_CAMNOC_XO_CLK 4 +#define CAM_CC_CCI_0_CLK 5 +#define CAM_CC_CCI_0_CLK_SRC 6 +#define CAM_CC_CCI_1_CLK 7 +#define CAM_CC_CCI_1_CLK_SRC 8 +#define CAM_CC_CCI_2_CLK 9 +#define CAM_CC_CCI_2_CLK_SRC 10 +#define CAM_CC_CCI_3_CLK 11 +#define CAM_CC_CCI_3_CLK_SRC 12 +#define CAM_CC_CCI_4_CLK 13 +#define CAM_CC_CCI_4_CLK_SRC 14 +#define CAM_CC_CCU_FAST_AHB_CLK 15 +#define CAM_CC_CORE_AHB_CLK 16 +#define CAM_CC_CPHY_RX_CLK_SRC 17 +#define CAM_CC_CSI0PHYTIMER_CLK 18 +#define CAM_CC_CSI0PHYTIMER_CLK_SRC 19 +#define CAM_CC_CSI1PHYTIMER_CLK 20 +#define CAM_CC_CSI1PHYTIMER_CLK_SRC 21 +#define CAM_CC_CSI2PHYTIMER_CLK 22 +#define CAM_CC_CSI2PHYTIMER_CLK_SRC 23 +#define CAM_CC_CSI3PHYTIMER_CLK 24 +#define CAM_CC_CSI3PHYTIMER_CLK_SRC 25 +#define CAM_CC_CSI4PHYTIMER_CLK 26 +#define CAM_CC_CSI4PHYTIMER_CLK_SRC 27 +#define CAM_CC_CSID_CLK 28 +#define CAM_CC_CSID_CLK_SRC 29 +#define CAM_CC_CSID_CSIPHY_RX_CLK 30 +#define CAM_CC_CSIPHY0_CLK 31 +#define CAM_CC_CSIPHY1_CLK 32 +#define CAM_CC_CSIPHY2_CLK 33 +#define CAM_CC_CSIPHY3_CLK 34 +#define CAM_CC_CSIPHY4_CLK 35 +#define CAM_CC_FAST_AHB_CLK_SRC 36 +#define CAM_CC_GDSC_CLK 37 +#define CAM_CC_ICP_0_AHB_CLK 38 +#define CAM_CC_ICP_0_CLK 39 +#define CAM_CC_ICP_0_CLK_SRC 40 +#define CAM_CC_ICP_1_AHB_CLK 41 +#define CAM_CC_ICP_1_CLK 42 +#define CAM_CC_ICP_1_CLK_SRC 43 +#define CAM_CC_IFE_0_MAIN_CLK 44 +#define CAM_CC_IFE_0_MAIN_CLK_SRC 45 +#define CAM_CC_IFE_0_MAIN_FAST_AHB_CLK 46 +#define CAM_CC_IFE_0_PCP_CLK 47 +#define CAM_CC_IFE_0_PCP_FAST_AHB_CLK 48 +#define CAM_CC_IFE_0_SCALAR_CLK 49 +#define CAM_CC_IFE_0_SCALAR_FAST_AHB_CLK 50 +#define CAM_CC_IFE_0_TMC_CLK 51 +#define CAM_CC_IFE_0_TMC_FAST_AHB_CLK 52 +#define CAM_CC_IFE_1_MAIN_CLK 53 +#define CAM_CC_IFE_1_MAIN_CLK_SRC 54 +#define CAM_CC_IFE_1_MAIN_FAST_AHB_CLK 55 +#define CAM_CC_IFE_1_PCP_CLK 56 +#define CAM_CC_IFE_1_PCP_FAST_AHB_CLK 57 +#define CAM_CC_IFE_1_SCALAR_CLK 58 +#define CAM_CC_IFE_1_SCALAR_FAST_AHB_CLK 59 +#define CAM_CC_IFE_1_TMC_CLK 60 +#define CAM_CC_IFE_1_TMC_FAST_AHB_CLK 61 +#define CAM_CC_IFE_2_MAIN_CLK 62 +#define CAM_CC_IFE_2_MAIN_CLK_SRC 63 +#define CAM_CC_IFE_2_MAIN_FAST_AHB_CLK 64 +#define CAM_CC_IFE_2_PCP_CLK 65 +#define CAM_CC_IFE_2_PCP_FAST_AHB_CLK 66 +#define CAM_CC_IFE_2_SCALAR_CLK 67 +#define CAM_CC_IFE_2_SCALAR_FAST_AHB_CLK 68 +#define CAM_CC_IFE_2_TMC_CLK 69 +#define CAM_CC_IFE_2_TMC_FAST_AHB_CLK 70 +#define CAM_CC_IFE_LITE_AHB_CLK 71 +#define CAM_CC_IFE_LITE_CLK 72 +#define CAM_CC_IFE_LITE_CLK_SRC 73 +#define CAM_CC_IFE_LITE_CPHY_RX_CLK 74 +#define CAM_CC_IFE_LITE_CSID_CLK 75 +#define CAM_CC_IFE_LITE_CSID_CLK_SRC 76 +#define CAM_CC_IPE_0_AHB_CLK 77 +#define CAM_CC_IPE_0_CLK 78 +#define CAM_CC_IPE_0_CLK_SRC 79 +#define CAM_CC_IPE_0_FAST_AHB_CLK 80 +#define CAM_CC_IPE_1_AHB_CLK 81 +#define CAM_CC_IPE_1_CLK 82 +#define CAM_CC_IPE_1_CLK_SRC 83 +#define CAM_CC_IPE_1_FAST_AHB_CLK 84 +#define CAM_CC_PLL0 85 +#define CAM_CC_PLL0_OUT_EVEN 86 +#define CAM_CC_PLL0_OUT_ODD 87 +#define CAM_CC_PLL2 88 +#define CAM_CC_PLL2_OUT_EVEN 89 +#define CAM_CC_PLL3 90 +#define CAM_CC_PLL3_OUT_EVEN 91 +#define CAM_CC_PLL4 92 +#define CAM_CC_PLL4_OUT_EVEN 93 +#define CAM_CC_PLL5 94 +#define CAM_CC_PLL5_OUT_EVEN 95 +#define CAM_CC_PLL6 96 +#define CAM_CC_PLL6_OUT_EVEN 97 +#define CAM_CC_QDSS_DEBUG_CLK 98 +#define CAM_CC_QDSS_DEBUG_CLK_SRC 99 +#define CAM_CC_QDSS_DEBUG_XO_CLK 100 +#define CAM_CC_QUP_AHBM_CLK 101 +#define CAM_CC_QUP_AHBM_CLK_SRC 102 +#define CAM_CC_QUP_CORE_2X_CLK 103 +#define CAM_CC_QUP_CORE_2X_CLK_SRC 104 +#define CAM_CC_QUP_CORE_2X_DIV_CLK_SRC 105 +#define CAM_CC_QUP_CORE_CLK 106 +#define CAM_CC_QUP_SE_CLK 107 +#define CAM_CC_QUP_SE_CLK_SRC 108 +#define CAM_CC_QUP_SLEEP_CLK 109 +#define CAM_CC_SFE_LITE_0_CLK 110 +#define CAM_CC_SFE_LITE_0_FAST_AHB_CLK 111 +#define CAM_CC_SFE_LITE_1_CLK 112 +#define CAM_CC_SFE_LITE_1_FAST_AHB_CLK 113 +#define CAM_CC_SFE_LITE_2_CLK 114 +#define CAM_CC_SFE_LITE_2_FAST_AHB_CLK 115 +#define CAM_CC_SLEEP_CLK 116 +#define CAM_CC_SLEEP_CLK_SRC 117 +#define CAM_CC_SLOW_AHB_CLK_SRC 118 +#define CAM_CC_SM_OBS_CLK 119 +#define CAM_CC_TOP_AHB_CLK 120 +#define CAM_CC_TOP_FAST_AHB_CLK 121 +#define CAM_CC_TOP_IFE_0_CLK 122 +#define CAM_CC_TOP_IFE_1_CLK 123 +#define CAM_CC_TOP_IFE_2_CLK 124 +#define CAM_CC_TOP_IFE_LITE_CLK 125 +#define CAM_CC_TOP_IPE_0_CLK 126 +#define CAM_CC_TOP_IPE_1_CLK 127 +#define CAM_CC_TOP_QUP_AHBM_CLK 128 +#define CAM_CC_TOP_SFE_LITE_0_CLK 129 +#define CAM_CC_TOP_SFE_LITE_1_CLK 130 +#define CAM_CC_TOP_SFE_LITE_2_CLK 131 +#define CAM_CC_TPG_CSIPHY_RX_CLK 132 +#define CAM_CC_XO_CLK_SRC 133 + +/* CAM_CC power domains */ +#define CAM_CC_IFE_0_GDSC 0 +#define CAM_CC_IFE_1_GDSC 1 +#define CAM_CC_IFE_2_GDSC 2 +#define CAM_CC_IPE_0_GDSC 3 +#define CAM_CC_IPE_1_GDSC 4 +#define CAM_CC_TITAN_TOP_GDSC 5 + +/* CAM_CC resets */ +#define CAM_CC_CCU_BCR 0 +#define CAM_CC_ICP_0_BCR 1 +#define CAM_CC_ICP_1_BCR 2 +#define CAM_CC_IFE_0_BCR 3 +#define CAM_CC_IFE_1_BCR 4 +#define CAM_CC_IFE_2_BCR 5 +#define CAM_CC_IPE_0_BCR 6 +#define CAM_CC_IPE_1_BCR 7 +#define CAM_CC_QDSS_DEBUG_BCR 8 +#define CAM_CC_SFE_LITE_0_BCR 9 +#define CAM_CC_SFE_LITE_1_BCR 10 +#define CAM_CC_SFE_LITE_2_BCR 11 + +#endif diff --git a/include/dt-bindings/clock/qcom,nord-dispcc.h b/include/dt-bindings/clock/qcom,nord-dispcc.h new file mode 100644 index 000000000000..9f6c9979e0f3 --- /dev/null +++ b/include/dt-bindings/clock/qcom,nord-dispcc.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_NORD_H +#define _DT_BINDINGS_CLK_QCOM_DISP_CC_NORD_H + +/* DISP_CC_0 clocks */ +#define MDSS_DISP_CC_ACMU_CLK 0 +#define MDSS_DISP_CC_MDSS_ACCU_SHIFT_CLK 1 +#define MDSS_DISP_CC_MDSS_AHB1_CLK 2 +#define MDSS_DISP_CC_MDSS_AHB_CLK 3 +#define MDSS_DISP_CC_MDSS_AHB_CLK_SRC 4 +#define MDSS_DISP_CC_MDSS_BYTE0_CLK 5 +#define MDSS_DISP_CC_MDSS_BYTE0_CLK_SRC 6 +#define MDSS_DISP_CC_MDSS_BYTE0_DIV_CLK_SRC 7 +#define MDSS_DISP_CC_MDSS_BYTE0_INTF_CLK 8 +#define MDSS_DISP_CC_MDSS_BYTE1_CLK 9 +#define MDSS_DISP_CC_MDSS_BYTE1_CLK_SRC 10 +#define MDSS_DISP_CC_MDSS_BYTE1_DIV_CLK_SRC 11 +#define MDSS_DISP_CC_MDSS_BYTE1_INTF_CLK 12 +#define MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK 13 +#define MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK_SRC 14 +#define MDSS_DISP_CC_MDSS_DPTX0_CRYPTO_CLK 15 +#define MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK 16 +#define MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC 17 +#define MDSS_DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC 18 +#define MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK 19 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK 20 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC 21 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK 22 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC 23 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK 24 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC 25 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK 26 +#define MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC 27 +#define MDSS_DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK 28 +#define MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK 29 +#define MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK_SRC 30 +#define MDSS_DISP_CC_MDSS_DPTX1_CRYPTO_CLK 31 +#define MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK 32 +#define MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK_SRC 33 +#define MDSS_DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC 34 +#define MDSS_DISP_CC_MDSS_DPTX1_LINK_INTF_CLK 35 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK 36 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC 37 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK 38 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC 39 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL2_CLK 40 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL2_CLK_SRC 41 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL3_CLK 42 +#define MDSS_DISP_CC_MDSS_DPTX1_PIXEL3_CLK_SRC 43 +#define MDSS_DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK 44 +#define MDSS_DISP_CC_MDSS_DPTX2_AUX_CLK 45 +#define MDSS_DISP_CC_MDSS_DPTX2_AUX_CLK_SRC 46 +#define MDSS_DISP_CC_MDSS_DPTX2_CRYPTO_CLK 47 +#define MDSS_DISP_CC_MDSS_DPTX2_LINK_CLK 48 +#define MDSS_DISP_CC_MDSS_DPTX2_LINK_CLK_SRC 49 +#define MDSS_DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC 50 +#define MDSS_DISP_CC_MDSS_DPTX2_LINK_INTF_CLK 51 +#define MDSS_DISP_CC_MDSS_DPTX2_PIXEL0_CLK 52 +#define MDSS_DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC 53 +#define MDSS_DISP_CC_MDSS_DPTX2_PIXEL1_CLK 54 +#define MDSS_DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC 55 +#define MDSS_DISP_CC_MDSS_DPTX3_AUX_CLK 56 +#define MDSS_DISP_CC_MDSS_DPTX3_AUX_CLK_SRC 57 +#define MDSS_DISP_CC_MDSS_DPTX3_CRYPTO_CLK 58 +#define MDSS_DISP_CC_MDSS_DPTX3_LINK_CLK 59 +#define MDSS_DISP_CC_MDSS_DPTX3_LINK_CLK_SRC 60 +#define MDSS_DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC 61 +#define MDSS_DISP_CC_MDSS_DPTX3_LINK_INTF_CLK 62 +#define MDSS_DISP_CC_MDSS_DPTX3_PIXEL0_CLK 63 +#define MDSS_DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC 64 +#define MDSS_DISP_CC_MDSS_ESC0_CLK 65 +#define MDSS_DISP_CC_MDSS_ESC0_CLK_SRC 66 +#define MDSS_DISP_CC_MDSS_ESC1_CLK 67 +#define MDSS_DISP_CC_MDSS_ESC1_CLK_SRC 68 +#define MDSS_DISP_CC_MDSS_MDP1_CLK 69 +#define MDSS_DISP_CC_MDSS_MDP_CLK 70 +#define MDSS_DISP_CC_MDSS_MDP_CLK_SRC 71 +#define MDSS_DISP_CC_MDSS_MDP_LUT1_CLK 72 +#define MDSS_DISP_CC_MDSS_MDP_LUT_CLK 73 +#define MDSS_DISP_CC_MDSS_NON_GDSC_AHB_CLK 74 +#define MDSS_DISP_CC_MDSS_PCLK0_CLK 75 +#define MDSS_DISP_CC_MDSS_PCLK0_CLK_SRC 76 +#define MDSS_DISP_CC_MDSS_PCLK1_CLK 77 +#define MDSS_DISP_CC_MDSS_PCLK1_CLK_SRC 78 +#define MDSS_DISP_CC_MDSS_PCLK2_CLK 79 +#define MDSS_DISP_CC_MDSS_PCLK2_CLK_SRC 80 +#define MDSS_DISP_CC_MDSS_RSCC_AHB_CLK 81 +#define MDSS_DISP_CC_MDSS_RSCC_VSYNC_CLK 82 +#define MDSS_DISP_CC_MDSS_VSYNC1_CLK 83 +#define MDSS_DISP_CC_MDSS_VSYNC_CLK 84 +#define MDSS_DISP_CC_MDSS_VSYNC_CLK_SRC 85 +#define MDSS_DISP_CC_PLL0 86 +#define MDSS_DISP_CC_PLL1 87 +#define MDSS_DISP_CC_PLL2 88 +#define MDSS_DISP_CC_PLL3 89 +#define MDSS_DISP_CC_SLEEP_CLK 90 +#define MDSS_DISP_CC_SLEEP_CLK_SRC 91 +#define MDSS_DISP_CC_SM_DIV_CLK_SRC 92 +#define MDSS_DISP_CC_XO_CLK 93 +#define MDSS_DISP_CC_XO_CLK_SRC 94 + +/* DISP_CC_0 power domains */ +#define MDSS_DISP_CC_MDSS_CORE_GDSC 0 +#define MDSS_DISP_CC_MDSS_CORE_INT2_GDSC 1 + +/* DISP_CC_0 resets */ +#define MDSS_DISP_CC_MDSS_CORE_BCR 0 +#define MDSS_DISP_CC_MDSS_CORE_INT2_BCR 1 +#define MDSS_DISP_CC_MDSS_RSCC_BCR 2 + +#endif diff --git a/include/dt-bindings/clock/qcom,nord-gpucc.h b/include/dt-bindings/clock/qcom,nord-gpucc.h new file mode 100644 index 000000000000..a673e4854d66 --- /dev/null +++ b/include/dt-bindings/clock/qcom,nord-gpucc.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_NORD_H +#define _DT_BINDINGS_CLK_QCOM_GPU_CC_NORD_H + +/* GPU_CC clocks */ +#define GPU_CC_ACD_GFX3D_CLK 0 +#define GPU_CC_ACMU_CLK 1 +#define GPU_CC_AHB_CLK 2 +#define GPU_CC_CRC_AHB_CLK 3 +#define GPU_CC_CX_ACCU_SHIFT_CLK 4 +#define GPU_CC_CX_FF_CLK 5 +#define GPU_CC_CX_GMU_CLK 6 +#define GPU_CC_CXO_AON_CLK 7 +#define GPU_CC_CXO_CLK 8 +#define GPU_CC_DEMET_CLK 9 +#define GPU_CC_DPM_CLK 10 +#define GPU_CC_FF_CLK_SRC 11 +#define GPU_CC_FREQ_MEASURE_CLK 12 +#define GPU_CC_GMU_CLK_SRC 13 +#define GPU_CC_GPU_SMMU_VOTE_CLK 14 +#define GPU_CC_HUB_AON_CLK 15 +#define GPU_CC_HUB_CLK_SRC 16 +#define GPU_CC_HUB_CX_INT_CLK 17 +#define GPU_CC_HUB_DIV_CLK_SRC 18 +#define GPU_CC_MEMNOC_GFX_CLK 19 +#define GPU_CC_MND1X_GFX3D_CLK 20 +#define GPU_CC_MND1X_1_GFX3D_CLK 21 +#define GPU_CC_PLL0 22 +#define GPU_CC_PLL1 23 +#define GPU_CC_SLEEP_CLK 24 + +/* GPU_CC power domains */ +#define GPU_CC_CX_GDSC 0 +#define GPU_CC_GX_GDSC 1 + +/* GPU_CC resets */ +#define GPU_CC_ACD_BCR 0 +#define GPU_CC_CB_BCR 1 +#define GPU_CC_CX_BCR 2 +#define GPU_CC_FAST_HUB_BCR 3 +#define GPU_CC_FF_BCR 4 +#define GPU_CC_GFX3D_AON_BCR 5 +#define GPU_CC_GMU_BCR 6 +#define GPU_CC_GX_BCR 7 +#define GPU_CC_XO_BCR 8 + +#endif diff --git a/include/dt-bindings/clock/qcom,nord-negcc.h b/include/dt-bindings/clock/qcom,nord-negcc.h index 95f333d8e1aa..a7024317e90b 100644 --- a/include/dt-bindings/clock/qcom,nord-negcc.h +++ b/include/dt-bindings/clock/qcom,nord-negcc.h @@ -120,5 +120,6 @@ #define NE_GCC_USB3_PHY_SEC_BCR 10 #define NE_GCC_USB3PHY_PHY_PRIM_BCR 11 #define NE_GCC_USB3PHY_PHY_SEC_BCR 12 +#define NE_GCC_QUSB2PHY_PRIM_BCR 13 #endif diff --git a/include/dt-bindings/clock/qcom,nord-videocc.h b/include/dt-bindings/clock/qcom,nord-videocc.h new file mode 100644 index 000000000000..8d7546021109 --- /dev/null +++ b/include/dt-bindings/clock/qcom,nord-videocc.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_NORD_H +#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_NORD_H + +/* VIDEO_CC clocks */ +#define VIDEO_CC_AHB_CLK_SRC 0 +#define VIDEO_CC_MVS0_CLK 1 +#define VIDEO_CC_MVS0_CLK_SRC 2 +#define VIDEO_CC_MVS0_DIV_CLK_SRC 3 +#define VIDEO_CC_MVS0_FREERUN_CLK 4 +#define VIDEO_CC_MVS0_SHIFT_CLK 5 +#define VIDEO_CC_MVS0C_CLK 6 +#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC 7 +#define VIDEO_CC_MVS0C_FREERUN_CLK 8 +#define VIDEO_CC_MVS0C_SHIFT_CLK 9 +#define VIDEO_CC_MVS1_CLK 10 +#define VIDEO_CC_MVS1_DIV_CLK_SRC 11 +#define VIDEO_CC_MVS1_FREERUN_CLK 12 +#define VIDEO_CC_MVS1_SHIFT_CLK 13 +#define VIDEO_CC_PLL0 14 +#define VIDEO_CC_SLEEP_CLK_SRC 15 +#define VIDEO_CC_XO_CLK_SRC 16 + +/* VIDEO_CC power domains */ +#define VIDEO_CC_MVS0_GDSC 0 +#define VIDEO_CC_MVS0C_GDSC 1 +#define VIDEO_CC_MVS1_GDSC 2 + +/* VIDEO_CC resets */ +#define VIDEO_CC_INTERFACE_BCR 0 +#define VIDEO_CC_MVS0_BCR 1 +#define VIDEO_CC_MVS0C_CLK_ARES 2 +#define VIDEO_CC_MVS0C_BCR 3 +#define VIDEO_CC_MVS1_BCR 4 + +#endif diff --git a/include/dt-bindings/clock/qcom,rpmh.h b/include/dt-bindings/clock/qcom,rpmh.h index 0a7d1be0d124..2d62d5d0b08d 100644 --- a/include/dt-bindings/clock/qcom,rpmh.h +++ b/include/dt-bindings/clock/qcom,rpmh.h @@ -33,5 +33,7 @@ #define RPMH_HWKM_CLK 24 #define RPMH_QLINK_CLK 25 #define RPMH_QLINK_CLK_A 26 +#define RPMH_LN_BB_CLK4 27 +#define RPMH_LN_BB_CLK4_A 28 #endif diff --git a/include/dt-bindings/clock/qcom,x1e80100-camcc.h b/include/dt-bindings/clock/qcom,x1e80100-camcc.h index d72fdfb06a7c..06c316022fb0 100644 --- a/include/dt-bindings/clock/qcom,x1e80100-camcc.h +++ b/include/dt-bindings/clock/qcom,x1e80100-camcc.h @@ -115,6 +115,9 @@ #define CAM_CC_SLEEP_CLK_SRC 105 #define CAM_CC_SLOW_AHB_CLK_SRC 106 #define CAM_CC_XO_CLK_SRC 107 +#define CAM_CC_QDSS_DEBUG_CLK 108 +#define CAM_CC_QDSS_DEBUG_CLK_SRC 109 +#define CAM_CC_QDSS_DEBUG_XO_CLK 110 /* CAM_CC power domains */ #define CAM_CC_BPS_GDSC 0 diff --git a/include/dt-bindings/clock/qcom,x1p42100-videocc.h b/include/dt-bindings/clock/qcom,x1p42100-videocc.h new file mode 100644 index 000000000000..996408d1a0c3 --- /dev/null +++ b/include/dt-bindings/clock/qcom,x1p42100-videocc.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_X1P42100_H +#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_X1P42100_H + +/* VIDEO_CC clocks */ +#define VIDEO_CC_MVS0_CLK 0 +#define VIDEO_CC_MVS0_CLK_SRC 1 +#define VIDEO_CC_MVS0_DIV_CLK_SRC 2 +#define VIDEO_CC_MVS0C_CLK 3 +#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC 4 +#define VIDEO_CC_MVS1_CLK 5 +#define VIDEO_CC_MVS1_CLK_SRC 6 +#define VIDEO_CC_MVS1_DIV_CLK_SRC 7 +#define VIDEO_CC_MVS1C_CLK 8 +#define VIDEO_CC_MVS1C_DIV2_DIV_CLK_SRC 9 +#define VIDEO_CC_PLL0 10 +#define VIDEO_CC_PLL1 11 +#define VIDEO_CC_MVS0_SHIFT_CLK 12 +#define VIDEO_CC_MVS0C_SHIFT_CLK 13 +#define VIDEO_CC_MVS1_SHIFT_CLK 14 +#define VIDEO_CC_MVS1C_SHIFT_CLK 15 +#define VIDEO_CC_XO_CLK_SRC 16 +#define VIDEO_CC_MVS0_BSE_CLK 17 +#define VIDEO_CC_MVS0_BSE_CLK_SRC 18 +#define VIDEO_CC_MVS0_BSE_DIV4_DIV_CLK_SRC 19 + +/* VIDEO_CC power domains */ +#define VIDEO_CC_MVS0C_GDSC 0 +#define VIDEO_CC_MVS0_GDSC 1 +#define VIDEO_CC_MVS1C_GDSC 2 +#define VIDEO_CC_MVS1_GDSC 3 + +/* VIDEO_CC resets */ +#define CVP_VIDEO_CC_INTERFACE_BCR 0 +#define CVP_VIDEO_CC_MVS0_BCR 1 +#define CVP_VIDEO_CC_MVS0C_BCR 2 +#define CVP_VIDEO_CC_MVS1_BCR 3 +#define CVP_VIDEO_CC_MVS1C_BCR 4 +#define VIDEO_CC_MVS0C_CLK_ARES 5 +#define VIDEO_CC_MVS1C_CLK_ARES 6 +#define VIDEO_CC_XO_CLK_ARES 7 +#define VIDEO_CC_MVS0_BSE_BCR 8 + +#endif diff --git a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h index c4863e444458..aa47685f329a 100644 --- a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h +++ b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h @@ -34,5 +34,7 @@ #define R9A09G077_XSPI_CLK0 22 #define R9A09G077_XSPI_CLK1 23 #define R9A09G077_PCLKCAN 24 +#define R9A09G077_LCDC_CLKD 25 +#define R9A09G077_PCLKRTC 26 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__ */ diff --git a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h index 0d53f1e65077..1c73d0dcef18 100644 --- a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h +++ b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h @@ -34,5 +34,7 @@ #define R9A09G087_XSPI_CLK0 22 #define R9A09G087_XSPI_CLK1 23 #define R9A09G087_PCLKCAN 24 +#define R9A09G087_LCDC_CLKD 25 +#define R9A09G087_PCLKRTC 26 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__ */ diff --git a/include/dt-bindings/clock/rockchip,rk3588-cru.h b/include/dt-bindings/clock/rockchip,rk3588-cru.h index 0c7d3ca2d5bc..7528034cff56 100644 --- a/include/dt-bindings/clock/rockchip,rk3588-cru.h +++ b/include/dt-bindings/clock/rockchip,rk3588-cru.h @@ -734,6 +734,10 @@ #define PCLK_AV1_PRE 719 #define HCLK_SDIO_PRE 720 #define PCLK_VO1GRF 721 +#define I2S0_8CH_MCLKOUT_TO_IO 722 +#define I2S1_8CH_MCLKOUT_TO_IO 723 +#define I2S2_2CH_MCLKOUT_TO_IO 724 +#define I2S3_2CH_MCLKOUT_TO_IO 725 /* scmi-clocks indices */ diff --git a/include/dt-bindings/clock/spacemit,k3-clocks.h b/include/dt-bindings/clock/spacemit,k3-clocks.h index b22336f3ae40..dfae52547cda 100644 --- a/include/dt-bindings/clock/spacemit,k3-clocks.h +++ b/include/dt-bindings/clock/spacemit,k3-clocks.h @@ -380,6 +380,11 @@ #define CLK_APMU_ISIM_VCLK1 86 #define CLK_APMU_ISIM_VCLK2 87 #define CLK_APMU_ISIM_VCLK3 88 +#define CLK_APMU_PCIE_PORTA_DBI 89 +#define CLK_APMU_PCIE_PORTB_DBI 90 +#define CLK_APMU_PCIE_PORTC_DBI 91 +#define CLK_APMU_PCIE_PORTD_DBI 92 +#define CLK_APMU_PCIE_PORTE_DBI 93 /* DCIU clocks */ #define CLK_DCIU_HDMA 0 diff --git a/include/dt-bindings/interconnect/qcom,eliza-rpmh.h b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h index 95db2fe647de..dfe99feefb27 100644 --- a/include/dt-bindings/interconnect/qcom,eliza-rpmh.h +++ b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h @@ -57,6 +57,7 @@ #define SLAVE_PCIE_ANOC_CFG 27 #define SLAVE_QDSS_STM 28 #define SLAVE_TCU 29 +#define SLAVE_SDCC_1 30 #define MASTER_GEM_NOC_CNOC 0 #define MASTER_GEM_NOC_PCIE_SNOC 1 diff --git a/include/dt-bindings/interconnect/qcom,hawi-rpmh.h b/include/dt-bindings/interconnect/qcom,hawi-rpmh.h new file mode 100644 index 000000000000..a8b649679846 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,hawi-rpmh.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_HAWI_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_HAWI_H + +#define MASTER_QSPI_0 0 +#define MASTER_QUP_2 1 +#define MASTER_QUP_3 2 +#define MASTER_QUP_4 3 +#define MASTER_CRYPTO 4 +#define MASTER_IPA 5 +#define MASTER_QUP_1 6 +#define MASTER_SOCCP_PROC 7 +#define MASTER_QDSS_ETR 8 +#define MASTER_QDSS_ETR_1 9 +#define MASTER_SDCC_2 10 +#define MASTER_SDCC_4 11 +#define MASTER_UFS_MEM 12 +#define MASTER_USB3 13 +#define SLAVE_A1NOC_SNOC 14 + +#define MASTER_DDR_EFF_VETO 0 +#define MASTER_QUP_CORE_0 1 +#define MASTER_QUP_CORE_1 2 +#define MASTER_QUP_CORE_2 3 +#define MASTER_QUP_CORE_3 4 +#define MASTER_QUP_CORE_4 5 +#define SLAVE_DDR_EFF_VETO 6 +#define SLAVE_QUP_CORE_0 7 +#define SLAVE_QUP_CORE_1 8 +#define SLAVE_QUP_CORE_2 9 +#define SLAVE_QUP_CORE_3 10 +#define SLAVE_QUP_CORE_4 11 + +#define MASTER_GEM_NOC_CNOC 0 +#define MASTER_GEM_NOC_PCIE_SNOC 1 +#define SLAVE_AOSS 2 +#define SLAVE_IPA_CFG 3 +#define SLAVE_IPC_ROUTER_FENCE 4 +#define SLAVE_SOCCP 5 +#define SLAVE_TME_CFG 6 +#define SLAVE_CNOC_CFG 7 +#define SLAVE_DDRSS_CFG 8 +#define SLAVE_IMEM 9 +#define SLAVE_PCIE_0 10 +#define SLAVE_PCIE_1 11 + +#define MASTER_GIC 0 +#define MASTER_GPU_TCU 1 +#define MASTER_SYS_TCU 2 +#define MASTER_APPSS_PROC 3 +#define MASTER_GFX3D 4 +#define MASTER_LPASS_GEM_NOC 5 +#define MASTER_MSS_PROC 6 +#define MASTER_MNOC_HF_MEM_NOC 7 +#define MASTER_MNOC_SF_MEM_NOC 8 +#define MASTER_COMPUTE_NOC 9 +#define MASTER_ANOC_PCIE_GEM_NOC 10 +#define MASTER_QPACE 11 +#define MASTER_SNOC_SF_MEM_NOC 12 +#define MASTER_WLAN_Q6 13 +#define SLAVE_GEM_NOC_CNOC 14 +#define SLAVE_LLCC 15 +#define SLAVE_MEM_NOC_PCIE_SNOC 16 + +#define MASTER_LPIAON_NOC_LLCLPI_NOC 0 +#define SLAVE_LPASS_LPI_CC 1 +#define SLAVE_LLCC_ISLAND 2 +#define SLAVE_SERVICE_LLCLPI_NOC 3 +#define SLAVE_SERVICE_LLCLPI_NOC_CHIPCX 4 + +#define MASTER_LPIAON_NOC 0 +#define SLAVE_LPASS_GEM_NOC 1 + +#define MASTER_LPASS_LPINOC 0 +#define SLAVE_LPIAON_NOC_LLCLPI_NOC 1 +#define SLAVE_LPIAON_NOC_LPASS_AG_NOC 2 + +#define MASTER_LPASS_PROC 0 +#define SLAVE_LPICX_NOC_LPIAON_NOC 1 + +#define MASTER_LLCC 0 +#define MASTER_DDR_RT 1 +#define SLAVE_EBI1 2 +#define SLAVE_DDR_RT 3 + +#define MASTER_CAMNOC_HF 0 +#define MASTER_CAMNOC_NRT_ICP_SF 1 +#define MASTER_CAMNOC_RT_CDM_SF 2 +#define MASTER_CAMNOC_SF 3 +#define MASTER_MDP 4 +#define MASTER_MDSS_DCP 5 +#define MASTER_CDSP_HCP 6 +#define MASTER_VIDEO_CV_PROC 7 +#define MASTER_VIDEO_EVA 8 +#define MASTER_VIDEO_MVP 9 +#define MASTER_VIDEO_V_PROC 10 +#define SLAVE_MNOC_HF_MEM_NOC 11 +#define SLAVE_MNOC_SF_MEM_NOC 12 + +#define MASTER_CDSP_PROC 0 +#define SLAVE_CDSP_MEM_NOC 1 + +#define MASTER_PCIE_ANOC_CFG 0 +#define MASTER_PCIE_0 1 +#define MASTER_PCIE_1 2 +#define SLAVE_ANOC_PCIE_GEM_NOC 3 +#define SLAVE_SERVICE_PCIE_ANOC 4 + +#define MASTER_CFG_CENTER 0 +#define MASTER_CFG_EAST 1 +#define MASTER_CFG_MM 2 +#define MASTER_CFG_NORTH 3 +#define MASTER_CFG_SOUTH 4 +#define MASTER_CFG_SOUTHWEST 5 +#define SLAVE_AHB2PHY_SOUTH 6 +#define SLAVE_BOOT_ROM 7 +#define SLAVE_CAMERA_CFG 8 +#define SLAVE_CLK_CTL 9 +#define SLAVE_CRYPTO_CFG 10 +#define SLAVE_DISPLAY_CFG 11 +#define SLAVE_EVA_CFG 12 +#define SLAVE_GFX3D_CFG 13 +#define SLAVE_I2C 14 +#define SLAVE_IMEM_CFG 15 +#define SLAVE_IPC_ROUTER_CFG 16 +#define SLAVE_IRIS_CFG 17 +#define SLAVE_CNOC_MSS 18 +#define SLAVE_PCIE_0_CFG 19 +#define SLAVE_PCIE_1_CFG 20 +#define SLAVE_PRNG 21 +#define SLAVE_QSPI_0 22 +#define SLAVE_QUP_1 23 +#define SLAVE_QUP_2 24 +#define SLAVE_QUP_3 25 +#define SLAVE_QUP_4 26 +#define SLAVE_SDCC_2 27 +#define SLAVE_SDCC_4 28 +#define SLAVE_TLMM 29 +#define SLAVE_UFS_MEM_CFG 30 +#define SLAVE_USB3 31 +#define SLAVE_VSENSE_CTRL_CFG 32 +#define SLAVE_PCIE_ANOC_CFG 33 +#define SLAVE_QDSS_CFG 34 +#define SLAVE_QDSS_STM 35 +#define SLAVE_TCSR 36 +#define SLAVE_TCU 37 + +#define MASTER_CNOC_STARDUST 0 +#define SLAVE_STARDUST_CENTER_CFG 1 +#define SLAVE_STARDUST_EAST_CFG 2 +#define SLAVE_STARDUST_MM_CFG 3 +#define SLAVE_STARDUST_NORTH_CFG 4 +#define SLAVE_STARDUST_SOUTH_CFG 5 +#define SLAVE_STARDUST_SOUTHWEST_CFG 6 + +#define MASTER_A1NOC_SNOC 0 +#define MASTER_APSS_NOC 1 +#define MASTER_CNOC_SNOC 2 +#define SLAVE_SNOC_GEM_NOC_SF 3 + +#endif diff --git a/include/dt-bindings/interconnect/qcom,nord-rpmh.h b/include/dt-bindings/interconnect/qcom,nord-rpmh.h new file mode 100644 index 000000000000..5bdce6a9bab7 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,nord-rpmh.h @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_NORD_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_NORD_H + +#define MASTER_QSPI_0 0 +#define MASTER_SAILSS_MD1 1 +#define MASTER_QUP_3 2 +#define SLAVE_A1NOC_SNOC 3 + +#define MASTER_QUP_2 0 +#define MASTER_CRYPTO_CORE0 1 +#define MASTER_CRYPTO_CORE1 2 +#define MASTER_CRYPTO_CORE2 3 +#define MASTER_SDCC_4 4 +#define MASTER_UFS_MEM 5 +#define MASTER_USB2 6 +#define MASTER_USB3_0 7 +#define MASTER_USB3_1 8 +#define SLAVE_A1NOC_HSCNOC 9 + +#define MASTER_IPA 0 +#define MASTER_SOCCP_AGGR_NOC 1 +#define MASTER_QDSS_ETR 2 +#define MASTER_QDSS_ETR_1 3 +#define SLAVE_A2NOC_SNOC 4 + +#define MASTER_QUP_0 0 +#define MASTER_QUP_1 1 +#define MASTER_EMAC_0 2 +#define MASTER_EMAC_1 3 +#define SLAVE_A2NOC_HSCNOC 4 + +#define MASTER_QUP_CORE_0 0 +#define MASTER_QUP_CORE_1 1 +#define MASTER_QUP_CORE_2 2 +#define MASTER_QUP_CORE_3 3 +#define SLAVE_QUP_CORE_0 4 +#define SLAVE_QUP_CORE_1 5 +#define SLAVE_QUP_CORE_2 6 +#define SLAVE_QUP_CORE_3 7 + +#define MASTER_CNOC_CFG 0 +#define SLAVE_PS_ETH_0 1 +#define SLAVE_PS_ETH_1 2 +#define SLAVE_SHS_SERVER 3 +#define SLAVE_AHB2PHY_0 4 +#define SLAVE_AHB2PHY_1 5 +#define SLAVE_AHB2PHY_2 6 +#define SLAVE_AHB2PHY_3 7 +#define SLAVE_AHB2PHY_ETH_0 8 +#define SLAVE_AHB2PHY_ETH_1 9 +#define SLAVE_CAMERA_CFG 10 +#define SLAVE_CLK_CTL 11 +#define SLAVE_CRYPTO_0_CFG 12 +#define SLAVE_CRYPTO_1_CFG 13 +#define SLAVE_CRYPTO_2_CFG 14 +#define SLAVE_DISPLAY_1_CFG 15 +#define SLAVE_DISPLAY_CFG 16 +#define SLAVE_DPRX0 17 +#define SLAVE_DPRX1 18 +#define SLAVE_EVA_CFG 19 +#define SLAVE_GFX3D_CFG 20 +#define SLAVE_GFX3D_1_CFG 21 +#define SLAVE_I2C 22 +#define SLAVE_IMEM_CFG 23 +#define SLAVE_MCW_PCIE 24 +#define SLAVE_MM_RSCC 25 +#define SLAVE_NE_CLK_CTL 26 +#define SLAVE_NSPSS0_CFG 27 +#define SLAVE_NSPSS1_CFG 28 +#define SLAVE_NSPSS2_CFG 29 +#define SLAVE_NSPSS3_CFG 30 +#define SLAVE_NW_CLK_CTL 31 +#define SLAVE_PRNG 32 +#define SLAVE_QDSS_CFG 33 +#define SLAVE_QSPI_0 34 +#define SLAVE_QUP_0 35 +#define SLAVE_QUP_3 36 +#define SLAVE_QUP_1 37 +#define SLAVE_QUP_2 38 +#define SLAVE_SAFEDMA_CFG 39 +#define SLAVE_SDCC_4 40 +#define SLAVE_SE_CLK_CTL 41 +#define SLAVE_TCSR 42 +#define SLAVE_TLMM 43 +#define SLAVE_TSC_CFG 44 +#define SLAVE_UFS_MEM_CFG 45 +#define SLAVE_USB2 46 +#define SLAVE_USB3_0 47 +#define SLAVE_USB3_1 48 +#define SLAVE_VENUS_CFG 49 +#define SLAVE_COMPUTENOC_CFG 50 +#define SLAVE_PCIE_NOC_CFG 51 +#define SLAVE_QTC_CFG 52 +#define SLAVE_QDSS_STM 53 +#define SLAVE_SYS_TCU0_CFG 54 +#define SLAVE_SYS_TCU1_CFG 55 +#define SLAVE_SYS_TCU2_CFG 56 + +#define MASTER_MM_RSCC 0 +#define MASTER_HSCNOC_CNOC 1 +#define SLAVE_AOSS 2 +#define SLAVE_HBCU 3 +#define SLAVE_IPA_CFG 4 +#define SLAVE_IPC_ROUTER_CFG 5 +#define SLAVE_SOCCP 6 +#define SLAVE_TME_CFG 7 +#define SLAVE_PCIE_DMA 8 +#define SLAVE_CNOC_CFG 9 +#define SLAVE_DDRSS_CFG 10 +#define SLAVE_IMEM 11 + +#define MASTER_HPASS_PROC_0 0 +#define MASTER_HPASS_PROC_1 1 +#define MASTER_HPASS_PROC_2 2 +#define SLAVE_HPASS_AGNOC_AUDIO 3 + +#define MASTER_GPU_TCU 0 +#define MASTER_QTC_TCU 1 +#define MASTER_SYS_TCU_0 2 +#define MASTER_SYS_TCU_1 3 +#define MASTER_SYS_TCU_2 4 +#define MASTER_APPSS_PROC 5 +#define MASTER_A1NOC_TILE_HSCNOC 6 +#define MASTER_A2NOC_TILE_HSCNOC 7 +#define MASTER_GFX3D 8 +#define MASTER_GFX3D_1 9 +#define MASTER_HPASS_ADAS_HSCNOC 10 +#define MASTER_HPASS_AUDIO_HSCNOC 11 +#define MASTER_MNOC_HF_MEM_NOC 12 +#define MASTER_MNOC_SF_MEM_NOC 13 +#define MASTER_NSP0_HSCNOC 14 +#define MASTER_NSP1_HSCNOC 15 +#define MASTER_NSP2_HSCNOC 16 +#define MASTER_NSP3_HSCNOC 17 +#define MASTER_ANOC_PCIE_GEM_NOC 18 +#define MASTER_SAILSS_MD0_HSCNOC 19 +#define MASTER_SNOC_SF_MEM_NOC 20 +#define MASTER_GIC 21 +#define SLAVE_HSCNOC_CNOC 22 +#define SLAVE_LLCC 23 +#define SLAVE_MEM_NOC_PCIE_SNOC 24 + +#define MASTER_LLCC 0 +#define SLAVE_EBI1 1 + +#define MASTER_CAMNOC_HF 0 +#define MASTER_CAMNOC_NRT_ICP_SF 1 +#define MASTER_CAMNOC_RT_CDM_SF 2 +#define MASTER_CAMNOC_SF 3 +#define MASTER_DPRX0 4 +#define MASTER_DPRX1 5 +#define MASTER_MDP0 6 +#define MASTER_MDP1 7 +#define MASTER_VIDEO_CV_PROC 8 +#define MASTER_VIDEO_EVA 9 +#define MASTER_VIDEO_MVP0 10 +#define MASTER_VIDEO_MVP1 11 +#define MASTER_VIDEO_V_PROC 12 +#define SLAVE_MNOC_HF_MEM_NOC 13 +#define SLAVE_MNOC_SF_MEM_NOC 14 + +#define MASTER_NSP0_PROC 0 +#define SLAVE_NSP0_HSC_NOC 1 + +#define MASTER_NSP1_PROC 0 +#define SLAVE_NSP1_HSC_NOC 1 + +#define MASTER_NSP2_PROC 0 +#define SLAVE_NSP2_HSC_NOC 1 + +#define MASTER_NSP3_PROC 0 +#define SLAVE_NSP3_HSC_NOC 1 + +#define MASTER_PCIE_NOC_CFG 0 +#define SLAVE_PCIE_AHB2PHY_CFG 1 +#define SLAVE_PCIE_CFG_0 2 +#define SLAVE_PCIE_CFG_1 3 +#define SLAVE_PCIE_CFG_2 4 +#define SLAVE_PCIE_CFG_3 5 +#define SLAVE_PCIE_DMA_0_CFG 6 +#define SLAVE_PCIE_DMA_1_CFG 7 +#define SLAVE_PCIE_DMA_2_CFG 8 + +#define MASTER_PCIE_DMA_0 0 +#define MASTER_PCIE_DMA_1 1 +#define MASTER_PCIE_DMA_2 2 +#define MASTER_PCIE_0 3 +#define MASTER_PCIE_1 4 +#define MASTER_PCIE_2 5 +#define MASTER_PCIE_3 6 +#define SLAVE_PCIE_HSCNOC 7 +#define SLAVE_PCIE_OBNOC_DMA 8 + +#define MASTER_CNOC_PCIE_DMA 0 +#define MASTER_ANOC_PCIE_HSCNOC 1 +#define MASTER_PCIE_IBNOC_DMA 2 +#define SLAVE_PCIE_DMA_0 3 +#define SLAVE_PCIE_DMA_1 4 +#define SLAVE_PCIE_DMA_2 5 +#define SLAVE_PCIE_0 6 +#define SLAVE_PCIE_1 7 +#define SLAVE_PCIE_2 8 +#define SLAVE_PCIE_3 9 + +#define MASTER_A1NOC_SNOC 0 +#define MASTER_A2NOC_SNOC 1 +#define MASTER_CNOC_SNOC 2 +#define MASTER_NSINOC_SNOC 3 +#define MASTER_SAFE_DMA 4 +#define SLAVE_SNOC_HSCNOC_SF 5 + +#endif diff --git a/include/dt-bindings/interconnect/qcom,shikra.h b/include/dt-bindings/interconnect/qcom,shikra.h new file mode 100644 index 000000000000..a42ea22ee162 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,shikra.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SHIKRA_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_SHIKRA_H + +#define MASTER_QUP_CORE_0 0 +#define SLAVE_QUP_CORE_0 1 + +#define SNOC_CNOC_MAS 0 +#define MASTER_QDSS_DAP 1 +#define SLAVE_AHB2PHY_USB 2 +#define SLAVE_APSS_THROTTLE_CFG 3 +#define SLAVE_AUDIO 4 +#define SLAVE_BOOT_ROM 5 +#define SLAVE_CAMERA_NRT_THROTTLE_CFG 6 +#define SLAVE_CAMERA_CFG 7 +#define SLAVE_CDSP_THROTTLE_CFG 8 +#define SLAVE_CLK_CTL 9 +#define SLAVE_DSP_CFG 10 +#define SLAVE_RBCPR_CX_CFG 11 +#define SLAVE_RBCPR_MX_CFG 12 +#define SLAVE_CRYPTO_0_CFG 13 +#define SLAVE_DDR_SS_CFG 14 +#define SLAVE_DISPLAY_CFG 15 +#define SLAVE_EMAC0_CFG 16 +#define SLAVE_EMAC1_CFG 17 +#define SLAVE_GPU_CFG 18 +#define SLAVE_GPU_THROTTLE_CFG 19 +#define SLAVE_HWKM 20 +#define SLAVE_IMEM_CFG 21 +#define SLAVE_MAPSS 22 +#define SLAVE_MDSP_MPU_CFG 23 +#define SLAVE_MESSAGE_RAM 24 +#define SLAVE_MSS 25 +#define SLAVE_PCIE_CFG 26 +#define SLAVE_PDM 27 +#define SLAVE_PIMEM_CFG 28 +#define SLAVE_PKA_WRAPPER_CFG 29 +#define SLAVE_PMIC_ARB 30 +#define SLAVE_QDSS_CFG 31 +#define SLAVE_QM_CFG 32 +#define SLAVE_QM_MPU_CFG 33 +#define SLAVE_QPIC 34 +#define SLAVE_QUP_0 35 +#define SLAVE_RPM 36 +#define SLAVE_SDCC_1 37 +#define SLAVE_SDCC_2 38 +#define SLAVE_SECURITY 39 +#define SLAVE_SNOC_CFG 40 +#define SNOC_SF_THROTTLE_CFG 41 +#define SLAVE_TLMM 42 +#define SLAVE_TSCSS 43 +#define SLAVE_USB2 44 +#define SLAVE_USB3 45 +#define SLAVE_VENUS_CFG 46 +#define SLAVE_VENUS_THROTTLE_CFG 47 +#define SLAVE_VSENSE_CTRL_CFG 48 +#define SLAVE_SERVICE_CNOC 49 + +#define MASTER_LLCC 0 +#define SLAVE_EBI_CH0 1 + +#define MASTER_GRAPHICS_3D 0 +#define MASTER_MNOC_HF_MEM_NOC 1 +#define MASTER_ANOC_PCIE_MEM_NOC 2 +#define MASTER_SNOC_SF_MEM_NOC 3 +#define MASTER_AMPSS_M0 4 +#define MASTER_SYS_TCU 5 +#define SLAVE_LLCC 6 +#define SLAVE_MEMNOC_SNOC 7 +#define SLAVE_MEM_NOC_PCIE_SNOC 8 + +#define MASTER_CAMNOC_SF 0 +#define MASTER_VIDEO_P0 1 +#define MASTER_VIDEO_PROC 2 +#define SLAVE_MMNRT_VIRT 3 + +#define MASTER_CAMNOC_HF 0 +#define MASTER_MDP_PORT0 1 +#define MASTER_MMRT_VIRT 2 +#define SLAVE_MM_MEMNOC 3 + +#define MASTER_SNOC_CFG 0 +#define MASTER_TIC 1 +#define MASTER_ANOC_SNOC 2 +#define MASTER_MEMNOC_PCIE 3 +#define MASTER_MEMNOC_SNOC 4 +#define MASTER_PIMEM 5 +#define MASTER_PCIE2_0 6 +#define MASTER_QDSS_BAM 7 +#define MASTER_QPIC 8 +#define MASTER_QUP_0 9 +#define CNOC_SNOC_MAS 10 +#define MASTER_AUDIO 11 +#define MASTER_EMAC_0 12 +#define MASTER_EMAC_1 13 +#define MASTER_QDSS_ETR 14 +#define MASTER_SDCC_1 15 +#define MASTER_SDCC_2 16 +#define MASTER_USB2_0 17 +#define MASTER_USB3 18 +#define MASTER_CRYPTO_CORE0 19 +#define SLAVE_APPSS 20 +#define SLAVE_MCUSS 21 +#define SLAVE_WCSS 22 +#define SLAVE_MEMNOC_SF 23 +#define SNOC_CNOC_SLV 24 +#define SLAVE_BOOTIMEM 25 +#define SLAVE_OCIMEM 26 +#define SLAVE_PIMEM 27 +#define SLAVE_SERVICE_SNOC 28 +#define SLAVE_PCIE2_0 29 +#define SLAVE_QDSS_STM 30 +#define SLAVE_TCU 31 +#define SLAVE_PCIE_MEMNOC 32 +#define SLAVE_ANOC_SNOC 33 + +#endif diff --git a/include/dt-bindings/power/qcom,rpmhpd.h b/include/dt-bindings/power/qcom,rpmhpd.h index 07bd2a7b0150..6c9653264bb7 100644 --- a/include/dt-bindings/power/qcom,rpmhpd.h +++ b/include/dt-bindings/power/qcom,rpmhpd.h @@ -32,6 +32,7 @@ #define RPMHPD_GBX 22 #define RPMHPD_NSP3 23 #define RPMHPD_GFX1 24 +#define RPMHPD_GFX0 25 /* RPMh Power Domain performance levels */ #define RPMH_REGULATOR_LEVEL_RETENTION 16 diff --git a/include/dt-bindings/reset/altr,rst-mgr-s10.h b/include/dt-bindings/reset/altr,rst-mgr-s10.h index 04c4d0c6fd34..c2505b9eb63e 100644 --- a/include/dt-bindings/reset/altr,rst-mgr-s10.h +++ b/include/dt-bindings/reset/altr,rst-mgr-s10.h @@ -22,7 +22,7 @@ #define USB0_RESET 35 #define USB1_RESET 36 #define NAND_RESET 37 -/* 38 is empty */ +#define COMBOPHY_RESET 38 #define SDMMC_RESET 39 #define EMAC0_OCP_RESET 40 #define EMAC1_OCP_RESET 41 diff --git a/include/kunit/test.h b/include/kunit/test.h index e52452e58305..da5312e0dfa5 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -285,6 +285,7 @@ struct kunit_suite { struct string_stream *log; int suite_init_err; bool is_init; + enum kunit_status status; }; /* Stores an array of suites, end points one past the end */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 67effb91fa98..60ab50cb8930 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -13,7 +13,7 @@ #include /* for struct resource */ #include #include -#include +#include #include #include #include @@ -798,6 +798,8 @@ int acpi_get_local_u64_address(acpi_handle handle, u64 *addr); int acpi_get_local_address(acpi_handle handle, u32 *addr); const char *acpi_get_subsystem_id(acpi_handle handle); +struct device *acpi_bus_find_device_by_name(const char *name); + #ifdef CONFIG_ACPI_MRRM int acpi_mrrm_max_mem_region(void); #endif @@ -1106,6 +1108,11 @@ static inline const char *acpi_get_subsystem_id(acpi_handle handle) return ERR_PTR(-ENODEV); } +static inline struct device *acpi_bus_find_device_by_name(const char *name) +{ + return NULL; +} + static inline int acpi_register_wakeup_handler(int wake_irq, bool (*wakeup)(void *context), void *context) { diff --git a/include/linux/adi-axi-common.h b/include/linux/adi-axi-common.h index 37962ba530df..e7ba393061ee 100644 --- a/include/linux/adi-axi-common.h +++ b/include/linux/adi-axi-common.h @@ -51,6 +51,7 @@ enum adi_axi_fpga_technology { ADI_AXI_FPGA_TECH_SERIES7, ADI_AXI_FPGA_TECH_ULTRASCALE, ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS, + ADI_AXI_FPGA_TECH_VERSAL, }; enum adi_axi_fpga_family { @@ -71,6 +72,7 @@ enum adi_axi_fpga_speed_grade { ADI_AXI_FPGA_SPEED_2 = 20, ADI_AXI_FPGA_SPEED_2L = 21, ADI_AXI_FPGA_SPEED_2LV = 22, + ADI_AXI_FPGA_SPEED_2MP = 23, ADI_AXI_FPGA_SPEED_3 = 30, }; diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 02de2ede560f..068ba2e77c5d 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -163,11 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } -void alloc_tag_add_early_pfn(unsigned long pfn); +void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} -static inline void alloc_tag_add_early_pfn(unsigned long pfn) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {} #endif /* Caller should verify both ref and tag to be valid */ diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index 6c54d5c0d21f..80a74cd2da7e 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include diff --git a/include/linux/ata.h b/include/linux/ata.h index 8fd48bcb2a46..ac5616a9668b 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -75,6 +75,7 @@ enum { ATA_ID_HW_CONFIG = 93, ATA_ID_SPG = 98, ATA_ID_LBA_CAPACITY_2 = 100, + ATA_ID_MAX_PAGES_PER_DSM = 105, ATA_ID_SECTOR_SIZE = 106, ATA_ID_WWN = 108, ATA_ID_LOGICAL_SECTOR_SIZE = 117, /* and 118 */ @@ -928,6 +929,18 @@ static inline bool ata_id_has_trim(const u16 *id) return false; } +static inline u16 ata_id_dsm_max_pages(const u16 *id) +{ + /* + * IDENTIFY DEVICE word 105: MAX PAGES PER DSM COMMAND. Maximum number + * of 512-byte pages of LBA Range Entries the device accepts in a + * single DATA SET MANAGEMENT command. Zero means the device does not + * specify a limit. The field is reserved unless TRIM is supported, so + * callers must gate on ata_id_has_trim(). + */ + return id[ATA_ID_MAX_PAGES_PER_DSM]; +} + static inline bool ata_id_has_zero_after_trim(const u16 *id) { /* DSM supported, deterministic read, and read zero after trim set */ diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index 4e1ad8ccbcdd..de0ecd0fb05a 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -9,7 +9,7 @@ #define _AUXILIARY_BUS_H_ #include -#include +#include /** * DOC: DEVICE_LIFESPAN diff --git a/include/linux/bcma/bcma.h b/include/linux/bcma/bcma.h index 60b94b944e9f..f02cb3909375 100644 --- a/include/linux/bcma/bcma.h +++ b/include/linux/bcma/bcma.h @@ -3,7 +3,7 @@ #define LINUX_BCMA_H_ #include -#include +#include #include #include diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index af5178434ec6..c3dda32fd803 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -141,7 +141,7 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page, } #endif /* CONFIG_BLK_DEV_INTEGRITY */ -void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer); +void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer); void bio_integrity_free_buf(struct bio_integrity_payload *bip); void bio_integrity_setup_default(struct bio *bio); diff --git a/include/linux/bio.h b/include/linux/bio.h index 8f33f717b14f..3a0e9cb6c2f6 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio) return page_folio(bio_first_page_all(bio)); } +static inline struct bio_vec *bio_last_bvec_all(struct bio *bio) +{ + WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); + return &bio->bi_io_vec[bio->bi_vcnt - 1]; +} + /** * struct folio_iter - State for iterating all folios in a bio. * @folio: The current folio we're iterating. NULL after the last folio. diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index b007d54a9036..8854acf77869 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -2,7 +2,7 @@ #ifndef __LINUX_BITMAP_H #define __LINUX_BITMAP_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -894,6 +894,6 @@ void bitmap_write(unsigned long *map, unsigned long value, #define bitmap_set_value8(map, value, start) \ bitmap_write(map, value, start, BITS_PER_BYTE) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __LINUX_BITMAP_H */ diff --git a/include/linux/bits.h b/include/linux/bits.h index a40cc861b3a7..b7509f15718e 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -17,7 +17,7 @@ * position @h. For example * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) /* * Missing asm support @@ -75,7 +75,7 @@ #define BIT_U32(nr) BIT_TYPE(u32, nr) #define BIT_U64(nr) BIT_TYPE(u64, nr) -#else /* defined(__ASSEMBLY__) */ +#else /* defined(__ASSEMBLER__) */ /* * BUILD_BUG_ON_ZERO is not available in h files included from asm files, @@ -84,6 +84,6 @@ #define GENMASK(h, l) __GENMASK(h, l) #define GENMASK_ULL(h, l) __GENMASK_ULL(h, l) -#endif /* !defined(__ASSEMBLY__) */ +#endif /* !defined(__ASSEMBLER__) */ #endif /* __LINUX_BITS_H */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c..5a725a0cd35f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -66,7 +66,7 @@ struct block_device { int bd_holders; struct kobject *bd_holder_dir; - atomic_t bd_fsfreeze_count; /* number of freeze requests */ + atomic_t bd_fsfreeze_count; /* >0 freeze requests, <0 freeze deniers */ struct mutex bd_fsfreeze_mutex; /* serialize freeze/thaw */ struct partition_meta_info *bd_meta_info; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5070851cf924..dbb549cdfb77 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -126,8 +126,6 @@ struct blk_integrity { unsigned char pi_tuple_size; }; -typedef unsigned int __bitwise blk_mode_t; - /* open for reading */ #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) /* open for writing */ @@ -1222,16 +1220,12 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) __blk_flush_plug(plug, async); } -/* - * tsk == current here - */ -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static __always_inline void blk_plug_invalidate_ts(void) { - struct blk_plug *plug = tsk->plug; - - if (plug) - plug->cur_ktime = 0; - current->flags &= ~PF_BLOCK_TS; + if (unlikely(current->flags & PF_BLOCK_TS)) { + current->plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; + } } int blkdev_issue_flush(struct block_device *bdev); @@ -1257,7 +1251,7 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) { } -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static inline void blk_plug_invalidate_ts(void) { } @@ -1774,13 +1768,6 @@ struct blk_holder_ops { __releases(&bdev->bd_holder_lock); }; -/* - * For filesystems using @fs_holder_ops, the @holder argument passed to - * helpers used to open and claim block devices via - * bd_prepare_to_claim() must point to a superblock. - */ -extern const struct blk_holder_ops fs_holder_ops; - /* * Return the correct open flags for blkdev_get_by_* for super block flags * as stored in sb->s_flags. @@ -1841,7 +1828,10 @@ static inline int early_lookup_bdev(const char *pathname, dev_t *dev) int bdev_freeze(struct block_device *bdev); int bdev_thaw(struct block_device *bdev); +int bdev_deny_freeze(struct block_device *bdev); +void bdev_allow_freeze(struct block_device *bdev); void bdev_fput(struct file *bdev_file); +void bdev_yield_claim(struct file *bdev_file); struct io_comp_batch { struct rq_list req_list; diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h index 1c7f3b74ffcf..deda507500da 100644 --- a/include/linux/bootconfig.h +++ b/include/linux/bootconfig.h @@ -308,4 +308,18 @@ static inline const char *xbc_get_embedded_bootconfig(size_t *size) } #endif +/* Bootconfig opt-in detection, shared by setup_arch() and setup_boot_config() */ +#ifdef CONFIG_BOOT_CONFIG +bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset); +#endif + +/* Build-time-rendered bootconfig cmdline prepended in setup_arch() */ +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG +void __init xbc_prepend_embedded_cmdline(char *dst, size_t size); +bool __init xbc_embedded_cmdline_applied(void); +#else +static inline void xbc_prepend_embedded_cmdline(char *dst, size_t size) { } +static inline bool xbc_embedded_cmdline_applied(void) { return false; } +#endif + #endif diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f6528445..ba09795e0bfd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 143775a27a2a..dda272d78f01 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -14,6 +14,8 @@ #ifdef CONFIG_BPF_LSM +extern bool bpf_lsm_initialized __ro_after_init; + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ RET bpf_lsm_##NAME(__VA_ARGS__); #include @@ -56,6 +58,8 @@ bool bpf_lsm_hook_returns_errno(u32 btf_id); #else /* !CONFIG_BPF_LSM */ +#define bpf_lsm_initialized false + static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) { return false; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 39a851e690ec..76b8b7627a10 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1243,6 +1243,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, bt->stack_masks[frame] |= 1ull << slot; } +static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) +{ + bt->stack_masks[frame] |= mask; +} + static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) { bt->stack_arg_masks[frame] |= 1 << slot; diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index f54770f110bc..715b026ad95b 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include #define MAX_CDX_DEV_RESOURCES 4 diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index de2cd6238c2a..7a631a257613 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -896,7 +896,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk) * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups * @tsk: target task * - * Counterpart of cgroup_threadcgroup_change_begin(). + * Counterpart of cgroup_threadgroup_change_begin(). */ static inline void cgroup_threadgroup_change_end(struct task_struct *tsk) { diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f2aa46a4f871..b905208942bf 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -480,7 +480,7 @@ static inline void cgroup_unlock(void) rcu_read_lock_sched_held() || \ lockdep_is_held(&cgroup_mutex) || \ lockdep_is_held(&css_set_lock) || \ - ((task)->flags & PF_EXITING) || (__c)) + (data_race((task)->flags) & PF_EXITING) || (__c)) #else #define task_css_set_check(task, __c) \ rcu_dereference((task)->cgroups) diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h index c360df9fa735..c9495558cd5c 100644 --- a/include/linux/clk/renesas.h +++ b/include/linux/clk/renesas.h @@ -53,6 +53,9 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { } * various parameters used to configure a PLL. These limits ensure * the PLL operates within valid and stable ranges. * + * @input_fref: Reference input frequency to the PLL (in Hz). If set + * to 0, a default value of 24MHz is used. + * * @fout: Output frequency range (in MHz) * @fout.min: Minimum allowed output frequency * @fout.max: Maximum allowed output frequency @@ -78,6 +81,8 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { } * @k.max: Maximum delta-sigma value */ struct rzv2h_pll_limits { + u32 input_fref; + struct { u32 min; u32 max; @@ -164,28 +169,51 @@ struct rzv2h_pll_div_pars { .k = { .min = -32768, .max = 32767 }, \ } \ -#ifdef CONFIG_CLK_RZV2H -bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_pars *pars, u64 freq_millihz); +#define RZG3E_CPG_PLL_DSI0_LIMITS(name) \ + static const struct rzv2h_pll_limits (name) = { \ + .fout = { .min = 25 * MEGA, .max = 1218 * MEGA }, \ + .fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA }, \ + .m = { .min = 64, .max = 533 }, \ + .p = { .min = 1, .max = 4 }, \ + .s = { .min = 0, .max = 6 }, \ + .k = { .min = -32768, .max = 32767 }, \ + } \ -bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_div_pars *pars, - const u8 *table, u8 table_size, u64 freq_millihz); +#define RZG3E_CPG_PLL_DSI1_LIMITS(name) \ + static const struct rzv2h_pll_limits (name) = { \ + .fout = { .min = 25 * MEGA, .max = 609 * MEGA }, \ + .fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA }, \ + .m = { .min = 64, .max = 533 }, \ + .p = { .min = 1, .max = 4 }, \ + .s = { .min = 0, .max = 6 }, \ + .k = { .min = -32768, .max = 32767 }, \ + } \ + +#ifdef CONFIG_CLK_RZV2H_CPG_LIB +bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_pars *pars, u64 freq_millihz); + +bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_div_pars *pars, + const u8 *table, u8 table_size, u64 freq_millihz); #else -static inline bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_pars *pars, - u64 freq_millihz) +static inline bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_pars *pars, + u64 freq_millihz) { return false; } -static inline bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_div_pars *pars, - const u8 *table, u8 table_size, - u64 freq_millihz) +static inline bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_div_pars *pars, + const u8 *table, u8 table_size, + u64 freq_millihz) { return false; } #endif +#define rzv2h_get_pll_pars rzv2h_cpg_get_pll_pars +#define rzv2h_get_pll_divs_pars rzv2h_cpg_get_pll_divs_pars + #endif diff --git a/include/linux/cnt32_to_63.h b/include/linux/cnt32_to_63.h deleted file mode 100644 index 064428479f2d..000000000000 --- a/include/linux/cnt32_to_63.h +++ /dev/null @@ -1,104 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Extend a 32-bit counter to 63 bits - * - * Author: Nicolas Pitre - * Created: December 3, 2006 - * Copyright: MontaVista Software, Inc. - */ - -#ifndef __LINUX_CNT32_TO_63_H__ -#define __LINUX_CNT32_TO_63_H__ - -#include -#include -#include - -/* this is used only to give gcc a clue about good code generation */ -union cnt32_to_63 { - struct { -#if defined(__LITTLE_ENDIAN) - u32 lo, hi; -#elif defined(__BIG_ENDIAN) - u32 hi, lo; -#endif - }; - u64 val; -}; - - -/** - * cnt32_to_63 - Expand a 32-bit counter to a 63-bit counter - * @cnt_lo: The low part of the counter - * - * Many hardware clock counters are only 32 bits wide and therefore have - * a relatively short period making wrap-arounds rather frequent. This - * is a problem when implementing sched_clock() for example, where a 64-bit - * non-wrapping monotonic value is expected to be returned. - * - * To overcome that limitation, let's extend a 32-bit counter to 63 bits - * in a completely lock free fashion. Bits 0 to 31 of the clock are provided - * by the hardware while bits 32 to 62 are stored in memory. The top bit in - * memory is used to synchronize with the hardware clock half-period. When - * the top bit of both counters (hardware and in memory) differ then the - * memory is updated with a new value, incrementing it when the hardware - * counter wraps around. - * - * Because a word store in memory is atomic then the incremented value will - * always be in synch with the top bit indicating to any potential concurrent - * reader if the value in memory is up to date or not with regards to the - * needed increment. And any race in updating the value in memory is harmless - * as the same value would simply be stored more than once. - * - * The restrictions for the algorithm to work properly are: - * - * 1) this code must be called at least once per each half period of the - * 32-bit counter; - * - * 2) this code must not be preempted for a duration longer than the - * 32-bit counter half period minus the longest period between two - * calls to this code; - * - * Those requirements ensure proper update to the state bit in memory. - * This is usually not a problem in practice, but if it is then a kernel - * timer should be scheduled to manage for this code to be executed often - * enough. - * - * And finally: - * - * 3) the cnt_lo argument must be seen as a globally incrementing value, - * meaning that it should be a direct reference to the counter data which - * can be evaluated according to a specific ordering within the macro, - * and not the result of a previous evaluation stored in a variable. - * - * For example, this is wrong: - * - * u32 partial = get_hw_count(); - * u64 full = cnt32_to_63(partial); - * return full; - * - * This is fine: - * - * u64 full = cnt32_to_63(get_hw_count()); - * return full; - * - * Note that the top bit (bit 63) in the returned value should be considered - * as garbage. It is not cleared here because callers are likely to use a - * multiplier on the returned value which can get rid of the top bit - * implicitly by making the multiplier even, therefore saving on a runtime - * clear-bit instruction. Otherwise caller must remember to clear the top - * bit explicitly. - */ -#define cnt32_to_63(cnt_lo) \ -({ \ - static u32 __m_cnt_hi; \ - union cnt32_to_63 __x; \ - __x.hi = __m_cnt_hi; \ - smp_rmb(); \ - __x.lo = (cnt_lo); \ - if (unlikely((s32)(__x.hi ^ __x.lo) < 0)) \ - __m_cnt_hi = __x.hi = (__x.hi ^ 0x80000000) + (__x.hi >> 31); \ - __x.val; \ -}) - -#endif diff --git a/include/linux/codetag.h b/include/linux/codetag.h index 8ea2a5f7c98a..ddae7484ca45 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -74,8 +74,9 @@ struct codetag_iterator { .flags = 0, \ } -void codetag_lock_module_list(struct codetag_type *cttype, bool lock); +void codetag_lock_module_list(struct codetag_type *cttype); bool codetag_trylock_module_list(struct codetag_type *cttype); +void codetag_unlock_module_list(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index 6180b803795c..539d488fdc03 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -28,8 +28,7 @@ int conv_uni_to_pc(struct vc_data *conp, long ucs); u32 conv_8bit_to_uni(unsigned char c); int conv_uni_to_8bit(u32 uni); void console_map_init(void); -bool ucs_is_double_width(uint32_t cp); -bool ucs_is_zero_width(uint32_t cp); +unsigned int ucs_get_width(uint32_t cp); u32 ucs_recompose(u32 base, u32 mark); u32 ucs_get_fallback(u32 cp); #else @@ -62,14 +61,9 @@ static inline int conv_uni_to_8bit(u32 uni) static inline void console_map_init(void) { } -static inline bool ucs_is_double_width(uint32_t cp) +static inline unsigned int ucs_get_width(uint32_t cp) { - return false; -} - -static inline bool ucs_is_zero_width(uint32_t cp) -{ - return false; + return 1; } static inline u32 ucs_recompose(u32 base, u32 mark) diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 1cf85d772bea..ddf18c970e34 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -141,6 +141,8 @@ struct csdev_access { .base = (_addr), \ }) +#define CORESIGHT_DESC_CPU_BOUND BIT(0) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -153,6 +155,8 @@ struct csdev_access { * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. * @access: Describe access to the device + * @flags: The descritpion flags. + * @cpu: The CPU this component is affined to. */ struct coresight_desc { enum coresight_dev_type type; @@ -163,6 +167,8 @@ struct coresight_desc { const struct attribute_group **groups; const char *name; struct csdev_access access; + u32 flags; + int cpu; }; /** @@ -251,6 +257,7 @@ struct coresight_trace_id_map { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. + * @path: Activated path pointer (only used for source device). * @mode: The device mode, i.e sysFS, Perf or disabled. This is actually * an 'enum cs_mode' but stored in an atomic type. Access is always * through atomic APIs, ensuring SMP-safe synchronisation between @@ -260,6 +267,7 @@ struct coresight_trace_id_map { * device's spinlock when the coresight_mutex held and mode == * CS_MODE_SYSFS. Otherwise it must be accessed from inside the * spinlock. + * @cpu: The CPU this component is affined to (-1 for not CPU bound). * @orphan: true if the component has connections that haven't been linked. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be @@ -284,8 +292,10 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; + struct coresight_path *path; atomic_t mode; int refcnt; + int cpu; bool orphan; /* sink specific fields */ bool sysfs_sink_activated; @@ -334,9 +344,9 @@ struct coresight_path { }; enum cs_mode { - CS_MODE_DISABLED, - CS_MODE_SYSFS, - CS_MODE_PERF, + CS_MODE_DISABLED = 0, + CS_MODE_SYSFS = BIT(0), + CS_MODE_PERF = BIT(1), }; #define coresight_ops(csdev) csdev->ops @@ -387,15 +397,12 @@ struct coresight_ops_link { /** * struct coresight_ops_source - basic operations for a source * Operations available for sources. - * @cpu_id: returns the value of the CPU number this component - * is associated to. * @enable: enables tracing for a source. * @disable: disables tracing for a source. * @resume_perf: resumes tracing for a source in perf session. * @pause_perf: pauses tracing for a source in perf session. */ struct coresight_ops_source { - int (*cpu_id)(struct coresight_device *csdev); int (*enable)(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode, struct coresight_path *path); void (*disable)(struct coresight_device *csdev, @@ -433,6 +440,8 @@ struct coresight_ops_panic { struct coresight_ops { int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode, struct coresight_device *sink); + int (*pm_save_disable)(struct coresight_device *csdev); + void (*pm_restore_enable)(struct coresight_device *csdev); const struct coresight_ops_sink *sink_ops; const struct coresight_ops_link *link_ops; const struct coresight_ops_source *source_ops; @@ -602,6 +611,12 @@ static inline bool coresight_is_percpu_source(struct coresight_device *csdev) (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC); } +static inline bool coresight_is_software_source(struct coresight_device *csdev) +{ + return csdev && coresight_is_device_source(csdev) && + (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE); +} + static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) { return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) && diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 22ba327ec227..0fb3a2a62eb0 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -180,7 +180,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_STARTING, CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, @@ -200,6 +199,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_AFFINITY_ONLINE, CPUHP_AP_BLK_MQ_ONLINE, CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, + CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE, diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..a67ba1b32d16 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,10 +80,11 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); @@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; @@ -224,7 +230,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} -static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return 1; } diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index c1dee3f971a9..bc087124cd78 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -59,6 +59,9 @@ extern int crash_exclude_mem_range(struct crash_mem *mem, unsigned long long mend); extern int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz); +extern int crash_prepare_headers(int need_kernel_map, void **addr, + unsigned long *sz, unsigned long *nr_mem_ranges); +extern int crash_exclude_core_ranges(struct crash_mem **cmem); struct kimage; struct kexec_segment; @@ -76,6 +79,12 @@ int kexec_should_crash(struct task_struct *p); int kexec_crash_loaded(void); void crash_save_cpu(struct pt_regs *regs, int cpu); extern int kimage_crash_copy_vmcoreinfo(struct kimage *image); +extern unsigned int arch_get_system_nr_ranges(void); +extern int arch_crash_populate_cmem(struct crash_mem *cmem); +extern int arch_crash_exclude_ranges(struct crash_mem *cmem); +extern int arch_crash_exclude_mem_range(struct crash_mem **mem, + unsigned long long mstart, + unsigned long long mend); #else /* !CONFIG_CRASH_DUMP*/ struct pt_regs; diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h index f0dc03d94ca2..30864d90d7f5 100644 --- a/include/linux/crash_reserve.h +++ b/include/linux/crash_reserve.h @@ -14,9 +14,11 @@ extern struct resource crashk_res; extern struct resource crashk_low_res; extern struct range crashk_cma_ranges[]; + +#define CRASHK_CMA_RANGES_MAX 4 #if defined(CONFIG_CMA) && defined(CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION) #define CRASHKERNEL_CMA -#define CRASHKERNEL_CMA_RANGES_MAX 4 +#define CRASHKERNEL_CMA_RANGES_MAX (CRASHK_CMA_RANGES_MAX) extern int crashk_cma_cnt; #else #define crashk_cma_cnt 0 diff --git a/include/linux/damon.h b/include/linux/damon.h index 6f7edb3590ef..805e089ff4f2 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * DAMON api - * - * Author: SeongJae Park */ #ifndef _DAMON_H_ @@ -47,8 +45,6 @@ struct damon_size_range { * @ar: The address range of the region. * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. - * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for - * each sampling interval. * @probe_hits: Number of probe-positive region samples. * @list: List head for siblings. * @age: Age of this region. @@ -61,13 +57,6 @@ struct damon_size_range { * not be done with direct access but with the helper function, * damon_update_region_access_rate(). * - * @nr_accesses_bp is another representation of @nr_accesses in basis point - * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a - * manner similar to moving sum. By the algorithm, this value becomes - * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can - * be used when the aggregation interval is too huge and therefore cannot wait - * for it before getting the access monitoring results. - * * @age is initially zero, increased for each aggregation interval, and reset * to zero again if the access frequency is significantly changed. If two * regions are merged into a new region, both @nr_accesses and @age of the new @@ -77,7 +66,6 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; - unsigned int nr_accesses_bp; unsigned char probe_hits[DAMON_MAX_PROBES]; struct list_head list; @@ -843,11 +831,13 @@ struct damon_attrs { * including damon_call() and damos_walk(). * * @ops: Set of monitoring operations for given use cases. + * @probes: Head of probes (&damon_probe) list. * @addr_unit: Scale factor for core to ops address conversion. * @min_region_sz: Minimum region size. * @pause: Pause kdamond main loop. * @adaptive_targets: Head of monitoring targets (&damon_target) list. * @schemes: Head of schemes (&damos) list. + * @rnd_state: Per-ctx PRNG state for damon_rand(). */ struct damon_ctx { struct damon_attrs attrs; @@ -905,7 +895,6 @@ struct damon_ctx { struct list_head adaptive_targets; struct list_head schemes; - /* Per-ctx PRNG state for damon_rand(); kdamond is the sole consumer. */ struct rnd_state rnd_state; }; @@ -1012,11 +1001,12 @@ struct damon_probe *damon_new_probe(void); void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); struct damon_region *damon_new_region(unsigned long start, unsigned long end); +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs); +void damon_update_region_access_rate(struct damon_region *r, bool accessed); struct damos_filter *damos_new_filter(enum damos_filter_type type, bool matching, bool allow); @@ -1065,9 +1055,13 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) { - /* {aggr,sample}_interval are unsigned long, hence could overflow */ - return min(attrs->aggr_interval / attrs->sample_interval, + unsigned long sample_interval; + unsigned long max_nr_accesses; + + sample_interval = attrs->sample_interval ? : 1; + max_nr_accesses = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); + return max_nr_accesses ? : 1; } diff --git a/include/linux/device-id/acpi.h b/include/linux/device-id/acpi.h new file mode 100644 index 000000000000..65800cefddca --- /dev/null +++ b/include/linux/device-id/acpi.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ACPI_H +#define LINUX_DEVICE_ID_ACPI_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define ACPI_ID_LEN 16 + +struct acpi_device_id { + __u8 id[ACPI_ID_LEN]; + kernel_ulong_t driver_data; + __u32 cls; + __u32 cls_msk; +}; + +/** + * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with + * the PCI-defined class-code information + * + * @_cls : the class, subclass, prog-if triple for this device + * @_msk : the class mask for this device + * + * This macro is used to create a struct acpi_device_id that matches a + * specific PCI class. The .id and .driver_data fields will be left + * initialized with the default value. + */ +#define ACPI_DEVICE_CLASS(_cls, _msk) .cls = (_cls), .cls_msk = (_msk), + +#endif /* ifndef LINUX_DEVICE_ID_ACPI_H */ diff --git a/include/linux/device-id/amba.h b/include/linux/device-id/amba.h new file mode 100644 index 000000000000..114d66a784ac --- /dev/null +++ b/include/linux/device-id/amba.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AMBA_H +#define LINUX_DEVICE_ID_AMBA_H + +/** + * struct amba_id - identifies a device on an AMBA bus + * @id: The significant bits if the hardware device ID + * @mask: Bitmask specifying which bits of the id field are significant when + * matching. A driver binds to a device when ((hardware device ID) & mask) + * == id. + * @data: Private data used by the driver. + */ +struct amba_id { + unsigned int id; + unsigned int mask; + void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_AMBA_H */ diff --git a/include/linux/device-id/ap.h b/include/linux/device-id/ap.h new file mode 100644 index 000000000000..0992333a34db --- /dev/null +++ b/include/linux/device-id/ap.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AP_H +#define LINUX_DEVICE_ID_AP_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define AP_DEVICE_ID_MATCH_CARD_TYPE 0x01 +#define AP_DEVICE_ID_MATCH_QUEUE_TYPE 0x02 + +/* s390 AP bus devices */ +struct ap_device_id { + __u16 match_flags; /* which fields to match against */ + __u8 dev_type; /* device type */ + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_AP_H */ diff --git a/include/linux/device-id/apr.h b/include/linux/device-id/apr.h new file mode 100644 index 000000000000..f282608ea018 --- /dev/null +++ b/include/linux/device-id/apr.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_APR_H +#define LINUX_DEVICE_ID_APR_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define APR_NAME_SIZE 32 +#define APR_MODULE_PREFIX "apr:" + +struct apr_device_id { + char name[APR_NAME_SIZE]; + __u32 domain_id; + __u32 svc_id; + __u32 svc_version; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_APR_H */ diff --git a/include/linux/device-id/auxiliary.h b/include/linux/device-id/auxiliary.h new file mode 100644 index 000000000000..9d512dfb23dd --- /dev/null +++ b/include/linux/device-id/auxiliary.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AUXILIARY_H +#define LINUX_DEVICE_ID_AUXILIARY_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define AUXILIARY_NAME_SIZE 40 +#define AUXILIARY_MODULE_PREFIX "auxiliary:" + +struct auxiliary_device_id { + char name[AUXILIARY_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_AUXILIARY_H */ diff --git a/include/linux/device-id/bcma.h b/include/linux/device-id/bcma.h new file mode 100644 index 000000000000..3e6b973dc4ae --- /dev/null +++ b/include/linux/device-id/bcma.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_BCMA_H +#define LINUX_DEVICE_ID_BCMA_H + +#ifdef __KERNEL__ +#include +#endif + +#define BCMA_CORE(_manuf, _id, _rev, _class) \ + { .manuf = _manuf, .id = _id, .rev = _rev, .class = _class, } + +#define BCMA_ANY_MANUF 0xFFFF +#define BCMA_ANY_ID 0xFFFF +#define BCMA_ANY_REV 0xFF +#define BCMA_ANY_CLASS 0xFF + +/* Broadcom's specific AMBA core, see drivers/bcma/ */ +struct bcma_device_id { + __u16 manuf; + __u16 id; + __u8 rev; + __u8 class; +} __attribute__((packed,aligned(2))); + +#endif /* ifndef LINUX_DEVICE_ID_BCMA_H */ diff --git a/include/linux/device-id/ccw.h b/include/linux/device-id/ccw.h new file mode 100644 index 000000000000..6b7086aa5ca8 --- /dev/null +++ b/include/linux/device-id/ccw.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CCW_H +#define LINUX_DEVICE_ID_CCW_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define CCW_DEVICE_ID_MATCH_CU_TYPE 0x01 +#define CCW_DEVICE_ID_MATCH_CU_MODEL 0x02 +#define CCW_DEVICE_ID_MATCH_DEVICE_TYPE 0x04 +#define CCW_DEVICE_ID_MATCH_DEVICE_MODEL 0x08 + +/* s390 CCW devices */ +struct ccw_device_id { + __u16 match_flags; /* which fields to match against */ + + __u16 cu_type; /* control unit type */ + __u16 dev_type; /* device type */ + __u8 cu_model; /* control unit model */ + __u8 dev_model; /* device model */ + + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CCW_H */ diff --git a/include/linux/device-id/cdx.h b/include/linux/device-id/cdx.h new file mode 100644 index 000000000000..c6cb2b5cab5a --- /dev/null +++ b/include/linux/device-id/cdx.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CDX_H +#define LINUX_DEVICE_ID_CDX_H + +#ifdef __KERNEL__ +#include +#endif + +#define CDX_ANY_ID (0xFFFF) + +enum { + CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1, +}; + +/** + * struct cdx_device_id - CDX device identifier + * @vendor: Vendor ID + * @device: Device ID + * @subvendor: Subsystem vendor ID (or CDX_ANY_ID) + * @subdevice: Subsystem device ID (or CDX_ANY_ID) + * @class: Device class + * Most drivers do not need to specify class/class_mask + * as vendor/device is normally sufficient. + * @class_mask: Limit which sub-fields of the class field are compared. + * @override_only: Match only when dev->driver_override is this driver. + * + * Type of entries in the "device Id" table for CDX devices supported by + * a CDX device driver. + */ +struct cdx_device_id { + __u16 vendor; + __u16 device; + __u16 subvendor; + __u16 subdevice; + __u32 class; + __u32 class_mask; + __u32 override_only; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CDX_H */ diff --git a/include/linux/device-id/coreboot.h b/include/linux/device-id/coreboot.h new file mode 100644 index 000000000000..ff459879781e --- /dev/null +++ b/include/linux/device-id/coreboot.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_COREBOOT_H +#define LINUX_DEVICE_ID_COREBOOT_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/** + * struct coreboot_device_id - Identifies a coreboot table entry + * @tag: tag ID + * @driver_data: driver specific data + */ +struct coreboot_device_id { + __u32 tag; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_COREBOOT_H */ diff --git a/include/linux/device-id/css.h b/include/linux/device-id/css.h new file mode 100644 index 000000000000..67435bb22618 --- /dev/null +++ b/include/linux/device-id/css.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CSS_H +#define LINUX_DEVICE_ID_CSS_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* s390 css bus devices (subchannels) */ +struct css_device_id { + __u8 match_flags; + __u8 type; /* subchannel type */ + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CSS_H */ diff --git a/include/linux/device-id/dfl.h b/include/linux/device-id/dfl.h new file mode 100644 index 000000000000..bd0c9dbafeeb --- /dev/null +++ b/include/linux/device-id/dfl.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_DFL_H +#define LINUX_DEVICE_ID_DFL_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* + * DFL (Device Feature List) + * + * DFL defines a linked list of feature headers within the device MMIO space to + * provide an extensible way of adding features. Software can walk through these + * predefined data structures to enumerate features. It is now used in the FPGA. + * See Documentation/fpga/dfl.rst for more information. + * + * The dfl bus type is introduced to match the individual feature devices (dfl + * devices) for specific dfl drivers. + */ + +/** + * struct dfl_device_id - dfl device identifier + * @type: DFL FIU type of the device. See enum dfl_id_type. + * @feature_id: feature identifier local to its DFL FIU type. + * @driver_data: driver specific data. + */ +struct dfl_device_id { + __u16 type; + __u16 feature_id; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_DFL_H */ diff --git a/include/linux/device-id/dmi.h b/include/linux/device-id/dmi.h new file mode 100644 index 000000000000..fdc4adbad133 --- /dev/null +++ b/include/linux/device-id/dmi.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_DMI_H +#define LINUX_DEVICE_ID_DMI_H + +#define DMI_MATCH(a, b) { .slot = a, .substr = b } +#define DMI_EXACT_MATCH(a, b) { .slot = a, .substr = b, .exact_match = 1 } + +/* dmi */ +enum dmi_field { + DMI_NONE, + DMI_BIOS_VENDOR, + DMI_BIOS_VERSION, + DMI_BIOS_DATE, + DMI_BIOS_RELEASE, + DMI_EC_FIRMWARE_RELEASE, + DMI_SYS_VENDOR, + DMI_PRODUCT_NAME, + DMI_PRODUCT_VERSION, + DMI_PRODUCT_SERIAL, + DMI_PRODUCT_UUID, + DMI_PRODUCT_SKU, + DMI_PRODUCT_FAMILY, + DMI_BOARD_VENDOR, + DMI_BOARD_NAME, + DMI_BOARD_VERSION, + DMI_BOARD_SERIAL, + DMI_BOARD_ASSET_TAG, + DMI_CHASSIS_VENDOR, + DMI_CHASSIS_TYPE, + DMI_CHASSIS_VERSION, + DMI_CHASSIS_SERIAL, + DMI_CHASSIS_ASSET_TAG, + DMI_STRING_MAX, + DMI_OEM_STRING, /* special case - will not be in dmi_ident */ +}; + +struct dmi_strmatch { + unsigned char slot:7; + unsigned char exact_match:1; + char substr[79]; +}; + +struct dmi_system_id { + int (*callback)(const struct dmi_system_id *); + const char *ident; + struct dmi_strmatch matches[4]; + void *driver_data; +}; + +/* + * struct dmi_device_id appears during expansion of + * "MODULE_DEVICE_TABLE(dmi, x)". Compiler doesn't look inside it + * but this is enough for gcc 3.4.6 to error out: + * error: storage size of '__mod_dmi_device_table' isn't known + */ +#define dmi_device_id dmi_system_id + +#endif /* ifndef LINUX_DEVICE_ID_DMI_H */ diff --git a/include/linux/device-id/eisa.h b/include/linux/device-id/eisa.h new file mode 100644 index 000000000000..1eeae6247524 --- /dev/null +++ b/include/linux/device-id/eisa.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_EISA_H +#define LINUX_DEVICE_ID_EISA_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* EISA */ + +#define EISA_SIG_LEN 8 + +/* The EISA signature, in ASCII form, null terminated */ +struct eisa_device_id { + char sig[EISA_SIG_LEN]; + kernel_ulong_t driver_data; +}; + +#define EISA_DEVICE_MODALIAS_FMT "eisa:s%s" + +#endif /* ifndef LINUX_DEVICE_ID_EISA_H */ diff --git a/include/linux/device-id/fsl_mc.h b/include/linux/device-id/fsl_mc.h new file mode 100644 index 000000000000..0cfa94923670 --- /dev/null +++ b/include/linux/device-id/fsl_mc.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_FSL_MC_H +#define LINUX_DEVICE_ID_FSL_MC_H + +#ifdef __KERNEL__ +#include +#endif + +/** + * struct fsl_mc_device_id - MC object device identifier + * @vendor: vendor ID + * @obj_type: MC object type + * + * Type of entries in the "device Id" table for MC object devices supported by + * a MC object device driver. The last entry of the table has vendor set to 0x0 + */ +struct fsl_mc_device_id { + __u16 vendor; + const char obj_type[16]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_FSL_MC_H */ diff --git a/include/linux/device-id/hda.h b/include/linux/device-id/hda.h new file mode 100644 index 000000000000..42542580f955 --- /dev/null +++ b/include/linux/device-id/hda.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HDA_H +#define LINUX_DEVICE_ID_HDA_H + +#ifdef __KERNEL__ +#include +#endif + +struct hda_device_id { + __u32 vendor_id; + __u32 rev_id; + __u8 api_version; + const char *name; + unsigned long driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_HDA_H */ diff --git a/include/linux/device-id/hid.h b/include/linux/device-id/hid.h new file mode 100644 index 000000000000..e865fc64bf94 --- /dev/null +++ b/include/linux/device-id/hid.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HID_H +#define LINUX_DEVICE_ID_HID_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define HID_ANY_ID (~0) +#define HID_BUS_ANY 0xffff +#define HID_GROUP_ANY 0x0000 + +struct hid_device_id { + __u16 bus; + __u16 group; + __u32 vendor; + __u32 product; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_HID_H */ diff --git a/include/linux/device-id/hv_vmbus.h b/include/linux/device-id/hv_vmbus.h new file mode 100644 index 000000000000..a7682c05619f --- /dev/null +++ b/include/linux/device-id/hv_vmbus.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HV_VMBUS_H +#define LINUX_DEVICE_ID_HV_VMBUS_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* + * For Hyper-V devices we use the device guid as the id. + */ +struct hv_vmbus_device_id { + guid_t guid; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_HV_VMBUS_H */ diff --git a/include/linux/device-id/i2c.h b/include/linux/device-id/i2c.h new file mode 100644 index 000000000000..21f9b581e7a9 --- /dev/null +++ b/include/linux/device-id/i2c.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_I2C_H +#define LINUX_DEVICE_ID_I2C_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* i2c */ + +#define I2C_NAME_SIZE 20 +#define I2C_MODULE_PREFIX "i2c:" + +struct i2c_device_id { + char name[I2C_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_I2C_H */ diff --git a/include/linux/device-id/i3c.h b/include/linux/device-id/i3c.h new file mode 100644 index 000000000000..5d8222c9f908 --- /dev/null +++ b/include/linux/device-id/i3c.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_I3C_H +#define LINUX_DEVICE_ID_I3C_H + +#ifdef __KERNEL__ +#include +#endif + +/* i3c */ + +#define I3C_MATCH_DCR 0x1 +#define I3C_MATCH_MANUF 0x2 +#define I3C_MATCH_PART 0x4 +#define I3C_MATCH_EXTRA_INFO 0x8 + +struct i3c_device_id { + __u8 match_flags; + __u8 dcr; + __u16 manuf_id; + __u16 part_id; + __u16 extra_info; + + const void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_I3C_H */ diff --git a/include/linux/device-id/ieee1394.h b/include/linux/device-id/ieee1394.h new file mode 100644 index 000000000000..63023964724a --- /dev/null +++ b/include/linux/device-id/ieee1394.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_IEEE1394_H +#define LINUX_DEVICE_ID_IEEE1394_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define IEEE1394_MATCH_VENDOR_ID 0x0001 +#define IEEE1394_MATCH_MODEL_ID 0x0002 +#define IEEE1394_MATCH_SPECIFIER_ID 0x0004 +#define IEEE1394_MATCH_VERSION 0x0008 + +struct ieee1394_device_id { + __u32 match_flags; + __u32 vendor_id; + __u32 model_id; + __u32 specifier_id; + __u32 version; + union { + kernel_ulong_t driver_data; + const void *driver_data_ptr; + }; +}; + +#endif /* ifndef LINUX_DEVICE_ID_IEEE1394_H */ diff --git a/include/linux/device-id/input.h b/include/linux/device-id/input.h new file mode 100644 index 000000000000..66d7e78d32c0 --- /dev/null +++ b/include/linux/device-id/input.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_INPUT_H +#define LINUX_DEVICE_ID_INPUT_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* Input */ +#define INPUT_DEVICE_ID_EV_MAX 0x1f +#define INPUT_DEVICE_ID_KEY_MIN_INTERESTING 0x71 +#define INPUT_DEVICE_ID_KEY_MAX 0x2ff +#define INPUT_DEVICE_ID_REL_MAX 0x0f +#define INPUT_DEVICE_ID_ABS_MAX 0x3f +#define INPUT_DEVICE_ID_MSC_MAX 0x07 +#define INPUT_DEVICE_ID_LED_MAX 0x0f +#define INPUT_DEVICE_ID_SND_MAX 0x07 +#define INPUT_DEVICE_ID_FF_MAX 0x7f +#define INPUT_DEVICE_ID_SW_MAX 0x11 +#define INPUT_DEVICE_ID_PROP_MAX 0x1f + +#define INPUT_DEVICE_ID_MATCH_BUS 1 +#define INPUT_DEVICE_ID_MATCH_VENDOR 2 +#define INPUT_DEVICE_ID_MATCH_PRODUCT 4 +#define INPUT_DEVICE_ID_MATCH_VERSION 8 + +#define INPUT_DEVICE_ID_MATCH_EVBIT 0x0010 +#define INPUT_DEVICE_ID_MATCH_KEYBIT 0x0020 +#define INPUT_DEVICE_ID_MATCH_RELBIT 0x0040 +#define INPUT_DEVICE_ID_MATCH_ABSBIT 0x0080 +#define INPUT_DEVICE_ID_MATCH_MSCIT 0x0100 +#define INPUT_DEVICE_ID_MATCH_LEDBIT 0x0200 +#define INPUT_DEVICE_ID_MATCH_SNDBIT 0x0400 +#define INPUT_DEVICE_ID_MATCH_FFBIT 0x0800 +#define INPUT_DEVICE_ID_MATCH_SWBIT 0x1000 +#define INPUT_DEVICE_ID_MATCH_PROPBIT 0x2000 + +struct input_device_id { + + kernel_ulong_t flags; + + __u16 bustype; + __u16 vendor; + __u16 product; + __u16 version; + + kernel_ulong_t evbit[INPUT_DEVICE_ID_EV_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t keybit[INPUT_DEVICE_ID_KEY_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t relbit[INPUT_DEVICE_ID_REL_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t absbit[INPUT_DEVICE_ID_ABS_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t mscbit[INPUT_DEVICE_ID_MSC_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t ledbit[INPUT_DEVICE_ID_LED_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t sndbit[INPUT_DEVICE_ID_SND_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t ffbit[INPUT_DEVICE_ID_FF_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t swbit[INPUT_DEVICE_ID_SW_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t propbit[INPUT_DEVICE_ID_PROP_MAX / BITS_PER_LONG + 1]; + + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_INPUT_H */ diff --git a/include/linux/device-id/ipack.h b/include/linux/device-id/ipack.h new file mode 100644 index 000000000000..7f9b425e1e30 --- /dev/null +++ b/include/linux/device-id/ipack.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_IPACK_H +#define LINUX_DEVICE_ID_IPACK_H + +#ifdef __KERNEL__ +#include +#endif + +#define IPACK_ANY_FORMAT 0xff +#define IPACK_ANY_ID (~0) +struct ipack_device_id { + __u8 format; /* Format version or IPACK_ANY_ID */ + __u32 vendor; /* Vendor ID or IPACK_ANY_ID */ + __u32 device; /* Device ID or IPACK_ANY_ID */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_IPACK_H */ diff --git a/include/linux/device-id/isapnp.h b/include/linux/device-id/isapnp.h new file mode 100644 index 000000000000..ba659c32650a --- /dev/null +++ b/include/linux/device-id/isapnp.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ISAPNP_H +#define LINUX_DEVICE_ID_ISAPNP_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define ISAPNP_ANY_ID 0xffff +struct isapnp_device_id { + unsigned short card_vendor, card_device; + unsigned short vendor, function; + kernel_ulong_t driver_data; /* data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_ISAPNP_H */ diff --git a/include/linux/device-id/ishtp.h b/include/linux/device-id/ishtp.h new file mode 100644 index 000000000000..c66f144b4cb6 --- /dev/null +++ b/include/linux/device-id/ishtp.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ISHTP_H +#define LINUX_DEVICE_ID_ISHTP_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* ISHTP (Integrated Sensor Hub Transport Protocol) */ + +#define ISHTP_MODULE_PREFIX "ishtp:" + +/** + * struct ishtp_device_id - ISHTP device identifier + * @guid: GUID of the device. + * @driver_data: pointer to driver specific data + */ +struct ishtp_device_id { + guid_t guid; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_ISHTP_H */ diff --git a/include/linux/device-id/mcb.h b/include/linux/device-id/mcb.h new file mode 100644 index 000000000000..a5daec32c21c --- /dev/null +++ b/include/linux/device-id/mcb.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MCB_H +#define LINUX_DEVICE_ID_MCB_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +struct mcb_device_id { + __u16 device; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MCB_H */ diff --git a/include/linux/device-id/mdio.h b/include/linux/device-id/mdio.h new file mode 100644 index 000000000000..e6cda2d91ddb --- /dev/null +++ b/include/linux/device-id/mdio.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MDIO_H +#define LINUX_DEVICE_ID_MDIO_H + +#ifdef __KERNEL__ +#include +#endif + +#define MDIO_MODULE_PREFIX "mdio:" + +#define MDIO_ID_FMT "%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u" +#define MDIO_ID_ARGS(_id) \ + ((_id)>>31) & 1, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \ + ((_id)>>27) & 1, ((_id)>>26) & 1, ((_id)>>25) & 1, ((_id)>>24) & 1, \ + ((_id)>>23) & 1, ((_id)>>22) & 1, ((_id)>>21) & 1, ((_id)>>20) & 1, \ + ((_id)>>19) & 1, ((_id)>>18) & 1, ((_id)>>17) & 1, ((_id)>>16) & 1, \ + ((_id)>>15) & 1, ((_id)>>14) & 1, ((_id)>>13) & 1, ((_id)>>12) & 1, \ + ((_id)>>11) & 1, ((_id)>>10) & 1, ((_id)>>9) & 1, ((_id)>>8) & 1, \ + ((_id)>>7) & 1, ((_id)>>6) & 1, ((_id)>>5) & 1, ((_id)>>4) & 1, \ + ((_id)>>3) & 1, ((_id)>>2) & 1, ((_id)>>1) & 1, (_id) & 1 + +/** + * struct mdio_device_id - identifies PHY devices on an MDIO/MII bus + * @phy_id: The result of + * (mdio_read(&MII_PHYSID1) << 16 | mdio_read(&MII_PHYSID2)) & @phy_id_mask + * for this PHY type + * @phy_id_mask: Defines the significant bits of @phy_id. A value of 0 + * is used to terminate an array of struct mdio_device_id. + */ +struct mdio_device_id { + __u32 phy_id; + __u32 phy_id_mask; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MDIO_H */ diff --git a/include/linux/device-id/mei_cl.h b/include/linux/device-id/mei_cl.h new file mode 100644 index 000000000000..aeeb95da4b29 --- /dev/null +++ b/include/linux/device-id/mei_cl.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MEI_CL_H +#define LINUX_DEVICE_ID_MEI_CL_H + +#ifdef __KERNEL__ +#include +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define MEI_CL_MODULE_PREFIX "mei:" +#define MEI_CL_NAME_SIZE 32 +#define MEI_CL_VERSION_ANY 0xff + +/** + * struct mei_cl_device_id - MEI client device identifier + * @name: helper name + * @uuid: client uuid + * @version: client protocol version + * @driver_info: information used by the driver. + * + * identifies mei client device by uuid and name + */ +struct mei_cl_device_id { + char name[MEI_CL_NAME_SIZE]; + uuid_le uuid; + __u8 version; + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MEI_CL_H */ diff --git a/include/linux/device-id/mhi.h b/include/linux/device-id/mhi.h new file mode 100644 index 000000000000..30a27a71a57f --- /dev/null +++ b/include/linux/device-id/mhi.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MHI_H +#define LINUX_DEVICE_ID_MHI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define MHI_DEVICE_MODALIAS_FMT "mhi:%s" +#define MHI_NAME_SIZE 32 + +#define MHI_EP_DEVICE_MODALIAS_FMT "mhi_ep:%s" + +/** + * struct mhi_device_id - MHI device identification + * @chan: MHI channel name + * @driver_data: driver data; + */ +struct mhi_device_id { + const char chan[MHI_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MHI_H */ diff --git a/include/linux/device-id/mips_cdmm.h b/include/linux/device-id/mips_cdmm.h new file mode 100644 index 000000000000..e1623884103a --- /dev/null +++ b/include/linux/device-id/mips_cdmm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MIPS_CDMM_H +#define LINUX_DEVICE_ID_MIPS_CDMM_H + +#ifdef __KERNEL__ +#include +#endif + +/** + * struct mips_cdmm_device_id - identifies devices in MIPS CDMM bus + * @type: Device type identifier. + */ +struct mips_cdmm_device_id { + __u8 type; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MIPS_CDMM_H */ diff --git a/include/linux/device-id/of.h b/include/linux/device-id/of.h new file mode 100644 index 000000000000..28ea360f8f9a --- /dev/null +++ b/include/linux/device-id/of.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_OF_H +#define LINUX_DEVICE_ID_OF_H + +/* + * Struct used for matching a device + */ +struct of_device_id { + char name[32]; + char type[32]; + char compatible[128]; + const void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_OF_H */ diff --git a/include/linux/device-id/parisc.h b/include/linux/device-id/parisc.h new file mode 100644 index 000000000000..2974eb26a8be --- /dev/null +++ b/include/linux/device-id/parisc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PARISC_H +#define LINUX_DEVICE_ID_PARISC_H + +#ifdef __KERNEL__ +#include +#endif + +#define PA_HWTYPE_ANY_ID 0xff +#define PA_HVERSION_REV_ANY_ID 0xff +#define PA_HVERSION_ANY_ID 0xffff +#define PA_SVERSION_ANY_ID 0xffffffff + +struct parisc_device_id { + __u8 hw_type; /* 5 bits used */ + __u8 hversion_rev; /* 4 bits */ + __u16 hversion; /* 12 bits */ + __u32 sversion; /* 20 bits */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_PARISC_H */ diff --git a/include/linux/device-id/pci.h b/include/linux/device-id/pci.h new file mode 100644 index 000000000000..4a635d531cd6 --- /dev/null +++ b/include/linux/device-id/pci.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PCI_H +#define LINUX_DEVICE_ID_PCI_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define PCI_ANY_ID (~0) + +enum { + PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1, +}; + +/** + * struct pci_device_id - PCI device ID structure + * @vendor: Vendor ID to match (or PCI_ANY_ID) + * @device: Device ID to match (or PCI_ANY_ID) + * @subvendor: Subsystem vendor ID to match (or PCI_ANY_ID) + * @subdevice: Subsystem device ID to match (or PCI_ANY_ID) + * @class: Device class, subclass, and "interface" to match. + * See Appendix D of the PCI Local Bus Spec or + * include/linux/pci_ids.h for a full list of classes. + * Most drivers do not need to specify class/class_mask + * as vendor/device is normally sufficient. + * @class_mask: Limit which sub-fields of the class field are compared. + * See drivers/scsi/sym53c8xx_2/ for example of usage. + * @driver_data: Data private to the driver. + * Most drivers don't need to use driver_data field. + * Best practice is to use driver_data as an index + * into a static list of equivalent device types, + * instead of using it as a pointer. + * @override_only: Match only when dev->driver_override is this driver. + */ +struct pci_device_id { + __u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/ + __u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */ + __u32 class, class_mask; /* (class,subclass,prog-if) triplet */ + kernel_ulong_t driver_data; /* Data private to the driver */ + __u32 override_only; +}; + +/* pci_epf */ + +#define PCI_EPF_NAME_SIZE 20 +#define PCI_EPF_MODULE_PREFIX "pci_epf:" + +struct pci_epf_device_id { + char name[PCI_EPF_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PCI_H */ diff --git a/include/linux/device-id/pcmcia.h b/include/linux/device-id/pcmcia.h new file mode 100644 index 000000000000..c0d809be2338 --- /dev/null +++ b/include/linux/device-id/pcmcia.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PCMCIA_H +#define LINUX_DEVICE_ID_PCMCIA_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* PCMCIA */ + +#define PCMCIA_DEV_ID_MATCH_MANF_ID 0x0001 +#define PCMCIA_DEV_ID_MATCH_CARD_ID 0x0002 +#define PCMCIA_DEV_ID_MATCH_FUNC_ID 0x0004 +#define PCMCIA_DEV_ID_MATCH_FUNCTION 0x0008 +#define PCMCIA_DEV_ID_MATCH_PROD_ID1 0x0010 +#define PCMCIA_DEV_ID_MATCH_PROD_ID2 0x0020 +#define PCMCIA_DEV_ID_MATCH_PROD_ID3 0x0040 +#define PCMCIA_DEV_ID_MATCH_PROD_ID4 0x0080 +#define PCMCIA_DEV_ID_MATCH_DEVICE_NO 0x0100 +#define PCMCIA_DEV_ID_MATCH_FAKE_CIS 0x0200 +#define PCMCIA_DEV_ID_MATCH_ANONYMOUS 0x0400 + +struct pcmcia_device_id { + __u16 match_flags; + + __u16 manf_id; + __u16 card_id; + + __u8 func_id; + + /* for real multi-function devices */ + __u8 function; + + /* for pseudo multi-function devices */ + __u8 device_no; + + __u32 prod_id_hash[4]; + + /* not matched against in kernelspace */ + const char * prod_id[4]; + + /* not matched against */ + kernel_ulong_t driver_info; + char * cisfile; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PCMCIA_H */ diff --git a/include/linux/device-id/platform.h b/include/linux/device-id/platform.h new file mode 100644 index 000000000000..d6beeb4a2574 --- /dev/null +++ b/include/linux/device-id/platform.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PLATFORM_H +#define LINUX_DEVICE_ID_PLATFORM_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define PLATFORM_NAME_SIZE 24 +#define PLATFORM_MODULE_PREFIX "platform:" + +struct platform_device_id { + char name[PLATFORM_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PLATFORM_H */ diff --git a/include/linux/device-id/pnp.h b/include/linux/device-id/pnp.h new file mode 100644 index 000000000000..325f17216df8 --- /dev/null +++ b/include/linux/device-id/pnp.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PNP_H +#define LINUX_DEVICE_ID_PNP_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define PNP_ID_LEN 8 +#define PNP_MAX_DEVICES 8 + +struct pnp_device_id { + __u8 id[PNP_ID_LEN]; + kernel_ulong_t driver_data; +}; + +struct pnp_card_device_id { + __u8 id[PNP_ID_LEN]; + kernel_ulong_t driver_data; + struct { + __u8 id[PNP_ID_LEN]; + } devs[PNP_MAX_DEVICES]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PNP_H */ diff --git a/include/linux/device-id/rio.h b/include/linux/device-id/rio.h new file mode 100644 index 000000000000..31addf69ad1f --- /dev/null +++ b/include/linux/device-id/rio.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_RIO_H +#define LINUX_DEVICE_ID_RIO_H + +#ifdef __KERNEL__ +#include +#endif + +/* RapidIO */ + +#define RIO_ANY_ID 0xffff + +/** + * struct rio_device_id - RIO device identifier + * @did: RapidIO device ID + * @vid: RapidIO vendor ID + * @asm_did: RapidIO assembly device ID + * @asm_vid: RapidIO assembly vendor ID + * + * Identifies a RapidIO device based on both the device/vendor IDs and + * the assembly device/vendor IDs. + */ +struct rio_device_id { + __u16 did, vid; + __u16 asm_did, asm_vid; +}; + +#endif /* ifndef LINUX_DEVICE_ID_RIO_H */ diff --git a/include/linux/device-id/rpmsg.h b/include/linux/device-id/rpmsg.h new file mode 100644 index 000000000000..dd53e7b7dc4f --- /dev/null +++ b/include/linux/device-id/rpmsg.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_RPMSG_H +#define LINUX_DEVICE_ID_RPMSG_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* rpmsg */ + +#define RPMSG_NAME_SIZE 32 +#define RPMSG_DEVICE_MODALIAS_FMT "rpmsg:%s" + +struct rpmsg_device_id { + char name[RPMSG_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_RPMSG_H */ diff --git a/include/linux/device-id/sdio.h b/include/linux/device-id/sdio.h new file mode 100644 index 000000000000..609d60a10cd3 --- /dev/null +++ b/include/linux/device-id/sdio.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SDIO_H +#define LINUX_DEVICE_ID_SDIO_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* SDIO */ + +#define SDIO_ANY_ID (~0) + +struct sdio_device_id { + __u8 class; /* Standard interface or SDIO_ANY_ID */ + __u16 vendor; /* Vendor or SDIO_ANY_ID */ + __u16 device; /* Device ID or SDIO_ANY_ID */ + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SDIO_H */ diff --git a/include/linux/device-id/sdw.h b/include/linux/device-id/sdw.h new file mode 100644 index 000000000000..dbed7fde1ace --- /dev/null +++ b/include/linux/device-id/sdw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SDW_H +#define LINUX_DEVICE_ID_SDW_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +struct sdw_device_id { + __u16 mfg_id; + __u16 part_id; + __u8 sdw_version; + __u8 class_id; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SDW_H */ diff --git a/include/linux/device-id/serio.h b/include/linux/device-id/serio.h new file mode 100644 index 000000000000..b4c02adf220c --- /dev/null +++ b/include/linux/device-id/serio.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SERIO_H +#define LINUX_DEVICE_ID_SERIO_H + +#ifdef __KERNEL__ +#include +#endif + +#define SERIO_ANY 0xff + +struct serio_device_id { + __u8 type; + __u8 extra; + __u8 id; + __u8 proto; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SERIO_H */ diff --git a/include/linux/device-id/slim.h b/include/linux/device-id/slim.h new file mode 100644 index 000000000000..54f8abe09e18 --- /dev/null +++ b/include/linux/device-id/slim.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SLIM_H +#define LINUX_DEVICE_ID_SLIM_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* SLIMbus */ + +#define SLIMBUS_NAME_SIZE 32 +#define SLIMBUS_MODULE_PREFIX "slim:" + +struct slim_device_id { + __u16 manf_id, prod_code; + __u16 dev_index, instance; + + /* Data private to the driver */ + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SLIM_H */ diff --git a/include/linux/device-id/spi.h b/include/linux/device-id/spi.h new file mode 100644 index 000000000000..7a77d0d6672b --- /dev/null +++ b/include/linux/device-id/spi.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SPI_H +#define LINUX_DEVICE_ID_SPI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* spi */ + +#define SPI_NAME_SIZE 32 +#define SPI_MODULE_PREFIX "spi:" + +struct spi_device_id { + char name[SPI_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SPI_H */ diff --git a/include/linux/device-id/spmi.h b/include/linux/device-id/spmi.h new file mode 100644 index 000000000000..a821c42cfc9a --- /dev/null +++ b/include/linux/device-id/spmi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SPMI_H +#define LINUX_DEVICE_ID_SPMI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define SPMI_NAME_SIZE 32 +#define SPMI_MODULE_PREFIX "spmi:" + +struct spmi_device_id { + char name[SPMI_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SPMI_H */ diff --git a/include/linux/device-id/ssam.h b/include/linux/device-id/ssam.h new file mode 100644 index 000000000000..550cca115a55 --- /dev/null +++ b/include/linux/device-id/ssam.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SSAM_H +#define LINUX_DEVICE_ID_SSAM_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* Surface System Aggregator Module */ + +#define SSAM_MATCH_TARGET 0x1 +#define SSAM_MATCH_INSTANCE 0x2 +#define SSAM_MATCH_FUNCTION 0x4 + +struct ssam_device_id { + __u8 match_flags; + + __u8 domain; + __u8 category; + __u8 target; + __u8 instance; + __u8 function; + + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SSAM_H */ diff --git a/include/linux/device-id/ssb.h b/include/linux/device-id/ssb.h new file mode 100644 index 000000000000..678d40828299 --- /dev/null +++ b/include/linux/device-id/ssb.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SSB_H +#define LINUX_DEVICE_ID_SSB_H + +#ifdef __KERNEL__ +#include +#endif + +#define SSB_ANY_VENDOR 0xFFFF +#define SSB_ANY_ID 0xFFFF +#define SSB_ANY_REV 0xFF + +/* SSB core, see drivers/ssb/ */ +struct ssb_device_id { + __u16 vendor; + __u16 coreid; + __u8 revision; + __u8 __pad; +} __attribute__((packed, aligned(2))); + +#define SSB_DEVICE(_vendor, _coreid, _revision) \ + { .vendor = _vendor, .coreid = _coreid, .revision = _revision } + +#endif /* ifndef LINUX_DEVICE_ID_SSB_H */ diff --git a/include/linux/device-id/tb.h b/include/linux/device-id/tb.h new file mode 100644 index 000000000000..4c62edff8bb7 --- /dev/null +++ b/include/linux/device-id/tb.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TB_H +#define LINUX_DEVICE_ID_TB_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define TBSVC_MATCH_PROTOCOL_KEY 0x0001 +#define TBSVC_MATCH_PROTOCOL_ID 0x0002 +#define TBSVC_MATCH_PROTOCOL_VERSION 0x0004 +#define TBSVC_MATCH_PROTOCOL_REVISION 0x0008 + +/** + * struct tb_service_id - Thunderbolt service identifiers + * @match_flags: Flags used to match the structure + * @protocol_key: Protocol key the service supports + * @protocol_id: Protocol id the service supports + * @protocol_version: Version of the protocol + * @protocol_revision: Revision of the protocol software + * @driver_data: Driver specific data + * + * Thunderbolt XDomain services are exposed as devices where each device + * carries the protocol information the service supports. Thunderbolt + * XDomain service drivers match against that information. + */ +struct tb_service_id { + __u32 match_flags; + char protocol_key[8 + 1]; + __u32 protocol_id; + __u32 protocol_version; + __u32 protocol_revision; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TB_H */ diff --git a/include/linux/device-id/tee_client.h b/include/linux/device-id/tee_client.h new file mode 100644 index 000000000000..ed81f4228185 --- /dev/null +++ b/include/linux/device-id/tee_client.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TEE_CLIENT_H +#define LINUX_DEVICE_ID_TEE_CLIENT_H + +#ifdef __KERNEL__ +#include +#endif + +/** + * struct tee_client_device_id - tee based device identifier + * @uuid: For TEE based client devices we use the device uuid as + * the identifier. + */ +struct tee_client_device_id { + uuid_t uuid; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TEE_CLIENT_H */ diff --git a/include/linux/device-id/typec.h b/include/linux/device-id/typec.h new file mode 100644 index 000000000000..dc234733408b --- /dev/null +++ b/include/linux/device-id/typec.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TYPEC_H +#define LINUX_DEVICE_ID_TYPEC_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* USB Type-C Alternate Modes */ + +#define TYPEC_ANY_MODE 0x7 + +/** + * struct typec_device_id - USB Type-C alternate mode identifiers + * @svid: Standard or Vendor ID + * @mode: Mode index + * @driver_data: Driver specific data + */ +struct typec_device_id { + __u16 svid; + __u8 mode; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TYPEC_H */ diff --git a/include/linux/device-id/ulpi.h b/include/linux/device-id/ulpi.h new file mode 100644 index 000000000000..b5105b4cfacc --- /dev/null +++ b/include/linux/device-id/ulpi.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ULPI_H +#define LINUX_DEVICE_ID_ULPI_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +struct ulpi_device_id { + __u16 vendor; + __u16 product; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_ULPI_H */ diff --git a/include/linux/device-id/usb.h b/include/linux/device-id/usb.h new file mode 100644 index 000000000000..a7ce5f6e1106 --- /dev/null +++ b/include/linux/device-id/usb.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_USB_H +#define LINUX_DEVICE_ID_USB_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* + * Device table entry for "new style" table-driven USB drivers. + * User mode code can read these tables to choose which modules to load. + * Declare the table as a MODULE_DEVICE_TABLE. + * + * A probe() parameter will point to a matching entry from this table. + * Use the driver_info field for each match to hold information tied + * to that match: device quirks, etc. + * + * Terminate the driver's table with an all-zeroes entry. + * Use the flag values to control which fields are compared. + */ + +/** + * struct usb_device_id - identifies USB devices for probing and hotplugging + * @match_flags: Bit mask controlling which of the other fields are used to + * match against new devices. Any field except for driver_info may be + * used, although some only make sense in conjunction with other fields. + * This is usually set by a USB_DEVICE_*() macro, which sets all + * other fields in this structure except for driver_info. + * @idVendor: USB vendor ID for a device; numbers are assigned + * by the USB forum to its members. + * @idProduct: Vendor-assigned product ID. + * @bcdDevice_lo: Low end of range of vendor-assigned product version numbers. + * This is also used to identify individual product versions, for + * a range consisting of a single device. + * @bcdDevice_hi: High end of version number range. The range of product + * versions is inclusive. + * @bDeviceClass: Class of device; numbers are assigned + * by the USB forum. Products may choose to implement classes, + * or be vendor-specific. Device classes specify behavior of all + * the interfaces on a device. + * @bDeviceSubClass: Subclass of device; associated with bDeviceClass. + * @bDeviceProtocol: Protocol of device; associated with bDeviceClass. + * @bInterfaceClass: Class of interface; numbers are assigned + * by the USB forum. Products may choose to implement classes, + * or be vendor-specific. Interface classes specify behavior only + * of a given interface; other interfaces may support other classes. + * @bInterfaceSubClass: Subclass of interface; associated with bInterfaceClass. + * @bInterfaceProtocol: Protocol of interface; associated with bInterfaceClass. + * @bInterfaceNumber: Number of interface; composite devices may use + * fixed interface numbers to differentiate between vendor-specific + * interfaces. + * @driver_info: Holds information used by the driver. Usually it holds + * a pointer to a descriptor understood by the driver, or perhaps + * device flags. + * + * In most cases, drivers will create a table of device IDs by using + * USB_DEVICE(), or similar macros designed for that purpose. + * They will then export it to userspace using MODULE_DEVICE_TABLE(), + * and provide it to the USB core through their usb_driver structure. + * + * See the usb_match_id() function for information about how matches are + * performed. Briefly, you will normally use one of several macros to help + * construct these entries. Each entry you provide will either identify + * one or more specific products, or will identify a class of products + * which have agreed to behave the same. You should put the more specific + * matches towards the beginning of your table, so that driver_info can + * record quirks of specific products. + */ +struct usb_device_id { + /* which fields to match against? */ + __u16 match_flags; + + /* Used for product specific matches; range is inclusive */ + __u16 idVendor; + __u16 idProduct; + __u16 bcdDevice_lo; + __u16 bcdDevice_hi; + + /* Used for device class matches */ + __u8 bDeviceClass; + __u8 bDeviceSubClass; + __u8 bDeviceProtocol; + + /* Used for interface class matches */ + __u8 bInterfaceClass; + __u8 bInterfaceSubClass; + __u8 bInterfaceProtocol; + + /* Used for vendor-specific interface matches */ + __u8 bInterfaceNumber; + + /* not matched against */ + kernel_ulong_t driver_info + __attribute__((aligned(sizeof(kernel_ulong_t)))); +}; + +/* Some useful macros to use to create struct usb_device_id */ +#define USB_DEVICE_ID_MATCH_VENDOR 0x0001 +#define USB_DEVICE_ID_MATCH_PRODUCT 0x0002 +#define USB_DEVICE_ID_MATCH_DEV_LO 0x0004 +#define USB_DEVICE_ID_MATCH_DEV_HI 0x0008 +#define USB_DEVICE_ID_MATCH_DEV_CLASS 0x0010 +#define USB_DEVICE_ID_MATCH_DEV_SUBCLASS 0x0020 +#define USB_DEVICE_ID_MATCH_DEV_PROTOCOL 0x0040 +#define USB_DEVICE_ID_MATCH_INT_CLASS 0x0080 +#define USB_DEVICE_ID_MATCH_INT_SUBCLASS 0x0100 +#define USB_DEVICE_ID_MATCH_INT_PROTOCOL 0x0200 +#define USB_DEVICE_ID_MATCH_INT_NUMBER 0x0400 + +#endif /* ifndef LINUX_DEVICE_ID_USB_H */ diff --git a/include/linux/device-id/vchiq.h b/include/linux/device-id/vchiq.h new file mode 100644 index 000000000000..16b1b874c02d --- /dev/null +++ b/include/linux/device-id/vchiq.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VCHIQ_H +#define LINUX_DEVICE_ID_VCHIQ_H + +struct vchiq_device_id { + char name[32]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VCHIQ_H */ diff --git a/include/linux/device-id/vio.h b/include/linux/device-id/vio.h new file mode 100644 index 000000000000..8ed7ea9cec07 --- /dev/null +++ b/include/linux/device-id/vio.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VIO_H +#define LINUX_DEVICE_ID_VIO_H + +/* VIO */ +struct vio_device_id { + char type[32]; + char compat[32]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VIO_H */ diff --git a/include/linux/device-id/virtio.h b/include/linux/device-id/virtio.h new file mode 100644 index 000000000000..9648a42d4f26 --- /dev/null +++ b/include/linux/device-id/virtio.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VIRTIO_H +#define LINUX_DEVICE_ID_VIRTIO_H + +#ifdef __KERNEL__ +#include +#endif + +#define VIRTIO_DEV_ANY_ID 0xffffffff + +struct virtio_device_id { + __u32 device; + __u32 vendor; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VIRTIO_H */ diff --git a/include/linux/device-id/wmi.h b/include/linux/device-id/wmi.h new file mode 100644 index 000000000000..9f55c83c5203 --- /dev/null +++ b/include/linux/device-id/wmi.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_WMI_H +#define LINUX_DEVICE_ID_WMI_H + +/* WMI */ + +#define WMI_MODULE_PREFIX "wmi:" + +/** + * struct wmi_device_id - WMI device identifier + * @guid_string: 36 char string of the form fa50ff2b-f2e8-45de-83fa-65417f2f49ba + * @context: pointer to driver specific data + */ +struct wmi_device_id { + const char guid_string[UUID_STRING_LEN+1]; + const void *context; +}; + +#endif /* ifndef LINUX_DEVICE_ID_WMI_H */ diff --git a/include/linux/device-id/x86_cpu.h b/include/linux/device-id/x86_cpu.h new file mode 100644 index 000000000000..f44e5253ccca --- /dev/null +++ b/include/linux/device-id/x86_cpu.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_X86_CPU_H +#define LINUX_DEVICE_ID_X86_CPU_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +/* Wild cards for x86_cpu_id::vendor, family, model and feature */ +#define X86_VENDOR_ANY 0xffff +#define X86_FAMILY_ANY 0 +#define X86_MODEL_ANY 0 +#define X86_STEPPING_ANY 0 +#define X86_STEP_MIN 0 +#define X86_STEP_MAX 0xf +#define X86_PLATFORM_ANY 0x0 +#define X86_FEATURE_ANY 0 /* Same as FPU, you can't test for that */ +#define X86_CPU_TYPE_ANY 0 + +/* + * Match x86 CPUs for CPU specific drivers. + * See documentation of "x86_match_cpu" for details. + */ + +/* + * MODULE_DEVICE_TABLE expects this struct to be called x86cpu_device_id. + * Although gcc seems to ignore this error, clang fails without this define. + */ +#define x86cpu_device_id x86_cpu_id +struct x86_cpu_id { + __u16 vendor; + __u16 family; + __u16 model; + __u16 steppings; + __u16 feature; /* bit index */ + /* Solely for kernel-internal use: DO NOT EXPORT to userspace! */ + __u16 flags; + __u8 platform_mask; + __u8 type; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_X86_CPU_H */ diff --git a/include/linux/device-id/zorro.h b/include/linux/device-id/zorro.h new file mode 100644 index 000000000000..5fdac8168983 --- /dev/null +++ b/include/linux/device-id/zorro.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ZORRO_H +#define LINUX_DEVICE_ID_ZORRO_H + +#ifdef __KERNEL__ +#include +typedef unsigned long kernel_ulong_t; +#endif + +#define ZORRO_WILDCARD (0xffffffff) /* not official */ + +#define ZORRO_DEVICE_MODALIAS_FMT "zorro:i%08X" + +struct zorro_device_id { + __u32 id; /* Device ID or ZORRO_WILDCARD */ + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_ZORRO_H */ diff --git a/include/linux/device.h b/include/linux/device.h index 7b2baffdd2f5..aee79fd6b32b 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -99,7 +99,9 @@ struct device_type { * struct device_attribute - Interface for exporting device attributes. * @attr: sysfs attribute definition. * @show: Show handler. + * @show_const: Show handler (read-only). * @store: Store handler. + * @store_const: Store handler (read-only). */ struct device_attribute { struct attribute attr; diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 38048e74d10a..768a1334c0a1 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -19,6 +19,8 @@ #include #include #include +#include +#include /** * enum probe_type - device driver probe type to try diff --git a/include/linux/dfl.h b/include/linux/dfl.h index 1f02db0c1897..f28e70652080 100644 --- a/include/linux/dfl.h +++ b/include/linux/dfl.h @@ -9,7 +9,7 @@ #define __LINUX_DFL_H #include -#include +#include /** * enum dfl_id_type - define the DFL FIU types diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index 62df222fe0f1..7bfacdf79de2 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -8,6 +8,8 @@ #ifndef __LINUX_DMA_FENCE_UNWRAP_H #define __LINUX_DMA_FENCE_UNWRAP_H +#include + struct dma_fence; /** @@ -48,11 +50,11 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); for (fence = dma_fence_unwrap_first(head, cursor); fence; \ fence = dma_fence_unwrap_next(cursor)) -struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, +struct dma_fence *__dma_fence_unwrap_merge(size_t num_fences, struct dma_fence **fences, struct dma_fence_unwrap *cursors); -int dma_fence_dedup_array(struct dma_fence **array, int num_fences); +size_t dma_fence_dedup_array(struct dma_fence **array, size_t num_fences); /** * dma_fence_unwrap_merge - unwrap and merge fences diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index b52ab692b22e..158cd609f103 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -448,7 +448,7 @@ int dma_fence_add_callback(struct dma_fence *fence, dma_fence_func_t func); bool dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb); -void dma_fence_enable_sw_signaling(struct dma_fence *fence); +void dma_fence_enable_signaling(struct dma_fence *fence); /** * DOC: Safe external access to driver provided object members @@ -534,7 +534,7 @@ dma_fence_is_signaled_locked(struct dma_fence *fence) * Returns true if the fence was already signaled, false if not. Since this * function doesn't enable signaling, it is not guaranteed to ever return * true if dma_fence_add_callback(), dma_fence_wait() or - * dma_fence_enable_sw_signaling() haven't been called before. + * dma_fence_enable_signaling() haven't been called before. * * It's recommended for seqno fences to call dma_fence_signal when the * operation is complete, it makes it possible to prevent issues from diff --git a/include/linux/dma/qcom_bam_dma.h b/include/linux/dma/qcom_bam_dma.h index 68fc0e643b1b..d9d07a9ab313 100644 --- a/include/linux/dma/qcom_bam_dma.h +++ b/include/linux/dma/qcom_bam_dma.h @@ -13,9 +13,12 @@ * supported by BAM DMA Engine. * * @cmd_and_addr - upper 8 bits command and lower 24 bits register address. - * @data - for write command: content to be written into peripheral register. - * for read command: dest addr to write peripheral register value. - * @mask - register mask. + * @data - For write command: content to be written into peripheral register. + * For read command: lower 32 bits of destination address. + * @mask - For write command: register write mask. + * For read command on BAM v1.6.0+: upper 4 bits of destination address. + * For read command on BAM < v1.6.0: ignored by hardware. + * Setting to 0 ensures 32-bit addressing compatibility. * @reserved - for future usage. * */ @@ -42,6 +45,10 @@ enum bam_command_type { * @addr: target address * @cmd: BAM command * @data: actual data for write and dest addr for read in le32 + * + * For BAM v1.6.0+, the mask field behavior depends on command type: + * - Write commands: mask = write mask (typically 0xffffffff) + * - Read commands: mask = upper 4 bits of destination address (0 for 32-bit) */ static inline void bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, @@ -50,7 +57,11 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, bam_ce->cmd_and_addr = cpu_to_le32((addr & 0xffffff) | ((cmd & 0xff) << 24)); bam_ce->data = data; - bam_ce->mask = cpu_to_le32(0xffffffff); + if (cmd == BAM_READ_COMMAND) + bam_ce->mask = cpu_to_le32(0x0); /* 32-bit addressing */ + else + bam_ce->mask = cpu_to_le32(0xffffffff); /* Write mask */ + bam_ce->reserved = 0; } /* @@ -60,7 +71,7 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, * @bam_ce: BAM command element * @addr: target address * @cmd: BAM command - * @data: actual data for write and dest addr for read + * @data: actual data for write and destination address for read */ static inline void bam_prep_ce(struct bam_cmd_element *bam_ce, u32 addr, diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index b3d251c9734e..6fe46c0c9452 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -322,6 +322,8 @@ struct dma_router { * @slave: ptr to the device using this channel * @cookie: last cookie value returned to client * @completed_cookie: last completed cookie for this channel + * @lock: protect between config and prepare transfer when driver have not + * implemented callback device_prep_config_sg(). * @chan_id: channel ID for sysfs * @dev: class device for sysfs * @name: backlink name for sysfs @@ -341,6 +343,12 @@ struct dma_chan { dma_cookie_t cookie; dma_cookie_t completed_cookie; + /* + * protect between config and prepare transfer because *_prep() may be + * called from complete callback, which is in GFP_NOSLEEP context. + */ + spinlock_t lock; + /* sysfs */ int chan_id; struct dma_chan_dev *dev; @@ -835,6 +843,7 @@ struct dma_filter { * where the address and size of each segment is located in one entry of * the dma_vec array. * @device_prep_slave_sg: prepares a slave dma operation + * @device_prep_config_sg: prepares a slave DMA operation with dma_slave_config * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio. * The function takes a buffer of size buf_len. The callback function will * be called after period_len bytes have been transferred. @@ -934,6 +943,10 @@ struct dma_device { struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context); + struct dma_async_tx_descriptor *(*device_prep_config_sg)( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, struct dma_slave_config *config); struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, @@ -974,22 +987,44 @@ static inline bool is_slave_direction(enum dma_transfer_direction direction) (direction == DMA_DEV_TO_DEV); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single( - struct dma_chan *chan, dma_addr_t buf, size_t len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) { struct scatterlist sg; + + if (!chan || !chan->device) + return NULL; + sg_init_table(&sg, 1); sg_dma_address(&sg) = buf; sg_dma_len(&sg) = len; - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, &sg, 1, dir, + flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, &sg, 1, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_single(chan, buf, len, dir, flags, NULL); +} + /** * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor * @chan: The channel to be used for this descriptor @@ -1010,17 +1045,115 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( dir, flags); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, struct dma_slave_config *config) { - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (!chan || !chan->device) + return NULL; + + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, sgl, sg_len, + dir, flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, sgl, sg_len, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL); +} + +/** + * dmaengine_prep_config_sg_safe - prepare a scatter-gather DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @sgl: scatterlist for the transfer + * @sg_len: number of entries in @sgl + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Prepare a DMA scatter-gather transfer together with a corresponding slave + * configuration update in a re-entrant and race-safe manner. + * + * DMA engine drivers may implement the optional + * device_prep_config_sg() callback to perform both the slave configuration + * and descriptor preparation atomically. In this case, the operation is + * fully handled by the DMA engine driver. + * + * If the DMA engine driver does not implement device_prep_config_sg(), falls + * back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg(). + * The fallback path is protected by a per-channel spinlock to ensure that + * concurrent callers cannot interleave configuration and descriptor preparation + * on the same DMA channel. + * + * Return: Pointer to a prepared DMA async transaction descriptor on success, + * or %NULL if the transfer could not be prepared. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg_safe(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct dma_async_tx_descriptor *tx; + unsigned long spinlock_flags; + + if (!chan || !chan->device) + return NULL; + + if (!chan->device->device_prep_config_sg) + spin_lock_irqsave(&chan->lock, spinlock_flags); + + tx = dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, config); + + if (!chan->device->device_prep_config_sg) + spin_unlock_irqrestore(&chan->lock, spinlock_flags); + + return tx; +} + +/** + * dmaengine_prep_config_single_safe - prepare a single-buffer DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @buf: DMA buffer address + * @len: length of the transfer in bytes + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Detail see dmaengine_prep_config_sg_safe(). + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single_safe(struct dma_chan *chan, dma_addr_t buf, + size_t len, enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_dma_address(&sg) = buf; + sg_dma_len(&sg) = len; + + return dmaengine_prep_config_sg_safe(chan, &sg, 1, dir, flags, config); +} + #ifdef CONFIG_RAPIDIO_DMA_ENGINE struct rio_dma_ext; static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg( diff --git a/include/linux/dmi.h b/include/linux/dmi.h index c8700e6a694d..fbb62f7c3111 100644 --- a/include/linux/dmi.h +++ b/include/linux/dmi.h @@ -4,7 +4,7 @@ #include #include -#include +#include /* enum dmi_field is in mod_devicetable.h */ diff --git a/include/linux/dtlk.h b/include/linux/dtlk.h deleted file mode 100644 index 27b95e70bde3..000000000000 --- a/include/linux/dtlk.h +++ /dev/null @@ -1,86 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define DTLK_MINOR 0 -#define DTLK_IO_EXTENT 0x02 - - /* ioctl's use magic number of 0xa3 */ -#define DTLK_INTERROGATE 0xa390 /* get settings from the DoubleTalk */ -#define DTLK_STATUS 0xa391 /* get status from the DoubleTalk */ - - -#define DTLK_CLEAR 0x18 /* stops speech */ - -#define DTLK_MAX_RETRIES (loops_per_jiffy/(10000/HZ)) - - /* TTS Port Status Flags */ -#define TTS_READABLE 0x80 /* mask for bit which is nonzero if a - byte can be read from the TTS port */ -#define TTS_SPEAKING 0x40 /* mask for SYNC bit, which is nonzero - while DoubleTalk is producing - output with TTS, PCM or CVSD - synthesizers or tone generators - (that is, all but LPC) */ -#define TTS_SPEAKING2 0x20 /* mask for SYNC2 bit, - which falls to zero up to 0.4 sec - before speech stops */ -#define TTS_WRITABLE 0x10 /* mask for RDY bit, which when set to - 1, indicates the TTS port is ready - to accept a byte of data. The RDY - bit goes zero 2-3 usec after - writing, and goes 1 again 180-190 - usec later. */ -#define TTS_ALMOST_FULL 0x08 /* mask for AF bit: When set to 1, - indicates that less than 300 free - bytes are available in the TTS - input buffer. AF is always 0 in the - PCM, TGN and CVSD modes. */ -#define TTS_ALMOST_EMPTY 0x04 /* mask for AE bit: When set to 1, - indicates that less than 300 bytes - of data remain in DoubleTalk's - input (TTS or PCM) buffer. AE is - always 1 in the TGN and CVSD - modes. */ - - /* LPC speak commands */ -#define LPC_5220_NORMAL 0x60 /* 5220 format decoding table, normal rate */ -#define LPC_5220_FAST 0x64 /* 5220 format decoding table, fast rate */ -#define LPC_D6_NORMAL 0x20 /* D6 format decoding table, normal rate */ -#define LPC_D6_FAST 0x24 /* D6 format decoding table, fast rate */ - - /* LPC Port Status Flags (valid only after one of the LPC - speak commands) */ -#define LPC_SPEAKING 0x80 /* mask for TS bit: When set to 1, - indicates the LPC synthesizer is - producing speech.*/ -#define LPC_BUFFER_LOW 0x40 /* mask for BL bit: When set to 1, - indicates that the hardware LPC - data buffer has less than 30 bytes - remaining. (Total internal buffer - size = 4096 bytes.) */ -#define LPC_BUFFER_EMPTY 0x20 /* mask for BE bit: When set to 1, - indicates that the LPC data buffer - ran out of data (error condition if - TS is also 1). */ - - /* data returned by Interrogate command */ -struct dtlk_settings -{ - unsigned short serial_number; /* 0-7Fh:0-7Fh */ - unsigned char rom_version[24]; /* null terminated string */ - unsigned char mode; /* 0=Character; 1=Phoneme; 2=Text */ - unsigned char punc_level; /* nB; 0-7 */ - unsigned char formant_freq; /* nF; 0-9 */ - unsigned char pitch; /* nP; 0-99 */ - unsigned char speed; /* nS; 0-9 */ - unsigned char volume; /* nV; 0-9 */ - unsigned char tone; /* nX; 0-2 */ - unsigned char expression; /* nE; 0-9 */ - unsigned char ext_dict_loaded; /* 1=exception dictionary loaded */ - unsigned char ext_dict_status; /* 1=exception dictionary enabled */ - unsigned char free_ram; /* # pages (truncated) remaining for - text buffer */ - unsigned char articulation; /* nA; 0-9 */ - unsigned char reverb; /* nR; 0-9 */ - unsigned char eob; /* 7Fh value indicating end of - parameter block */ - unsigned char has_indexing; /* nonzero if indexing is implemented */ -}; diff --git a/include/linux/edac.h b/include/linux/edac.h index e6b4e51130e5..f7a8218f9cc0 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -598,9 +598,6 @@ struct mem_ctl_info { int op_state; struct dentry *debugfs; - u8 fake_inject_layer[EDAC_MAX_LAYERS]; - bool fake_inject_ue; - u16 fake_inject_count; /* * Memory Controller hierarchy diff --git a/include/linux/efs_vh.h b/include/linux/efs_vh.h deleted file mode 100644 index 206c5270f7b8..000000000000 --- a/include/linux/efs_vh.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * efs_vh.h - * - * Copyright (c) 1999 Al Smith - * - * Portions derived from IRIX header files (c) 1985 MIPS Computer Systems, Inc. - */ - -#ifndef __EFS_VH_H__ -#define __EFS_VH_H__ - -#define VHMAGIC 0xbe5a941 /* volume header magic number */ -#define NPARTAB 16 /* 16 unix partitions */ -#define NVDIR 15 /* max of 15 directory entries */ -#define BFNAMESIZE 16 /* max 16 chars in boot file name */ -#define VDNAMESIZE 8 - -struct volume_directory { - char vd_name[VDNAMESIZE]; /* name */ - __be32 vd_lbn; /* logical block number */ - __be32 vd_nbytes; /* file length in bytes */ -}; - -struct partition_table { /* one per logical partition */ - __be32 pt_nblks; /* # of logical blks in partition */ - __be32 pt_firstlbn; /* first lbn of partition */ - __be32 pt_type; /* use of partition */ -}; - -struct volume_header { - __be32 vh_magic; /* identifies volume header */ - __be16 vh_rootpt; /* root partition number */ - __be16 vh_swappt; /* swap partition number */ - char vh_bootfile[BFNAMESIZE]; /* name of file to boot */ - char pad[48]; /* device param space */ - struct volume_directory vh_vd[NVDIR]; /* other vol hdr contents */ - struct partition_table vh_pt[NPARTAB]; /* device partition layout */ - __be32 vh_csum; /* volume header checksum */ - __be32 vh_fill; /* fill out to 512 bytes */ -}; - -/* partition type sysv is used for EFS format CD-ROM partitions */ -#define SGI_SYSV 0x05 -#define SGI_EFS 0x07 -#define IS_EFS(x) (((x) == SGI_EFS) || ((x) == SGI_SYSV)) - -struct pt_types { - int pt_type; - char *pt_name; -}; - -#endif /* __EFS_VH_H__ */ - diff --git a/include/linux/eisa.h b/include/linux/eisa.h index cf55630b595b..52a97dc4c85a 100644 --- a/include/linux/eisa.h +++ b/include/linux/eisa.h @@ -4,7 +4,7 @@ #include #include -#include +#include #define EISA_MAX_SLOTS 8 diff --git a/include/linux/err.h b/include/linux/err.h index 8c37be0620ab..d3e38d5b3a98 100644 --- a/include/linux/err.h +++ b/include/linux/err.h @@ -36,7 +36,7 @@ * * Return: A pointer with @error encoded within its value. */ -static inline void * __must_check ERR_PTR(long error) +static __always_inline void * __must_check ERR_PTR(long error) { return (void *) error; } @@ -60,7 +60,7 @@ static inline void * __must_check ERR_PTR(long error) * @ptr: An error pointer. * Return: The error code within @ptr. */ -static inline long __must_check PTR_ERR(__force const void *ptr) +static __always_inline long __must_check PTR_ERR(__force const void *ptr) { return (long) ptr; } @@ -73,7 +73,7 @@ static inline long __must_check PTR_ERR(__force const void *ptr) * @ptr: The pointer to check. * Return: true if @ptr is an error pointer, false otherwise. */ -static inline bool __must_check IS_ERR(__force const void *ptr) +static __always_inline bool __must_check IS_ERR(__force const void *ptr) { return IS_ERR_VALUE((unsigned long)ptr); } @@ -87,7 +87,7 @@ static inline bool __must_check IS_ERR(__force const void *ptr) * * Like IS_ERR(), but also returns true for a null pointer. */ -static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) +static __always_inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) { return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr); } @@ -99,7 +99,7 @@ static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) * Explicitly cast an error-valued pointer to another pointer type in such a * way as to make it clear that's what's going on. */ -static inline void * __must_check ERR_CAST(__force const void *ptr) +static __always_inline void * __must_check ERR_CAST(__force const void *ptr) { /* cast away the const */ return (void *) ptr; @@ -122,7 +122,7 @@ static inline void * __must_check ERR_CAST(__force const void *ptr) * * Return: The error code within @ptr if it is an error pointer; 0 otherwise. */ -static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) +static __always_inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) { if (IS_ERR(ptr)) return PTR_ERR(ptr); diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 1b834e2a522e..5d491a98265e 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -942,6 +942,7 @@ struct kernel_ethtool_ts_info { #define ETHTOOL_OP_NEEDS_RTNL_GPAUSEPARAM BIT(5) #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) #define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) +#define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) /** * struct ethtool_ops - optional netdev operations @@ -978,6 +979,7 @@ struct kernel_ethtool_ts_info { * - phylink helpers (note that phydev is currently unsupported!) * - netdev_update_features() * - netif_set_real_num_tx_queues() + * - ethtool_op_get_link() (syncs link watch under rtnl_lock) * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 829a59399dac..bb2b6cd5d507 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -107,6 +107,7 @@ enum f2fs_error { ERROR_CORRUPTED_XATTR, ERROR_INVALID_NODE_REFERENCE, ERROR_INCONSISTENT_NAT, + ERROR_INCONSISTENT_ORPHAN, ERROR_MAX, }; diff --git a/include/linux/filter.h b/include/linux/filter.h index 67d337ede91b..14acb2455746 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1314,6 +1315,15 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +/* + * Flush the indirect branch predictors before reusing JIT memory, so that + * indirect jumps into a newly written program don't reuse predictions left + * behind by an old program that occupied the same space. + */ +void bpf_arch_pred_flush(void); +DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); +DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); struct bpf_binary_header * @@ -1328,7 +1338,7 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) @@ -1342,7 +1352,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, unsigned int alignment, struct bpf_binary_header **rw_hdr, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns); + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic); int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header); void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, diff --git a/include/linux/firewire.h b/include/linux/firewire.h index 986d712e4d94..fd35a6570cd8 100644 --- a/include/linux/firewire.h +++ b/include/linux/firewire.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -271,8 +272,6 @@ static inline void fw_unit_put(struct fw_unit *unit) #define fw_parent_device(unit) fw_device(unit->device.parent) -struct ieee1394_device_id; - struct fw_driver { struct device_driver driver; int (*probe)(struct fw_unit *unit, const struct ieee1394_device_id *id); diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 935dba3633b5..9116512169dc 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -514,6 +514,25 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_SVC_VERSION \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_SVC_FUNCID_VERSION) +/** + * Request INTEL_SIP_SMC_ATF_BUILD_VER + * + * Sync call used to query the ATF Build Version + * + * Call register usage: + * a0 INTEL_SIP_SMC_ATF_BUILD_VER + * a1-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK + * a1 Major + * a2 Minor + * a3 Patch + */ +#define INTEL_SIP_SMC_ATF_BUILD_VERSION 155 +#define INTEL_SIP_SMC_ATF_BUILD_VER \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_ATF_BUILD_VERSION) + /** * SMC call protocol for FPGA Crypto Service (FCS) * FUNCID starts from 90 @@ -605,25 +624,21 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) /** * Request INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * Sync call to dump all the fuses and key hashes + * Async call to dump all the fuses and key hashes * * Call register usage: * a0 INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * a1 the physical address for firmware to write structure of fuse and - * key hashes - * a2-a7 not used + * a1-a7 not used * * Return status: * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_FCS_ERROR or * INTEL_SIP_SMC_FCS_REJECTED - * a1 mailbox error - * a2 physical address for the structure of fuse and key hashes - * a3 the size of structure + * a1-a3 not used * */ #define INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA 94 #define INTEL_SIP_SMC_FCS_GET_PROVISION_DATA \ - INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) /** * Request INTEL_SIP_SMC_HWMON_READTEMP diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 91013161e9db..3edd93502bf8 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -154,6 +154,10 @@ struct stratix10_svc_chan; * * @COMMAND_HWMON_READVOLT: query the voltage from the hardware monitor, * return status is SVC_STATUS_OK or SVC_STATUS_ERROR + * + * @COMMAND_SMC_ATF_BUILD_VER: Non-mailbox SMC ATF Build Version, + * return status is SVC_STATUS_OK + * */ enum stratix10_svc_command_code { /* for FPGA */ @@ -187,7 +191,8 @@ enum stratix10_svc_command_code { COMMAND_SMC_SVC_VERSION = 200, /* for HWMON */ COMMAND_HWMON_READTEMP, - COMMAND_HWMON_READVOLT + COMMAND_HWMON_READVOLT, + COMMAND_SMC_ATF_BUILD_VER }; /** diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index d70dcd462b44..347df66ee176 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -542,6 +542,18 @@ enum pm_gem_config_type { GEM_CONFIG_FIXED = 2, }; +/** + * enum pm_node_status - Device node status provided by xilpm fw + * @PM_NODE_UNUSED: Device is not used + * @PM_NODE_RUNNING: Device is power-on and out of reset + * @PM_NODE_HALT: Device is power-on but in the reset state + */ +enum pm_node_status { + PM_NODE_UNUSED = 0, + PM_NODE_RUNNING = 1, + PM_NODE_HALT = 12, +}; + /** * struct zynqmp_pm_query_data - PM query data * @qid: query ID @@ -630,6 +642,10 @@ int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode); int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode); int zynqmp_pm_get_node_status(const u32 node, u32 *const status, u32 *const requirements, u32 *const usage); +int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, u32 *const usage); +int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr); +int zynqmp_pm_stop_rpu(const u32 node); int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value); int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config, u32 value); @@ -939,6 +955,23 @@ static inline int zynqmp_pm_get_node_status(const u32 node, u32 *const status, return -ENODEV; } +static inline int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, + u32 *const usage) +{ + return -ENODEV; +} + +static inline int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr) +{ + return -ENODEV; +} + +static inline int zynqmp_pm_stop_rpu(const u32 node) +{ + return -ENODEV; +} + static inline int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value) diff --git a/include/linux/font.h b/include/linux/font.h index 6845f02d739a..ea23b727388b 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -101,6 +101,9 @@ font_data_t *font_data_import(const struct console_font *font, unsigned int vpit void font_data_get(font_data_t *fd); bool font_data_put(font_data_t *fd); unsigned int font_data_size(font_data_t *fd); +const unsigned char *font_data_glyph_buf(font_data_t *fd, + unsigned int width, unsigned int vpitch, + unsigned int c); bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs); int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch); diff --git a/include/linux/fs.h b/include/linux/fs.h index 6da44573ce45..6e4b8e19af05 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -460,7 +460,6 @@ struct mapping_metadata_bhs { * memory mappings. * @gfp_mask: Memory allocation flags to use for allocating pages. * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings. - * @nr_thps: Number of THPs in the pagecache (non-shmem only). * @i_mmap: Tree of private and shared mappings. * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable. * @nrpages: Number of page entries, protected by the i_pages lock. @@ -476,10 +475,6 @@ struct address_space { struct rw_semaphore invalidate_lock; gfp_t gfp_mask; atomic_t i_mmap_writable; -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - /* number of thp, only for non-shmem files */ - atomic_t nr_thps; -#endif struct rb_root_cached i_mmap; unsigned long nrpages; pgoff_t writeback_index; @@ -1921,8 +1916,6 @@ struct dir_context { struct io_uring_cmd; struct offset_ctx; -typedef unsigned int __bitwise fop_flags_t; - struct file_operations { struct module *owner; fop_flags_t fop_flags; @@ -2007,7 +2000,7 @@ struct inode_operations { int (*readlink) (struct dentry *, char __user *,int); int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, - umode_t, bool); + umode_t); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *, @@ -2449,6 +2442,11 @@ static inline struct mnt_idmap *file_mnt_idmap(const struct file *file) return mnt_idmap(file->f_path.mnt); } +static inline bool file_owner_or_capable(const struct file *file) +{ + return inode_owner_or_capable(file_mnt_idmap(file), file_inode(file)); +} + /** * is_idmapped_mnt - check whether a mount is mapped * @mnt: the mount to check diff --git a/include/linux/fs/super.h b/include/linux/fs/super.h index 405612678115..733d439f01ed 100644 --- a/include/linux/fs/super.h +++ b/include/linux/fs/super.h @@ -237,4 +237,12 @@ int thaw_super(struct super_block *super, enum freeze_holder who, int sb_init_dio_done_wq(struct super_block *sb); +struct file; +struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, + struct super_block *sb); +struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode, + void *holder, struct super_block *sb); +void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb); +void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb); + #endif /* _LINUX_FS_SUPER_H */ diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index ef7941e9dc79..c8172558750f 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -30,6 +30,7 @@ struct mount; struct mtd_info; struct quotactl_ops; struct shrinker; +struct super_dev; struct unicode_map; struct user_namespace; struct workqueue_struct; @@ -132,6 +133,7 @@ struct super_operations { struct super_block { struct list_head s_list; /* Keep this first */ dev_t s_dev; /* search index; _not_ kdev_t */ + struct super_dev *s_super_dev; /* sget_fc()'s device table claim */ unsigned char s_blocksize_bits; unsigned long s_blocksize; loff_t s_maxbytes; /* Max file size */ @@ -145,7 +147,7 @@ struct super_block { unsigned long s_magic; struct dentry *s_root; struct rw_semaphore s_umount; - int s_count; + refcount_t s_passive; atomic_t s_active; #ifdef CONFIG_SECURITY void *s_security; diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h index 0070764b790a..97eef8d3863d 100644 --- a/include/linux/fs_struct.h +++ b/include/linux/fs_struct.h @@ -6,6 +6,7 @@ #include #include #include +#include struct fs_struct { int users; @@ -16,6 +17,7 @@ struct fs_struct { } __randomize_layout; extern struct kmem_cache *fs_cachep; +extern struct fs_struct *userspace_init_fs; extern void exit_fs(struct task_struct *); extern void set_fs_root(struct fs_struct *, const struct path *); @@ -40,6 +42,8 @@ static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd) read_sequnlock_excl(&fs->seq); } +struct fs_struct *switch_fs_struct(struct fs_struct *new_fs); + extern bool current_chrooted(void); static inline int current_umask(void) @@ -47,4 +51,34 @@ static inline int current_umask(void) return current->fs->umask; } +/* + * Temporarily use userspace_init_fs for path resolution in kthreads. + * Callers should use scoped_with_init_fs() which automatically + * restores the original fs_struct at scope exit. + */ +static inline struct fs_struct *__override_init_fs(void) +{ + struct fs_struct *old_fs; + + old_fs = current->fs; + WRITE_ONCE(current->fs, userspace_init_fs); + return old_fs; +} + +static inline void __revert_init_fs(struct fs_struct *old_fs) +{ + VFS_WARN_ON_ONCE(current->fs != userspace_init_fs); + WRITE_ONCE(current->fs, old_fs); +} + +DEFINE_CLASS(__override_init_fs, + struct fs_struct *, + __revert_init_fs(_T), + __override_init_fs(), void) + +#define scoped_with_init_fs() \ + scoped_class(__override_init_fs, __UNIQUE_ID(label)) + +void __init init_userspace_fs(void); + #endif /* _LINUX_FS_STRUCT_H */ diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index 9f671e87c80c..c25f0f7e6dd4 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -11,7 +11,7 @@ #define _FSL_MC_H_ #include -#include +#include #include #include diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 54ca0c88bab6..463b551d12d9 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -136,7 +136,8 @@ enum { * %__GFP_THISNODE forces the allocation to be satisfied from the requested * node with no fallbacks or placement policy enforcements. * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. + * %__GFP_ACCOUNT causes the allocation to be accounted to the active + * cgroup context. * * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. * mark_obj_codetag_empty() should be called upon freeing for objects allocated @@ -320,7 +321,7 @@ enum { * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. * * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. + * accounted to the active cgroup context. * * %GFP_NOWAIT is for kernel allocations that should not stall for direct * reclaim, start physical IO or use any filesystem callback. It is very diff --git a/include/linux/goldfish.h b/include/linux/goldfish.h index bcc17f95b906..98a4719c6776 100644 --- a/include/linux/goldfish.h +++ b/include/linux/goldfish.h @@ -2,8 +2,6 @@ #ifndef __LINUX_GOLDFISH_H #define __LINUX_GOLDFISH_H -#include -#include #include /* Helpers for Goldfish virtual platform */ @@ -15,26 +13,4 @@ #define gf_iowrite32 iowrite32 #endif -static inline void gf_write_ptr(const void *ptr, void __iomem *portl, - void __iomem *porth) -{ - const unsigned long addr = (unsigned long)ptr; - - gf_iowrite32(lower_32_bits(addr), portl); -#ifdef CONFIG_64BIT - gf_iowrite32(upper_32_bits(addr), porth); -#endif -} - -static inline void gf_write_dma_addr(const dma_addr_t addr, - void __iomem *portl, - void __iomem *porth) -{ - gf_iowrite32(lower_32_bits(addr), portl); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - gf_iowrite32(upper_32_bits(addr), porth); -#endif -} - - #endif /* __LINUX_GOLDFISH_H */ diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h index 71941a039648..e037714563d8 100644 --- a/include/linux/gpu_buddy.h +++ b/include/linux/gpu_buddy.h @@ -173,6 +173,21 @@ struct gpu_buddy { * that fits in the remaining space. */ struct gpu_buddy_block **roots; + /* + * Per-order free block scoreboard: free_scoreboard[order] holds the + * number of blocks of that order currently in the free state. + * Incremented in mark_free(), decremented wherever rbtree_remove() is + * called on a free block. + */ + u64 *free_scoreboard; + /* + * Per-order used block scoreboard: used_scoreboard[order] holds the + * number of blocks of that order currently in the allocated state. + * Incremented in mark_allocated(), decremented in mark_free() (guarded + * by gpu_buddy_block_is_allocated()) and in __gpu_buddy_free() when an + * allocated block is consumed directly during buddy coalescing. + */ + u64 *used_scoreboard; /* public: */ unsigned int n_roots; unsigned int max_order; diff --git a/include/linux/hid-sensor-hub.h b/include/linux/hid-sensor-hub.h index e71056553108..ab5cc8db3fbb 100644 --- a/include/linux/hid-sensor-hub.h +++ b/include/linux/hid-sensor-hub.h @@ -43,6 +43,8 @@ struct hid_sensor_hub_attribute_info { * @attr_usage_id: Usage Id of a field, e.g. X-axis for a gyro. * @raw_size: Response size for a read request. * @raw_data: Place holder for received response. + * @index: Current write index into raw_data for multi-byte reads. + * @max_raw_size: Total buffer size for multi-byte reads; 0 for single-value reads. */ struct sensor_hub_pending { bool status; @@ -51,6 +53,8 @@ struct sensor_hub_pending { u32 attr_usage_id; int raw_size; u8 *raw_data; + u32 index; + u32 max_raw_size; }; /** @@ -183,6 +187,27 @@ int sensor_hub_input_attr_get_raw_value(struct hid_sensor_hub_device *hsdev, bool is_signed ); +/** + * sensor_hub_input_attr_read_values() - Synchronous multi-byte read request + * @hsdev: Hub device instance. + * @usage_id: Attribute usage id of parent physical device as per spec + * @attr_usage_id: Attribute usage id as per spec + * @report_id: Report id to look for + * @flag: Synchronous or asynchronous read + * @buffer_size: Size of the buffer in bytes + * @buffer: Buffer to store the read data + * + * Issues a synchronous or asynchronous read request for an input attribute, + * accumulating data into the provided buffer until it is full. + * Return: 0 on success, -ETIMEDOUT if the device did not respond, or a + * negative error code. + */ +int sensor_hub_input_attr_read_values(struct hid_sensor_hub_device *hsdev, + u32 usage_id, u32 attr_usage_id, + u32 report_id, + enum sensor_hub_read_flags flag, + u32 buffer_size, u8 *buffer); + /** * sensor_hub_set_feature() - Feature set request * @hsdev: Hub device instance. diff --git a/include/linux/hid.h b/include/linux/hid.h index 47dc0bc89fa4..b240baa95ab5 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -18,7 +18,7 @@ #include #include #include -#include /* hid_device_id */ +#include #include #include #include diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index c0d223d0c556..1487bf4af1a7 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -128,6 +128,8 @@ enum mthp_stat_item { MTHP_STAT_ANON_FAULT_ALLOC, MTHP_STAT_ANON_FAULT_FALLBACK, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, + MTHP_STAT_COLLAPSE_ALLOC, + MTHP_STAT_COLLAPSE_ALLOC_FAILED, MTHP_STAT_ZSWPOUT, MTHP_STAT_SWPIN, MTHP_STAT_SWPIN_FALLBACK, @@ -142,6 +144,9 @@ enum mthp_stat_item { MTHP_STAT_SPLIT_DEFERRED, MTHP_STAT_NR_ANON, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, + MTHP_STAT_COLLAPSE_EXCEED_SWAP, + MTHP_STAT_COLLAPSE_EXCEED_NONE, + MTHP_STAT_COLLAPSE_EXCEED_SHARED, __MTHP_STAT_COUNT }; @@ -414,27 +419,6 @@ static inline int split_huge_page_to_order(struct page *page, unsigned int new_o return split_huge_page_to_list_to_order(page, NULL, new_order); } -/** - * try_folio_split_to_order() - try to split a @folio at @page to @new_order - * using non uniform split. - * @folio: folio to be split - * @page: split to @new_order at the given page - * @new_order: the target split order - * - * Try to split a @folio at @page using non uniform split to @new_order, if - * non uniform split is not supported, fall back to uniform split. After-split - * folios are put back to LRU list. Use min_order_for_split() to get the lower - * bound of @new_order. - * - * Return: 0 - split is successful, otherwise split failed. - */ -static inline int try_folio_split_to_order(struct folio *folio, - struct page *page, unsigned int new_order) -{ - if (folio_check_splittable(folio, new_order, SPLIT_TYPE_NON_UNIFORM)) - return split_huge_page_to_order(&folio->page, new_order); - return folio_split(folio, new_order, page, NULL); -} static inline int split_huge_page(struct page *page) { return split_huge_page_to_list_to_order(page, NULL, 0); @@ -583,7 +567,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, @@ -672,8 +656,8 @@ static inline int split_folio_to_list(struct folio *folio, struct list_head *lis return -EINVAL; } -static inline int try_folio_split_to_order(struct folio *folio, - struct page *page, unsigned int new_order) +static inline int folio_split(struct folio *folio, unsigned int new_order, + struct page *page, struct list_head *list) { VM_WARN_ON_ONCE_FOLIO(1, folio); return -EINVAL; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2abaf99321e9..3ebdbc246857 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -2,6 +2,7 @@ #ifndef _LINUX_HUGETLB_H #define _LINUX_HUGETLB_H +#include #include #include #include @@ -154,7 +155,8 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); void folio_putback_hugetlb(struct folio *folio); -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); @@ -171,6 +173,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -424,7 +427,7 @@ static inline void folio_putback_hugetlb(struct folio *folio) } static inline void move_hugetlb_state(struct folio *old_folio, - struct folio *new_folio, int reason) + struct folio *new_folio, enum migrate_reason reason) { } @@ -673,24 +676,30 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 -bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); + +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + +enum hugetlb_alloc_flag { + HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0, + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT, +}; + +#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT) +#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) + +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -705,8 +714,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -956,7 +965,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { bool allowed_fallback = false; @@ -1137,9 +1146,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) @@ -1238,7 +1247,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { return false; } @@ -1293,6 +1302,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index e5d64b8b59c2..16d72c8c71f6 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, { } -#endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ +#endif /* CONFIG_CGROUP_HUGETLB */ #endif diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9de2c8d6037a..a2b484679eb4 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/i2c-atr.h b/include/linux/i2c-atr.h index 2bb54dc87c8e..b52a7b9ec536 100644 --- a/include/linux/i2c-atr.h +++ b/include/linux/i2c-atr.h @@ -71,7 +71,7 @@ struct i2c_atr_adap_desc { struct device *parent; struct fwnode_handle *bus_handle; size_t num_aliases; - u16 *aliases; + u16 *aliases __counted_by_ptr(num_aliases); }; /** diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 20fd41b51d5c..14ab4d3055af 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -12,7 +12,7 @@ #include /* for acpi_handle */ #include -#include +#include #include /* for struct device */ #include /* for completion */ #include diff --git a/include/linux/i3c/device.h b/include/linux/i3c/device.h index 971d53349b6f..0f065b883ee0 100644 --- a/include/linux/i3c/device.h +++ b/include/linux/i3c/device.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include /** diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 592b646f6134..4d2a68793324 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -494,7 +494,7 @@ struct i3c_master_controller_ops { int (*disable_hotjoin)(struct i3c_master_controller *master); int (*set_speed)(struct i3c_master_controller *master, enum i3c_open_drain_speed speed); int (*set_dev_nack_retry)(struct i3c_master_controller *master, - unsigned long dev_nack_retry_cnt); + unsigned int dev_nack_retry_cnt); }; /** @@ -511,15 +511,23 @@ struct i3c_master_controller_ops { * @hotjoin: true if the master support hotjoin * @rpm_allowed: true if Runtime PM allowed * @rpm_ibi_allowed: true if IBI and Hot-Join allowed while runtime suspended + * @shutting_down: set to true when master begins shutdown or unregister * @boardinfo.i3c: list of I3C boardinfo objects * @boardinfo.i2c: list of I2C boardinfo objects * @boardinfo: board-level information attached to devices connected on the bus * @bus: I3C bus exposed by this master - * @wq: workqueue which can be used by master + * @wq: freezable workqueue which can be used by master * drivers if they need to postpone operations that need to take place * in a thread context. Typical examples are Hot Join processing which * requires taking the bus lock in maintenance, which in turn, can only * be done from a sleep-able context + * @hj_work: work item used to run DAA after a Hot-Join event is detected. + * Queued to @wq by i3c_master_queue_hotjoin() + * @reg_work: work item used to register newly discovered I3C devices with + * the driver model. Queued to @wq by i3c_master_do_daa_ext() so + * that device registration is deferred out of the DAA caller's + * context (notably the resume path), and is skipped if the + * controller is shutting down * @dev_nack_retry_count: retry count when slave device nack * * A &struct i3c_master_controller has to be registered to the I3C subsystem @@ -537,12 +545,15 @@ struct i3c_master_controller { unsigned int hotjoin: 1; unsigned int rpm_allowed: 1; unsigned int rpm_ibi_allowed: 1; + bool shutting_down; struct { struct list_head i3c; struct list_head i2c; } boardinfo; struct i3c_bus bus; struct workqueue_struct *wq; + struct work_struct hj_work; + struct work_struct reg_work; unsigned int dev_nack_retry_count; }; @@ -596,14 +607,15 @@ int i3c_master_disec_locked(struct i3c_master_controller *master, u8 addr, u8 evts); int i3c_master_enec_locked(struct i3c_master_controller *master, u8 addr, u8 evts); +int i3c_master_enec_disec_locked(struct i3c_master_controller *master, u8 addr, + bool enable, u8 evts, bool suppress_m2); int i3c_master_entdaa_locked(struct i3c_master_controller *master); int i3c_master_defslvs_locked(struct i3c_master_controller *master); int i3c_master_get_free_addr(struct i3c_master_controller *master, u8 start_addr); -int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, - u8 addr); +void i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, u8 addr); int i3c_master_do_daa(struct i3c_master_controller *master); int i3c_master_do_daa_ext(struct i3c_master_controller *master, bool rstdaa); struct i3c_dma *i3c_master_dma_map_single(struct device *dev, void *ptr, @@ -613,6 +625,8 @@ void i3c_master_dma_unmap_single(struct i3c_dma *dma_xfer); DEFINE_FREE(i3c_master_dma_unmap_single, void *, if (_T) i3c_master_dma_unmap_single(_T)) +int i3c_master_reattach_i3c_dev_locked(struct i3c_dev_desc *dev, + u8 old_dyn_addr); int i3c_master_set_info(struct i3c_master_controller *master, const struct i3c_device_info *info); @@ -623,6 +637,7 @@ int i3c_master_register(struct i3c_master_controller *master, void i3c_master_unregister(struct i3c_master_controller *master); int i3c_master_enable_hotjoin(struct i3c_master_controller *master); int i3c_master_disable_hotjoin(struct i3c_master_controller *master); +void i3c_master_queue_hotjoin(struct i3c_master_controller *master); /** * i3c_dev_get_master_data() - get master private data attached to an I3C diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h index 18f9c662cf4c..c109722b1969 100644 --- a/include/linux/ieee80211-eht.h +++ b/include/linux/ieee80211-eht.h @@ -857,7 +857,7 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) const struct ieee80211_multi_link_elem *mle = (const void *)data; u8 fixed = sizeof(*mle); u8 common = 0; - bool check_common_len = false; + u8 common_len; u16 control; if (!data || len < fixed) @@ -868,7 +868,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) { case IEEE80211_ML_CONTROL_TYPE_BASIC: common += sizeof(struct ieee80211_mle_basic_common_info); - check_common_len = true; if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) common += 1; if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) @@ -888,9 +887,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) common += sizeof(struct ieee80211_mle_preq_common_info); if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID) common += 1; - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_RECONF: + common += 1; if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) common += ETH_ALEN; if (control & IEEE80211_MLC_RECONF_PRES_EML_CAPA) @@ -902,7 +901,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) break; case IEEE80211_ML_CONTROL_TYPE_TDLS: common += sizeof(struct ieee80211_mle_tdls_common_info); - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: common = ETH_ALEN + 1; @@ -915,11 +913,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) if (len < fixed + common) return false; - if (!check_common_len) - return true; + common_len = mle->variable[0]; - /* if present, common length is the first octet there */ - return mle->variable[0] >= common; + return common_len >= common && common_len <= len - fixed; } /** diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index d40484451e9a..084ad45aa2d8 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2616,6 +2616,7 @@ static inline int ieee80211_get_tdls_action(struct sk_buff *skb) /* convert frequencies */ #define MHZ_TO_KHZ(freq) ((freq) * 1000) #define KHZ_TO_MHZ(freq) ((freq) / 1000) +#define KHZ_TO_HZ(x) ((x) * 1000) #define PR_KHZ(f) KHZ_TO_MHZ(f), f % 1000 #define KHZ_F "%d.%03d" diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h index 4d15c2a9802c..3f95ed1fdf9e 100644 --- a/include/linux/iio/backend.h +++ b/include/linux/iio/backend.h @@ -261,6 +261,7 @@ int iio_backend_extend_chan_spec(struct iio_backend *back, bool iio_backend_has_caps(struct iio_backend *back, u32 caps); void *iio_backend_get_priv(const struct iio_backend *conv); struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name); +struct iio_backend *devm_iio_backend_get_by_index(struct device *dev, unsigned int index); struct iio_backend *devm_iio_backend_fwnode_get(struct device *dev, const char *name, struct fwnode_handle *fwnode); diff --git a/include/linux/iio/buffer.h b/include/linux/iio/buffer.h index d37f82678f71..745c98ef4e04 100644 --- a/include/linux/iio/buffer.h +++ b/include/linux/iio/buffer.h @@ -34,8 +34,16 @@ static inline int iio_push_to_buffers_with_timestamp(struct iio_dev *indio_dev, void *data, int64_t timestamp) { if (ACCESS_PRIVATE(indio_dev, scan_timestamp)) { - size_t ts_offset = indio_dev->scan_bytes / sizeof(int64_t) - 1; - ((int64_t *)data)[ts_offset] = timestamp; + size_t ts_offset = ACCESS_PRIVATE(indio_dev, scan_timestamp_offset); + + /* + * The size of indio_dev->scan_bytes is always aligned to the + * largest scan element's alignment (see iio_compute_scan_bytes()). + * So there may be padding after the timestamp. ts_offset contains + * the offset in bytes that was already computed for correctly + * aligning the timestamp. + */ + *(int64_t *)(data + ts_offset) = timestamp; } return iio_push_to_buffers(indio_dev, data); @@ -71,7 +79,7 @@ static inline int iio_push_to_buffers_with_ts(struct iio_dev *indio_dev, int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev, const void *data, size_t data_sz, - int64_t timestamp); + s64 timestamp); bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev, const unsigned long *mask); diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 2c91b7659ce9..711c00f67371 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -176,9 +176,32 @@ struct iio_event_spec { unsigned long mask_shared_by_all; }; +/** + * define IIO_SCAN_FORMAT_SIGNED_INT - signed integer data format + * + * &iio_scan_type.format value for signed integers (two's complement). + */ +#define IIO_SCAN_FORMAT_SIGNED_INT 's' + +/** + * define IIO_SCAN_FORMAT_UNSIGNED_INT - unsigned integer data format + * + * &iio_scan_type.format value for unsigned integers. + */ +#define IIO_SCAN_FORMAT_UNSIGNED_INT 'u' + +/** + * define IIO_SCAN_FORMAT_FLOAT - floating-point data format + * + * &iio_scan_type.format value for IEEE 754 floating-point numbers. + */ +#define IIO_SCAN_FORMAT_FLOAT 'f' + /** * struct iio_scan_type - specification for channel data format in buffer - * @sign: 's' or 'u' to specify signed or unsigned + * @sign: Deprecated, use @format instead. + * @format: Data format, can have any of the IIO_SCAN_FORMAT_* + * values. * @realbits: Number of valid bits of data * @storagebits: Realbits + padding * @shift: Shift right by this before masking out realbits. @@ -189,7 +212,10 @@ struct iio_event_spec { * @endianness: little or big endian */ struct iio_scan_type { - char sign; + union { + char sign; + char format; + }; u8 realbits; u8 storagebits; u8 shift; @@ -327,15 +353,15 @@ static inline bool iio_channel_has_available(const struct iio_chan_spec *chan, (chan->info_mask_shared_by_all_available & BIT(type)); } -#define IIO_CHAN_SOFT_TIMESTAMP(_si) { \ +#define IIO_CHAN_SOFT_TIMESTAMP(_si) (struct iio_chan_spec) { \ .type = IIO_TIMESTAMP, \ .channel = -1, \ .scan_index = _si, \ .scan_type = { \ .sign = 's', \ - .realbits = 64, \ + .realbits = 64, \ .storagebits = 64, \ - }, \ + }, \ } s64 iio_get_time_ns(const struct iio_dev *indio_dev); @@ -584,6 +610,8 @@ struct iio_buffer_setup_ops { * and owner * @buffer: [DRIVER] any buffer present * @scan_bytes: [INTERN] num bytes captured to be fed to buffer demux + * @scan_timestamp_offset: [INTERN] cache of the offset (in bytes) for the + * timestamp in the scan buffer * @available_scan_masks: [DRIVER] optional array of allowed bitmasks. Sort the * array in order of preference, the most preferred * masks first. @@ -610,6 +638,7 @@ struct iio_dev { struct iio_buffer *buffer; int scan_bytes; + unsigned int __private scan_timestamp_offset; const unsigned long *available_scan_masks; unsigned int __private masklength; diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h index 4e3099defc1d..924ac9dc6893 100644 --- a/include/linux/iio/types.h +++ b/include/linux/iio/types.h @@ -7,6 +7,9 @@ #ifndef _IIO_TYPES_H_ #define _IIO_TYPES_H_ +#include +#include + #include enum iio_event_info { @@ -34,6 +37,23 @@ enum iio_event_info { #define IIO_VAL_FRACTIONAL_LOG2 11 #define IIO_VAL_CHAR 12 +#define IIO_VAL_DECIMAL64_BASE 32 +#define IIO_VAL_DECIMAL64_MILLI (IIO_VAL_DECIMAL64_BASE + 3) +#define IIO_VAL_DECIMAL64_MICRO (IIO_VAL_DECIMAL64_BASE + 6) +#define IIO_VAL_DECIMAL64_NANO (IIO_VAL_DECIMAL64_BASE + 9) +#define IIO_VAL_DECIMAL64_PICO (IIO_VAL_DECIMAL64_BASE + 12) + +static inline s64 iio_val_s64_compose(s32 val0, s32 val1) +{ + return (s64)(((u64)val1 << 32) | (u32)val0); +} + +static inline void iio_val_s64_decompose(s64 dec64, s32 *val0, s32 *val1) +{ + *val0 = lower_32_bits(dec64); + *val1 = upper_32_bits(dec64); +} + enum iio_available_type { IIO_AVAIL_LIST, IIO_AVAIL_RANGE, diff --git a/include/linux/init.h b/include/linux/init.h index 40331923b9f4..6326c61e2332 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ #define __initdata __section(".init.data") #define __initconst __section(".init.rodata") #define __exitdata __section(".exit.data") -#define __exit_call __used __section(".exitcall.exit") +#define __exit_call __maybe_unused __section(".exitcall.exit") /* * modpost check for section mismatches during the kernel build. diff --git a/include/linux/init_task.h b/include/linux/init_task.h index a6cb241ea00c..61536be773f5 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -24,6 +24,7 @@ extern struct files_struct init_files; extern struct fs_struct init_fs; +extern struct fs_struct *userspace_init_fs; extern struct nsproxy init_nsproxy; #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE diff --git a/include/linux/input.h b/include/linux/input.h index 06ca62328db1..76f7aa226202 100644 --- a/include/linux/input.h +++ b/include/linux/input.h @@ -19,7 +19,7 @@ #include #include #include -#include +#include struct input_dev_poller; @@ -543,6 +543,8 @@ extern const struct class input_class; * @set_autocenter: Called to auto-center device * @destroy: called by input core when parent input device is being * destroyed + * @stop: called by input core when parent input device is being + * unregistered * @private: driver-specific data, will be freed automatically * @ffbit: bitmap of force feedback capabilities truly supported by * device (not emulated like ones in input_dev->ffbit) @@ -571,6 +573,7 @@ struct ff_device { void (*set_autocenter)(struct input_dev *dev, u16 magnitude); void (*destroy)(struct ff_device *); + void (*stop)(struct ff_device *); void *private; diff --git a/include/linux/intel-ish-client-if.h b/include/linux/intel-ish-client-if.h index 2cd4f65aaa37..a07d952a2b29 100644 --- a/include/linux/intel-ish-client-if.h +++ b/include/linux/intel-ish-client-if.h @@ -9,7 +9,7 @@ #define _INTEL_ISH_CLIENT_IF_H_ #include -#include +#include struct ishtp_cl_device; struct ishtp_device; diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h index 1fe5665a9d02..843cda8f8644 100644 --- a/include/linux/intel_vsec.h +++ b/include/linux/intel_vsec.h @@ -28,6 +28,7 @@ #define INTEL_DVSEC_TABLE_BAR(x) ((x) & GENMASK(2, 0)) #define INTEL_DVSEC_TABLE_OFFSET(x) ((x) & GENMASK(31, 3)) #define TABLE_OFFSET_SHIFT 3 +#define PMT_DISC_DWORDS 4 struct device; struct pci_dev; @@ -122,7 +123,7 @@ struct intel_vsec_platform_info { struct device *parent; struct intel_vsec_header **headers; const struct vsec_feature_dependency *deps; - u32 (*acpi_disc)[4]; + u32 (*acpi_disc)[PMT_DISC_DWORDS]; enum intel_vsec_disc_source src; void *priv_data; unsigned long caps; @@ -135,8 +136,6 @@ struct intel_vsec_platform_info { * struct intel_vsec_device - Auxbus specific device information * @auxdev: auxbus device struct for auxbus access * @dev: struct device associated with the device - * @resource: PCI discovery resources (BAR windows), one per discovery - * instance. Valid only when @src == INTEL_VSEC_DISC_PCI * @acpi_disc: ACPI discovery tables, each entry is two QWORDs * in little-endian format as defined by the PMT ACPI spec. * Valid only when @src == INTEL_VSEC_DISC_ACPI. @@ -149,12 +148,13 @@ struct intel_vsec_platform_info { * @quirks: specified quirks * @base_addr: base address of entries (if specified) * @cap_id: the enumerated id of the vsec feature + * @resource: PCI discovery resources (BAR windows), one per discovery + * instance. Valid only when @src == INTEL_VSEC_DISC_PCI */ struct intel_vsec_device { struct auxiliary_device auxdev; struct device *dev; - struct resource *resource; - u32 (*acpi_disc)[4]; + u32 (*acpi_disc)[PMT_DISC_DWORDS]; enum intel_vsec_disc_source src; struct ida *ida; int num_resources; @@ -164,6 +164,7 @@ struct intel_vsec_device { unsigned long quirks; u64 base_addr; unsigned long cap_id; + struct resource resource[] __counted_by(num_resources); }; /** diff --git a/include/linux/interconnect.h b/include/linux/interconnect.h index 4b12821528a6..75a32ad0482e 100644 --- a/include/linux/interconnect.h +++ b/include/linux/interconnect.h @@ -47,6 +47,7 @@ struct icc_path *of_icc_get(struct device *dev, const char *name); struct icc_path *devm_of_icc_get(struct device *dev, const char *name); int devm_of_icc_bulk_get(struct device *dev, int num_paths, struct icc_bulk_data *paths); struct icc_path *of_icc_get_by_index(struct device *dev, int idx); +struct icc_path *devm_of_icc_get_by_index(struct device *dev, int idx); void icc_put(struct icc_path *path); int icc_enable(struct icc_path *path); int icc_disable(struct icc_path *path); @@ -79,6 +80,11 @@ static inline struct icc_path *of_icc_get_by_index(struct device *dev, int idx) return NULL; } +static inline struct icc_path *devm_of_icc_get_by_index(struct device *dev, int idx) +{ + return NULL; +} + static inline void icc_put(struct icc_path *path) { } diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 6415a3353ee0..a2c623a67a25 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -149,8 +149,6 @@ struct io_uring_task { struct { /* task_work */ struct mpscq task_list; - /* BIT(0) guards adding tw only once */ - unsigned long tw_pending; struct callback_head task_work; } ____cacheline_aligned_in_smp; }; @@ -536,6 +534,8 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; + + struct kcov_common_handle_id kcov_handle; }; /* diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 3582ed1fe236..56b43d594e6e 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -622,6 +622,8 @@ extern struct bio_set iomap_ioend_bioset; #ifdef CONFIG_BLOCK int iomap_bio_read_folio_range(const struct iomap_iter *iter, struct iomap_read_folio_ctx *ctx, size_t plen); +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io); extern const struct iomap_read_ops iomap_bio_read_ops; diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 3c73c9c0d4f7..f7930b3dfd0a 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -261,7 +261,7 @@ extern int allocate_resource(struct resource *root, struct resource *new, struct resource *lookup_resource(struct resource *root, resource_size_t start); int adjust_resource(struct resource *res, resource_size_t start, resource_size_t size); -resource_size_t resource_alignment(struct resource *res); +resource_size_t resource_alignment(const struct resource *res); /** * resource_set_size - Calculate resource end address from size and start diff --git a/include/linux/ipack.h b/include/linux/ipack.h index 455f6c2a1903..7edbf9267338 100644 --- a/include/linux/ipack.h +++ b/include/linux/ipack.h @@ -6,7 +6,7 @@ * Author: Samuel Iglesias Gonsalvez */ -#include +#include #include #include diff --git a/include/linux/irqchip/irq-renesas-rzv2h.h b/include/linux/irqchip/irq-renesas-rzv2h.h index 618a60d2eac0..4ffa898eaaf2 100644 --- a/include/linux/irqchip/irq-renesas-rzv2h.h +++ b/include/linux/irqchip/irq-renesas-rzv2h.h @@ -11,13 +11,18 @@ #include #define RZV2H_ICU_DMAC_REQ_NO_DEFAULT 0x3ff +#define RZV2H_ICU_DMAC_ACK_NO_DEFAULT 0x7f #ifdef CONFIG_RENESAS_RZV2H_ICU void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 req_no); +void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index, + u8 dmac_channel, u16 ack_no); #else static inline void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 req_no) { } +static inline void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index, + u8 dmac_channel, u16 ack_no) { } #endif #endif /* __LINUX_IRQ_RENESAS_RZV2H */ diff --git a/include/linux/isapnp.h b/include/linux/isapnp.h index dba18c95844b..8f5a85ca6c1f 100644 --- a/include/linux/isapnp.h +++ b/include/linux/isapnp.h @@ -28,7 +28,7 @@ */ #ifdef __KERNEL__ -#include +#include #define DEVICE_COUNT_COMPATIBLE 4 diff --git a/include/linux/kcov.h b/include/linux/kcov.h index 0143358874b0..895b761b2db1 100644 --- a/include/linux/kcov.h +++ b/include/linux/kcov.h @@ -21,8 +21,6 @@ enum kcov_mode { KCOV_MODE_TRACE_PC = 2, /* Collecting comparison operands mode. */ KCOV_MODE_TRACE_CMP = 3, - /* The process owns a KCOV remote reference. */ - KCOV_MODE_REMOTE = 4, }; #define KCOV_IN_CTXSW (1 << 30) @@ -43,11 +41,11 @@ do { \ /* See Documentation/dev-tools/kcov.rst for usage details. */ void kcov_remote_start(u64 handle); void kcov_remote_stop(void); -u64 kcov_common_handle(void); +struct kcov_common_handle_id kcov_common_handle(void); -static inline void kcov_remote_start_common(u64 id) +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) { - kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id)); + kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id.val)); } static inline void kcov_remote_start_usb(u64 id) @@ -99,11 +97,11 @@ static inline void kcov_prepare_switch(struct task_struct *t) {} static inline void kcov_finish_switch(struct task_struct *t) {} static inline void kcov_remote_start(u64 handle) {} static inline void kcov_remote_stop(void) {} -static inline u64 kcov_common_handle(void) +static inline struct kcov_common_handle_id kcov_common_handle(void) { - return 0; + return (struct kcov_common_handle_id){}; } -static inline void kcov_remote_start_common(u64 id) {} +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) {} static inline void kcov_remote_start_usb(u64 id) {} static inline void kcov_remote_start_usb_softirq(u64 id) {} static inline void kcov_remote_stop_softirq(void) {} diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index fce1392e2140..9ca6c2259dfe 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -107,6 +107,30 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) } #ifdef CONFIG_NO_HZ_COMMON + +#ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE + +static inline void kcpustat_dyntick_start(u64 now) { } +static inline void kcpustat_dyntick_stop(u64 now) { } +static inline void kcpustat_irq_enter(u64 now) { } +static inline void kcpustat_irq_exit(u64 now) { } +static inline bool kcpustat_idle_dyntick(void) { return false; } + +extern u64 arch_kcpustat_field_idle(int cpu); +extern u64 arch_kcpustat_field_iowait(int cpu); + +static inline u64 kcpustat_field_idle(int cpu) +{ + return arch_kcpustat_field_idle(cpu); +} + +static inline u64 kcpustat_field_iowait(int cpu) +{ + return arch_kcpustat_field_iowait(cpu); +} + +#else /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + extern void kcpustat_dyntick_start(u64 now); extern void kcpustat_dyntick_stop(u64 now); extern void kcpustat_irq_enter(u64 now); @@ -118,6 +142,9 @@ static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); } + +#endif /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + #else static inline u64 kcpustat_field_idle(int cpu) { diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8a22bc9b8c6c..0af8ae4fdd08 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -13,7 +13,7 @@ #define IND_SOURCE (1 << IND_SOURCE_BIT) #define IND_FLAGS (IND_DESTINATION | IND_INDIRECTION | IND_DONE | IND_SOURCE) -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) #include #include diff --git a/include/linux/kho/abi/block.h b/include/linux/kho/abi/block.h new file mode 100644 index 000000000000..d06d64b963be --- /dev/null +++ b/include/linux/kho/abi/block.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +/** + * DOC: KHO Serialization Blocks ABI + * + * Subsystems using the KHO Serialization Blocks framework rely on the stable + * Application Binary Interface defined below to pass serialized state from a + * pre-update kernel to a post-update kernel. + * + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization + * structures defined here constitutes a breaking change. Such changes require + * incrementing the version number in the `KHO_FDT_COMPATIBLE` string to + * prevent a new kernel from misinterpreting data from an old kernel. + * + * Changes are allowed provided the compatibility version is incremented; + * however, backward/forward compatibility is only guaranteed for kernels + * supporting the same ABI version. + */ + +#ifndef _LINUX_KHO_ABI_BLOCK_H +#define _LINUX_KHO_ABI_BLOCK_H + +#include +#include + +/** + * KHO_BLOCK_SIZE - The size of each serialization block. + * + * This is defined as PAGE_SIZE. PAGE_SIZE is ABI compliant because live + * update between kernels with different page sizes is not supported by KHO. + */ +#define KHO_BLOCK_SIZE PAGE_SIZE + +/** + * struct kho_block_header_ser - Header for the serialized data block. + * @next: Physical address of the next struct kho_block_header_ser. + * @count: The number of entries that immediately follow this header in the + * memory block. + * + * This structure is located at the beginning of a block of physical memory + * preserved across a kexec. It provides the necessary metadata to interpret + * the array of entries that follow. + */ +struct kho_block_header_ser { + u64 next; + u64 count; +} __packed; + +#endif /* _LINUX_KHO_ABI_BLOCK_H */ diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h index db9bda6dd310..5e2eb8519bda 100644 --- a/include/linux/kho/abi/kexec_handover.h +++ b/include/linux/kho/abi/kexec_handover.h @@ -64,7 +64,7 @@ * Root KHO Node (/): * - compatible: "kho-v3" * - * Indentifies the overall KHO ABI version. + * Identifies the overall KHO ABI version. * * - preserved-memory-map: u64 * @@ -90,7 +90,7 @@ */ /* The compatible string for the KHO FDT root node. */ -#define KHO_FDT_COMPATIBLE "kho-v3" +#define KHO_FDT_COMPATIBLE "kho-v4" /* The FDT property for the preserved memory map. */ #define KHO_FDT_MEMORY_MAP_PROP_NAME "preserved-memory-map" diff --git a/include/linux/kho/abi/luo.h b/include/linux/kho/abi/luo.h index 46750a0ddf88..288076de6d4a 100644 --- a/include/linux/kho/abi/luo.h +++ b/include/linux/kho/abi/luo.h @@ -10,11 +10,11 @@ * * Live Update Orchestrator uses the stable Application Binary Interface * defined below to pass state from a pre-update kernel to a post-update - * kernel. The ABI is built upon the Kexec HandOver framework and uses a - * Flattened Device Tree to describe the preserved data. + * kernel. The ABI is built upon the Kexec HandOver framework and registers + * the central `struct luo_ser` via the KHO raw subtree API. * - * This interface is a contract. Any modification to the FDT structure, node - * properties, compatible strings, or the layout of the `__packed` serialization + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization * structures defined here constitutes a breaking change. Such changes require * incrementing the version number in the relevant `_COMPATIBLE` string to * prevent a new kernel from misinterpreting data from an old kernel. @@ -23,68 +23,20 @@ * however, backward/forward compatibility is only guaranteed for kernels * supporting the same ABI version. * - * FDT Structure Overview: + * KHO Structure Overview: * The entire LUO state is encapsulated within a single KHO entry named "LUO". - * This entry contains an FDT with the following layout: - * - * .. code-block:: none - * - * / { - * compatible = "luo-v1"; - * liveupdate-number = <...>; - * - * luo-session { - * compatible = "luo-session-v1"; - * luo-session-header = ; - * }; - * - * luo-flb { - * compatible = "luo-flb-v1"; - * luo-flb-header = ; - * }; - * }; - * - * Main LUO Node (/): - * - * - compatible: "luo-v1" - * Identifies the overall LUO ABI version. - * - liveupdate-number: u64 - * A counter tracking the number of successful live updates performed. - * - * Session Node (luo-session): - * This node describes all preserved user-space sessions. - * - * - compatible: "luo-session-v1" - * Identifies the session ABI version. - * - luo-session-header: u64 - * The physical address of a `struct luo_session_header_ser`. This structure - * is the header for a contiguous block of memory containing an array of - * `struct luo_session_ser`, one for each preserved session. - * - * File-Lifecycle-Bound Node (luo-flb): - * This node describes all preserved global objects whose lifecycle is bound - * to that of the preserved files (e.g., shared IOMMU state). - * - * - compatible: "luo-flb-v1" - * Identifies the FLB ABI version. - * - luo-flb-header: u64 - * The physical address of a `struct luo_flb_header_ser`. This structure is - * the header for a contiguous block of memory containing an array of - * `struct luo_flb_ser`, one for each preserved global object. + * This entry contains the `struct luo_ser` structure. * * Serialization Structures: - * The FDT properties point to memory regions containing arrays of simple, - * `__packed` structures. These structures contain the actual preserved state. - * - * - struct luo_session_header_ser: - * Header for the session array. Contains the total page count of the - * preserved memory block and the number of `struct luo_session_ser` - * entries that follow. + * - struct luo_ser: + * The central ABI structure that contains the overall state of the LUO. + * It includes the compatibility string, the liveupdate-number, and pointers + * to sessions and FLBs. * * - struct luo_session_ser: * Metadata for a single session, including its name and a physical pointer - * to another preserved memory block containing an array of - * `struct luo_file_ser` for all files in that session. + * to the first `struct kho_block_header_ser` for all files in that session. + * Multiple blocks are linked via the `next` field in the header. * * - struct luo_file_ser: * Metadata for a single preserved file. Contains the `compatible` string to @@ -105,17 +57,32 @@ #ifndef _LINUX_KHO_ABI_LUO_H #define _LINUX_KHO_ABI_LUO_H +#include +#include #include /* - * The LUO FDT hooks all LUO state for sessions, fds, etc. - * In the root it also carries "liveupdate-number" 64-bit property that - * corresponds to the number of live-updates performed on this machine. + * The LUO state is registered under this KHO entry name. */ -#define LUO_FDT_SIZE PAGE_SIZE -#define LUO_FDT_KHO_ENTRY_NAME "LUO" -#define LUO_FDT_COMPATIBLE "luo-v1" -#define LUO_FDT_LIVEUPDATE_NUM "liveupdate-number" +#define LUO_KHO_ENTRY_NAME "LUO" +#define LUO_ABI_COMPATIBLE "luo-v5" +#define LUO_ABI_COMPAT_LEN ALIGN(sizeof(LUO_ABI_COMPATIBLE), 8) + +/** + * struct luo_ser - Centralized LUO ABI header. + * @compatible: Compatibility string identifying the LUO ABI version. + * @liveupdate_num: A counter tracking the number of successful live updates. + * @sessions_pa: Physical address of the first session block header. + * @flbs_pa: Physical address of the FLB header. + * + * This structure is the root of all preserved LUO state. + */ +struct luo_ser { + char compatible[LUO_ABI_COMPAT_LEN]; + u64 liveupdate_num; + u64 sessions_pa; + u64 flbs_pa; +} __packed; #define LIVEUPDATE_HNDL_COMPAT_LENGTH 48 @@ -125,7 +92,7 @@ * @data: Private data * @token: User provided token for this file * - * If this structure is modified, LUO_SESSION_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_file_ser { char compatible[LIVEUPDATE_HNDL_COMPAT_LENGTH]; @@ -135,9 +102,10 @@ struct luo_file_ser { /** * struct luo_file_set_ser - Represents the serialized metadata for file set - * @files: The physical address of a contiguous memory block that holds - * the serialized state of files (array of luo_file_ser) in this file - * set. + * @files: The physical address of the first `struct kho_block_header_ser`. + * This structure is the header for a block of memory containing + * an array of `struct luo_file_ser` entries. Multiple blocks are + * linked via the `next` field in the header. * @count: The total number of files that were part of this session during * serialization. Used for iteration and validation during * restoration. @@ -147,30 +115,6 @@ struct luo_file_set_ser { u64 count; } __packed; -/* - * LUO FDT session node - * LUO_FDT_SESSION_HEADER: is a u64 physical address of struct - * luo_session_header_ser - */ -#define LUO_FDT_SESSION_NODE_NAME "luo-session" -#define LUO_FDT_SESSION_COMPATIBLE "luo-session-v2" -#define LUO_FDT_SESSION_HEADER "luo-session-header" - -/** - * struct luo_session_header_ser - Header for the serialized session data block. - * @count: The number of `struct luo_session_ser` entries that immediately - * follow this header in the memory block. - * - * This structure is located at the beginning of a contiguous block of - * physical memory preserved across the kexec. It provides the necessary - * metadata to interpret the array of session entries that follow. - * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. - */ -struct luo_session_header_ser { - u64 count; -} __packed; - /** * struct luo_session_ser - Represents the serialized metadata for a LUO session. * @name: The unique name of the session, provided by the userspace at @@ -182,7 +126,7 @@ struct luo_session_header_ser { * session) is created and passed to the new kernel, allowing it to reconstruct * the session context. * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_session_ser { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; @@ -192,10 +136,6 @@ struct luo_session_ser { /* The max size is set so it can be reliably used during in serialization */ #define LIVEUPDATE_FLB_COMPAT_LENGTH 48 -#define LUO_FDT_FLB_NODE_NAME "luo-flb" -#define LUO_FDT_FLB_COMPATIBLE "luo-flb-v1" -#define LUO_FDT_FLB_HEADER "luo-flb-header" - /** * struct luo_flb_header_ser - Header for the serialized FLB data block. * @pgcnt: The total number of pages occupied by the entire preserved memory @@ -205,11 +145,9 @@ struct luo_session_ser { * in the memory block. * * This structure is located at the physical address specified by the - * `LUO_FDT_FLB_HEADER` FDT property. It provides the new kernel with the - * necessary information to find and iterate over the array of preserved - * File-Lifecycle-Bound objects and to manage the underlying memory. + * flbs_pa in luo_ser. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_header_ser { u64 pgcnt; @@ -231,7 +169,7 @@ struct luo_flb_header_ser { * passed to the new kernel. Each entry allows the LUO core to restore one * global, shared object. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_ser { char name[LIVEUPDATE_FLB_COMPAT_LENGTH]; diff --git a/include/linux/kho_block.h b/include/linux/kho_block.h new file mode 100644 index 000000000000..93a7cc2be5f5 --- /dev/null +++ b/include/linux/kho_block.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +#ifndef _LINUX_KHO_BLOCK_H +#define _LINUX_KHO_BLOCK_H + +#include +#include +#include + +/** + * struct kho_block - Internal representation of a serialization block. + * @list: List head for linking blocks in memory. + * @ser: Pointer to the serialized header in preserved memory. + */ +struct kho_block { + struct list_head list; + struct kho_block_header_ser *ser; +}; + +/** + * struct kho_block_set - A set of blocks containing serialized entries of the same type. + * @blocks: The list of serialization blocks (struct kho_block). + * @nblocks: The number of allocated serialization blocks. + * @head_pa: Physical address of the first block header. + * @entry_size: The size of each entry in the blocks. + * @count_per_block: The maximum number of entries each block can hold. + * @incoming: True if this block set was restored from the previous kernel. + * + * Note: Synchronization and locking are the responsibility of the caller. + * The block set structure itself is not internally synchronized. + */ +struct kho_block_set { + struct list_head blocks; + long nblocks; + u64 head_pa; + size_t entry_size; + u64 count_per_block; + bool incoming; +}; + +/** + * struct kho_block_set_it - Iterator for serializing entries into blocks. + * @bs: The block set being iterated. + * @block: The current block. + * @i: The current entry index within @block. + */ +struct kho_block_set_it { + struct kho_block_set *bs; + struct kho_block *block; + u64 i; +}; + +/** + * KHO_BLOCK_SET_INIT - Initialize a static kho_block_set. + * @_name: Name of the kho_block_set variable. + * @_entry_size: The size of each entry in the block set. + */ +#define KHO_BLOCK_SET_INIT(_name, _entry_size) { \ + .blocks = LIST_HEAD_INIT((_name).blocks), \ + .entry_size = _entry_size, \ + .count_per_block = (KHO_BLOCK_SIZE - \ + sizeof(struct kho_block_header_ser)) / \ + (_entry_size), \ +} + +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size); + +int kho_block_set_grow(struct kho_block_set *bs, u64 count); +void kho_block_set_shrink(struct kho_block_set *bs, u64 count); + +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa); +void kho_block_set_destroy(struct kho_block_set *bs); +void kho_block_set_clear(struct kho_block_set *bs); + +/** + * kho_block_set_head_pa - Get the physical address of the first block header. + * @bs: The block set. + * + * Return: The physical address of the first block header, or 0 if empty. + */ +static inline u64 kho_block_set_head_pa(struct kho_block_set *bs) +{ + return bs->head_pa; +} + +/** + * kho_block_set_is_empty - Check if the block set has no allocated blocks. + * @bs: The block set. + * + * Return: True if there are no blocks in the set, false otherwise. + */ +static inline bool kho_block_set_is_empty(struct kho_block_set *bs) +{ + return list_empty(&bs->blocks); +} + +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs); +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it); +void *kho_block_set_it_read_entry(struct kho_block_set_it *it); +void *kho_block_set_it_prev(struct kho_block_set_it *it); + +#endif /* _LINUX_KHO_BLOCK_H */ diff --git a/include/linux/kstrtox.h b/include/linux/kstrtox.h index 6c9282866770..b41b9736886f 100644 --- a/include/linux/kstrtox.h +++ b/include/linux/kstrtox.h @@ -97,6 +97,9 @@ int __must_check kstrtou8(const char *s, unsigned int base, u8 *res); int __must_check kstrtos8(const char *s, unsigned int base, s8 *res); int __must_check kstrtobool(const char *s, bool *res); +int __must_check kstrtoudec64(const char *s, unsigned int scale, u64 *res); +int __must_check kstrtodec64(const char *s, unsigned int scale, s64 *res); + int __must_check kstrtoull_from_user(const char __user *s, size_t count, unsigned int base, unsigned long long *res); int __must_check kstrtoll_from_user(const char __user *s, size_t count, unsigned int base, long long *res); int __must_check kstrtoul_from_user(const char __user *s, size_t count, unsigned int base, unsigned long *res); diff --git a/include/linux/leafops.h b/include/linux/leafops.h index 992cd8bd8ed0..88888daeb018 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -108,6 +108,21 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** @@ -567,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -586,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? diff --git a/include/linux/leds.h b/include/linux/leds.h index b16b803cc1ac..bf31c246d9e2 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -607,6 +607,10 @@ enum led_trigger_netdev_modes { TRIGGER_NETDEV_LINK_2500, TRIGGER_NETDEV_LINK_5000, TRIGGER_NETDEV_LINK_10000, + TRIGGER_NETDEV_LINK_25000, + TRIGGER_NETDEV_LINK_40000, + TRIGGER_NETDEV_LINK_50000, + TRIGGER_NETDEV_LINK_100000, TRIGGER_NETDEV_HALF_DUPLEX, TRIGGER_NETDEV_FULL_DUPLEX, TRIGGER_NETDEV_TX, diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 30c5a39ff9e9..88722e5caf02 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -175,7 +176,7 @@ struct liveupdate_flb_ops { * @retrieved: True once the FLB's retrieve() callback has run. */ struct luo_flb_private_state { - long count; + refcount_t count; u64 data; void *obj; struct mutex lock; @@ -239,6 +240,8 @@ void liveupdate_unregister_flb(struct liveupdate_file_handler *fh, struct liveupdate_flb *flb); int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); + int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); #else /* CONFIG_LIVEUPDATE */ @@ -279,6 +282,10 @@ static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) +{ +} + static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) { diff --git a/include/linux/llist.h b/include/linux/llist.h index 607b2360c938..8846b7709669 100644 --- a/include/linux/llist.h +++ b/include/linux/llist.h @@ -26,8 +26,8 @@ * * | add | del_first | del_all * add | - | - | - - * del_first | | L | L - * del_all | | | - + * del_first | - | L | L + * del_all | - | - | - * * Where, a particular row's operation can happen concurrently with a column's * operation, with "-" being no lock needed, while "L" being lock is needed. diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h index b614e0deea72..db8207d4059f 100644 --- a/include/linux/lockd/bind.h +++ b/include/linux/lockd/bind.h @@ -16,17 +16,23 @@ struct svc_rqst; struct rpc_task; struct rpc_clnt; struct super_block; +struct module; -/* - * This is the set of functions for lockd->nfsd communication +/** + * struct nlmsvc_binding - lockd -> nfsd callback table + * @owner: module that provides this binding. + * @fopen: open a file by NFS file handle on behalf of an NLM request. + * @fclose: close a file that was previously opened via @fopen. + * Implementations MUST be semantically equivalent to fput(). */ struct nlmsvc_binding { + struct module *owner; int (*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f, struct file **filp, int flags); void (*fclose)(struct file *filp); }; -extern const struct nlmsvc_binding *nlmsvc_ops; +extern const struct nlmsvc_binding __rcu *nlmsvc_ops; /* * Similar to nfs_client_initdata, but without the NFS-specific diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..a6451ecbbe9a 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie); extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); +extern u32 lock_sequence(struct lockdep_map *lock); +#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map) + #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) #define lockdep_assert(cond) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..55c4b152fedf 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -253,7 +253,8 @@ struct held_lock { unsigned int hardirqs_off:1; unsigned int sync:1; unsigned int references:11; /* 32 bits */ - unsigned int pin_count; + unsigned int pin_count:24; + unsigned int seq_count:8; }; #else /* !CONFIG_LOCKDEP */ diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h index dc93287a2a01..26a238a6f941 100644 --- a/include/linux/mailbox_controller.h +++ b/include/linux/mailbox_controller.h @@ -120,6 +120,7 @@ struct mbox_controller { * @txdone_method: Way to detect TXDone chosen by the API * @cl: Pointer to the current owner of this channel * @tx_complete: Transmission completion + * @tx_status: Transmission status * @active_req: Currently active request hook * @msg_count: No. of mssg currently queued * @msg_free: Index of next available mssg slot @@ -132,6 +133,7 @@ struct mbox_chan { unsigned txdone_method; struct mbox_client *cl; struct completion tx_complete; + int tx_status; void *active_req; unsigned msg_count, msg_free; void *msg_data[MBOX_TX_QUEUE_LEN]; diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 4a5631906aff..5e0bd2857d94 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -29,13 +29,11 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) #else /* 32bit sizes */ #define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ #define MAPLE_NODE_MASK 255UL @@ -485,6 +483,12 @@ struct ma_state { unsigned char mas_flags; unsigned char end; /* The end of the node */ enum store_type store_type; /* The type of store needed for this operation */ +#ifdef CONFIG_LOCKDEP + u32 ld_seq; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + unsigned long rcu_gp; +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ +#endif /* CONFIG_LOCKDEP */ }; struct ma_wr_state { @@ -572,7 +576,7 @@ void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); diff --git a/include/linux/math64.h b/include/linux/math64.h index cc305206d89f..99189410d4bb 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -57,6 +57,20 @@ static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder) return dividend / divisor; } +/** + * div64_s64_rem - signed 64bit divide with 64bit divisor and remainder + * @dividend: signed 64bit dividend + * @divisor: signed 64bit divisor + * @remainder: pointer to signed 64bit remainder + * + * Return: sets ``*remainder``, then returns dividend / divisor + */ +static inline s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder) +{ + *remainder = dividend % divisor; + return dividend / divisor; +} + /** * div64_u64 - unsigned 64bit divide with 64bit divisor * @dividend: unsigned 64bit dividend @@ -102,6 +116,10 @@ extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder); extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder); #endif +#ifndef div64_s64_rem +extern s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder); +#endif + #ifndef div64_u64 extern u64 div64_u64(u64 dividend, u64 divisor); #endif diff --git a/include/linux/mcb.h b/include/linux/mcb.h index 4ab2691f51a6..874118765d0f 100644 --- a/include/linux/mcb.h +++ b/include/linux/mcb.h @@ -8,7 +8,7 @@ #ifndef _LINUX_MCB_H #define _LINUX_MCB_H -#include +#include #include #include diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index 5bdbd9e1d460..d5d29451eabf 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -7,7 +7,7 @@ #include #include -#include +#include struct mei_cl_device; struct mei_device; diff --git a/include/linux/memblock.h b/include/linux/memblock.h index b0f750d22a7b..5afcd99aa8c1 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -613,11 +613,28 @@ static inline void memtest_report_meminfo(struct seq_file *m) { } #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH void memblock_set_kho_scratch_only(void); void memblock_clear_kho_scratch_only(void); -void memmap_init_kho_scratch_pages(void); +bool memblock_is_kho_scratch_memory(phys_addr_t addr); + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + if (memblock_is_kho_scratch_memory(PFN_PHYS(pfn))) + return MIGRATE_CMA; + return mt; +} #else static inline void memblock_set_kho_scratch_only(void) { } static inline void memblock_clear_kho_scratch_only(void) { } -static inline void memmap_init_kho_scratch_pages(void) {} +static inline bool memblock_is_kho_scratch_memory(phys_addr_t addr) +{ + return false; +} + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + return mt; +} #endif #endif /* _LINUX_MEMBLOCK_H */ diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c60..06c58cb05779 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, diff --git a/include/linux/mfd/88pm886.h b/include/linux/mfd/88pm886.h index 38892ba7b8a4..2c24dd3032ab 100644 --- a/include/linux/mfd/88pm886.h +++ b/include/linux/mfd/88pm886.h @@ -11,6 +11,7 @@ #define PM886_PAGE_OFFSET_REGULATORS 1 #define PM886_PAGE_OFFSET_GPADC 2 +#define PM886_PAGE_OFFSET_BATTERY 3 #define PM886_REG_ID 0x00 @@ -128,9 +129,13 @@ #define PM886_GPADC_BIAS_LEVELS 16 #define PM886_GPADC_INDEX_TO_BIAS_uA(i) (1 + (i) * 5) +/* Battery block register definitions */ +#define PM886_REG_CLS_CONFIG1 0x71 + struct pm886_chip { struct i2c_client *client; unsigned int chip_id; struct regmap *regmap; + struct regmap *regmap_battery; }; #endif /* __MFD_88PM886_H */ diff --git a/include/linux/mfd/cs42l43.h b/include/linux/mfd/cs42l43.h index ff0f7e365a19..8e993fb535e6 100644 --- a/include/linux/mfd/cs42l43.h +++ b/include/linux/mfd/cs42l43.h @@ -86,7 +86,6 @@ struct cs42l43 { struct regmap_irq_chip_data *irq_data; struct work_struct boot_work; - struct completion device_attach; struct completion device_detach; struct completion firmware_download; int firmware_error; @@ -96,7 +95,6 @@ struct cs42l43 { struct mutex pll_lock; bool sdw_pll_active; - bool attached; bool hw_lock; long variant_id; }; diff --git a/include/linux/mfd/max14577-private.h b/include/linux/mfd/max14577-private.h index dd51a37fa37f..5957e15b568e 100644 --- a/include/linux/mfd/max14577-private.h +++ b/include/linux/mfd/max14577-private.h @@ -350,6 +350,9 @@ enum max77836_pmic_reg { #define MAX77836_CNFG1_LDO_PWRMD_SHIFT 6 #define MAX77836_CNFG1_LDO_TV_SHIFT 0 #define MAX77836_CNFG1_LDO_PWRMD_MASK (0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_OFF (0x0 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_LPM (0x1 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_NORMAL (0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) #define MAX77836_CNFG1_LDO_TV_MASK (0x3f << MAX77836_CNFG1_LDO_TV_SHIFT) /* LDO1/LDO2 CONFIG2 register */ diff --git a/include/linux/mfd/si476x-core.h b/include/linux/mfd/si476x-core.h index e913b2cdf77d..d9e3a322134c 100644 --- a/include/linux/mfd/si476x-core.h +++ b/include/linux/mfd/si476x-core.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -104,7 +105,7 @@ enum si476x_power_state { * @power_state: Current power state of the device. * @supplies: Structure containing handles to all power supplies used * by the device (NULL ones are ignored). - * @gpio_reset: GPIO pin connectet to the RSTB pin of the chip. + * @reset: GPIO connected to the RSTB pin of the chip. * @pinmux: Chip's configurable pins configuration. * @diversity_mode: Chips role when functioning in diversity mode. * @is_alive: Chip is initialized and active. @@ -142,7 +143,7 @@ struct si476x_core { struct regulator_bulk_data supplies[4]; - int gpio_reset; + struct gpio_desc *reset; struct si476x_pinmux pinmux; enum si476x_phase_diversity_mode diversity_mode; diff --git a/include/linux/mfd/si476x-platform.h b/include/linux/mfd/si476x-platform.h index cb99e16ca947..f9e1f6b27277 100644 --- a/include/linux/mfd/si476x-platform.h +++ b/include/linux/mfd/si476x-platform.h @@ -246,8 +246,6 @@ enum si476x_phase_diversity_mode { * Platform dependent definition */ struct si476x_platform_data { - int gpio_reset; /* < 0 if not used */ - struct si476x_power_up_args power_up_parameters; enum si476x_phase_diversity_mode diversity_mode; diff --git a/include/linux/mhi.h b/include/linux/mhi.h index fb3ba639f4f8..4b86ae6f6a82 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -14,6 +14,7 @@ #include #include #include +#include #define MHI_MAX_OEM_PK_HASH_SEGMENTS 16 diff --git a/include/linux/migrate.h b/include/linux/migrate.h index d5af2b7f577b..1f83924615d6 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -57,7 +57,8 @@ void putback_movable_pages(struct list_head *l); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, - unsigned long private, enum migrate_mode mode, int reason, + unsigned long private, enum migrate_mode mode, + enum migrate_reason reason, unsigned int *ret_succeeded); struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode); @@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type); static inline void putback_movable_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, + unsigned int *ret_succeeded) { return -ENOSYS; } static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 265c4328b36a..05102d4d2490 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -25,6 +25,7 @@ enum migrate_reason { MR_LONGTERM_PIN, MR_DEMOTION, MR_DAMON, + MR_NEVER, /* page has never been migrated */ MR_TYPES }; diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..113e1752548c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -37,6 +37,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -2286,23 +2287,45 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -int memdesc_nid(memdesc_flags_t mdf); +int memdesc_nid(const memdesc_flags_t *mdf); #else -static inline int memdesc_nid(memdesc_flags_t mdf) +#ifdef CONFIG_NUMA +static inline int memdesc_nid(const memdesc_flags_t *mdf) { - return (mdf.f >> NODES_PGSHIFT) & NODES_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); + return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; +} +#else +static inline int memdesc_nid(const memdesc_flags_t *mdf) +{ + return 0; } #endif +#endif + +#ifdef CONFIG_NUMA +static inline int page_to_nid(const struct page *page) +{ + const struct page *p = PF_POISONED_CHECK(page); + return memdesc_nid(&p->flags); +} + +static inline int folio_nid(const struct folio *folio) +{ + return memdesc_nid(&folio->flags); +} +#else static inline int page_to_nid(const struct page *page) { - return memdesc_nid(PF_POISONED_CHECK(page)->flags); + return 0; } static inline int folio_nid(const struct folio *folio) { - return memdesc_nid(folio->flags); + return 0; } +#endif #ifdef CONFIG_NUMA_BALANCING /* page access time bits needs to hold at least 4 seconds */ @@ -2541,12 +2564,15 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { - return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; +#if SECTIONS_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); +#endif + return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; } @@ -4042,7 +4068,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); extern void mem_init(void); extern void __init mmap_init(void); -extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); static inline void show_mem(void) { __show_mem(0, NULL, MAX_NR_ZONES - 1); @@ -4052,7 +4078,7 @@ extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern __printf(3, 4) -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index a8430a7ae054..efcddb9925ad 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -650,7 +650,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma) static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) { struct page *cur_page = pages[0]; - unsigned long section = memdesc_section(cur_page->flags); + unsigned long section = memdesc_section(&cur_page->flags); size_t i; for (i = 1; i < nr_pages; i++) { @@ -660,7 +660,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) * In unproblematic kernel configs, page_to_section() == 0 and * the whole check will get optimized out. */ - if (memdesc_section(cur_page->flags) != section) + if (memdesc_section(&cur_page->flags) != section) break; } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b18c2b2e7d2c..ae9bca4eda5c 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -427,6 +427,7 @@ struct folio { union { void *private; swp_entry_t swap; + unsigned long migrate_info; }; atomic_t _mapcount; atomic_t _refcount; diff --git a/include/linux/mmc/sdio_func.h b/include/linux/mmc/sdio_func.h index 4534bf462aac..5d63a6465b0d 100644 --- a/include/linux/mmc/sdio_func.h +++ b/include/linux/mmc/sdio_func.h @@ -9,7 +9,7 @@ #define LINUX_MMC_SDIO_FUNC_H #include -#include +#include #include diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 0685dd717e85..7dac5428afe0 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -118,6 +118,7 @@ #define SDIO_DEVICE_ID_MICROCHIP_WILC1000 0x5347 #define SDIO_VENDOR_ID_NXP 0x0471 +#define SDIO_DEVICE_ID_NXP_IW61X_BASE 0x0204 #define SDIO_DEVICE_ID_NXP_IW61X 0x0205 #define SDIO_VENDOR_ID_REALTEK 0x024c diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index ca2712187147..0507193b3ae3 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MMZONE_H #define _LINUX_MMZONE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __GENERATING_BOUNDS_H #include @@ -1272,31 +1272,33 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) -static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags) +static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags) { - ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT); - return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK; +#if ZONES_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT); +#endif + return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK; } static inline enum zone_type page_zonenum(const struct page *page) { - return memdesc_zonenum(page->flags); + return memdesc_zonenum(&page->flags); } static inline enum zone_type folio_zonenum(const struct folio *folio) { - return memdesc_zonenum(folio->flags); + return memdesc_zonenum(&folio->flags); } #ifdef CONFIG_ZONE_DEVICE -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return memdesc_zonenum(mdf) == ZONE_DEVICE; } static inline struct dev_pagemap *page_pgmap(const struct page *page) { - VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page); + VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap; } @@ -1311,9 +1313,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { - if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags)) + if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; - if (!memdesc_is_zone_device(a->flags)) + if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b); } @@ -1321,7 +1323,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return false; } @@ -1338,12 +1340,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool is_zone_device_page(const struct page *page) { - return memdesc_is_zone_device(page->flags); + return memdesc_is_zone_device(&page->flags); } static inline bool folio_is_zone_device(const struct folio *folio) { - return memdesc_is_zone_device(folio->flags); + return memdesc_is_zone_device(&folio->flags); } static inline bool is_zone_movable_page(const struct page *page) @@ -1815,7 +1817,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref) struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes); + const nodemask_t *nodes); /** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point @@ -1834,7 +1836,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z, */ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; @@ -1860,7 +1862,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes); @@ -2156,8 +2158,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) } void sparse_vmemmap_init_nid_early(int nid); -void sparse_vmemmap_init_nid_late(int nid); - #else static inline int preinited_vmemmap_section(const struct mem_section *section) { @@ -2166,10 +2166,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) static inline void sparse_vmemmap_init_nid_early(int nid) { } - -static inline void sparse_vmemmap_init_nid_late(int nid) -{ -} #endif static inline int online_section_nr(unsigned long nr) @@ -2374,7 +2370,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #else #define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define sparse_vmemmap_init_nid_late(_nid) do {} while (0) #define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ @@ -2389,5 +2384,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #endif /* !__GENERATING_BOUNDS.H */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _LINUX_MMZONE_H */ diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 23ff24080dfd..a397213bedac 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -9,703 +9,65 @@ #define LINUX_MOD_DEVICETABLE_H #ifdef __KERNEL__ -#include #include -#include -typedef unsigned long kernel_ulong_t; #endif -#define PCI_ANY_ID (~0) - -enum { - PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1, -}; - -/** - * struct pci_device_id - PCI device ID structure - * @vendor: Vendor ID to match (or PCI_ANY_ID) - * @device: Device ID to match (or PCI_ANY_ID) - * @subvendor: Subsystem vendor ID to match (or PCI_ANY_ID) - * @subdevice: Subsystem device ID to match (or PCI_ANY_ID) - * @class: Device class, subclass, and "interface" to match. - * See Appendix D of the PCI Local Bus Spec or - * include/linux/pci_ids.h for a full list of classes. - * Most drivers do not need to specify class/class_mask - * as vendor/device is normally sufficient. - * @class_mask: Limit which sub-fields of the class field are compared. - * See drivers/scsi/sym53c8xx_2/ for example of usage. - * @driver_data: Data private to the driver. - * Most drivers don't need to use driver_data field. - * Best practice is to use driver_data as an index - * into a static list of equivalent device types, - * instead of using it as a pointer. - * @override_only: Match only when dev->driver_override is this driver. - */ -struct pci_device_id { - __u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/ - __u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */ - __u32 class, class_mask; /* (class,subclass,prog-if) triplet */ - kernel_ulong_t driver_data; /* Data private to the driver */ - __u32 override_only; -}; - - -#define IEEE1394_MATCH_VENDOR_ID 0x0001 -#define IEEE1394_MATCH_MODEL_ID 0x0002 -#define IEEE1394_MATCH_SPECIFIER_ID 0x0004 -#define IEEE1394_MATCH_VERSION 0x0008 - -struct ieee1394_device_id { - __u32 match_flags; - __u32 vendor_id; - __u32 model_id; - __u32 specifier_id; - __u32 version; - kernel_ulong_t driver_data; -}; - - -/* - * Device table entry for "new style" table-driven USB drivers. - * User mode code can read these tables to choose which modules to load. - * Declare the table as a MODULE_DEVICE_TABLE. - * - * A probe() parameter will point to a matching entry from this table. - * Use the driver_info field for each match to hold information tied - * to that match: device quirks, etc. - * - * Terminate the driver's table with an all-zeroes entry. - * Use the flag values to control which fields are compared. - */ - -/** - * struct usb_device_id - identifies USB devices for probing and hotplugging - * @match_flags: Bit mask controlling which of the other fields are used to - * match against new devices. Any field except for driver_info may be - * used, although some only make sense in conjunction with other fields. - * This is usually set by a USB_DEVICE_*() macro, which sets all - * other fields in this structure except for driver_info. - * @idVendor: USB vendor ID for a device; numbers are assigned - * by the USB forum to its members. - * @idProduct: Vendor-assigned product ID. - * @bcdDevice_lo: Low end of range of vendor-assigned product version numbers. - * This is also used to identify individual product versions, for - * a range consisting of a single device. - * @bcdDevice_hi: High end of version number range. The range of product - * versions is inclusive. - * @bDeviceClass: Class of device; numbers are assigned - * by the USB forum. Products may choose to implement classes, - * or be vendor-specific. Device classes specify behavior of all - * the interfaces on a device. - * @bDeviceSubClass: Subclass of device; associated with bDeviceClass. - * @bDeviceProtocol: Protocol of device; associated with bDeviceClass. - * @bInterfaceClass: Class of interface; numbers are assigned - * by the USB forum. Products may choose to implement classes, - * or be vendor-specific. Interface classes specify behavior only - * of a given interface; other interfaces may support other classes. - * @bInterfaceSubClass: Subclass of interface; associated with bInterfaceClass. - * @bInterfaceProtocol: Protocol of interface; associated with bInterfaceClass. - * @bInterfaceNumber: Number of interface; composite devices may use - * fixed interface numbers to differentiate between vendor-specific - * interfaces. - * @driver_info: Holds information used by the driver. Usually it holds - * a pointer to a descriptor understood by the driver, or perhaps - * device flags. - * - * In most cases, drivers will create a table of device IDs by using - * USB_DEVICE(), or similar macros designed for that purpose. - * They will then export it to userspace using MODULE_DEVICE_TABLE(), - * and provide it to the USB core through their usb_driver structure. - * - * See the usb_match_id() function for information about how matches are - * performed. Briefly, you will normally use one of several macros to help - * construct these entries. Each entry you provide will either identify - * one or more specific products, or will identify a class of products - * which have agreed to behave the same. You should put the more specific - * matches towards the beginning of your table, so that driver_info can - * record quirks of specific products. - */ -struct usb_device_id { - /* which fields to match against? */ - __u16 match_flags; - - /* Used for product specific matches; range is inclusive */ - __u16 idVendor; - __u16 idProduct; - __u16 bcdDevice_lo; - __u16 bcdDevice_hi; - - /* Used for device class matches */ - __u8 bDeviceClass; - __u8 bDeviceSubClass; - __u8 bDeviceProtocol; - - /* Used for interface class matches */ - __u8 bInterfaceClass; - __u8 bInterfaceSubClass; - __u8 bInterfaceProtocol; - - /* Used for vendor-specific interface matches */ - __u8 bInterfaceNumber; - - /* not matched against */ - kernel_ulong_t driver_info - __attribute__((aligned(sizeof(kernel_ulong_t)))); -}; - -/* Some useful macros to use to create struct usb_device_id */ -#define USB_DEVICE_ID_MATCH_VENDOR 0x0001 -#define USB_DEVICE_ID_MATCH_PRODUCT 0x0002 -#define USB_DEVICE_ID_MATCH_DEV_LO 0x0004 -#define USB_DEVICE_ID_MATCH_DEV_HI 0x0008 -#define USB_DEVICE_ID_MATCH_DEV_CLASS 0x0010 -#define USB_DEVICE_ID_MATCH_DEV_SUBCLASS 0x0020 -#define USB_DEVICE_ID_MATCH_DEV_PROTOCOL 0x0040 -#define USB_DEVICE_ID_MATCH_INT_CLASS 0x0080 -#define USB_DEVICE_ID_MATCH_INT_SUBCLASS 0x0100 -#define USB_DEVICE_ID_MATCH_INT_PROTOCOL 0x0200 -#define USB_DEVICE_ID_MATCH_INT_NUMBER 0x0400 - -#define HID_ANY_ID (~0) -#define HID_BUS_ANY 0xffff -#define HID_GROUP_ANY 0x0000 - -struct hid_device_id { - __u16 bus; - __u16 group; - __u32 vendor; - __u32 product; - kernel_ulong_t driver_data; -}; - -/* s390 CCW devices */ -struct ccw_device_id { - __u16 match_flags; /* which fields to match against */ - - __u16 cu_type; /* control unit type */ - __u16 dev_type; /* device type */ - __u8 cu_model; /* control unit model */ - __u8 dev_model; /* device model */ - - kernel_ulong_t driver_info; -}; - -#define CCW_DEVICE_ID_MATCH_CU_TYPE 0x01 -#define CCW_DEVICE_ID_MATCH_CU_MODEL 0x02 -#define CCW_DEVICE_ID_MATCH_DEVICE_TYPE 0x04 -#define CCW_DEVICE_ID_MATCH_DEVICE_MODEL 0x08 - -/* s390 AP bus devices */ -struct ap_device_id { - __u16 match_flags; /* which fields to match against */ - __u8 dev_type; /* device type */ - kernel_ulong_t driver_info; -}; - -#define AP_DEVICE_ID_MATCH_CARD_TYPE 0x01 -#define AP_DEVICE_ID_MATCH_QUEUE_TYPE 0x02 - -/* s390 css bus devices (subchannels) */ -struct css_device_id { - __u8 match_flags; - __u8 type; /* subchannel type */ - kernel_ulong_t driver_data; -}; - -#define ACPI_ID_LEN 16 - -struct acpi_device_id { - __u8 id[ACPI_ID_LEN]; - kernel_ulong_t driver_data; - __u32 cls; - __u32 cls_msk; -}; - -/** - * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with - * the PCI-defined class-code information - * - * @_cls : the class, subclass, prog-if triple for this device - * @_msk : the class mask for this device - * - * This macro is used to create a struct acpi_device_id that matches a - * specific PCI class. The .id and .driver_data fields will be left - * initialized with the default value. - */ -#define ACPI_DEVICE_CLASS(_cls, _msk) .cls = (_cls), .cls_msk = (_msk), - -#define PNP_ID_LEN 8 -#define PNP_MAX_DEVICES 8 - -struct pnp_device_id { - __u8 id[PNP_ID_LEN]; - kernel_ulong_t driver_data; -}; - -struct pnp_card_device_id { - __u8 id[PNP_ID_LEN]; - kernel_ulong_t driver_data; - struct { - __u8 id[PNP_ID_LEN]; - } devs[PNP_MAX_DEVICES]; -}; - - -#define SERIO_ANY 0xff - -struct serio_device_id { - __u8 type; - __u8 extra; - __u8 id; - __u8 proto; -}; - -struct hda_device_id { - __u32 vendor_id; - __u32 rev_id; - __u8 api_version; - const char *name; - unsigned long driver_data; -}; - -struct sdw_device_id { - __u16 mfg_id; - __u16 part_id; - __u8 sdw_version; - __u8 class_id; - kernel_ulong_t driver_data; -}; - -/* - * Struct used for matching a device - */ -struct of_device_id { - char name[32]; - char type[32]; - char compatible[128]; - const void *data; -}; - -/* VIO */ -struct vio_device_id { - char type[32]; - char compat[32]; -}; - -/* PCMCIA */ - -struct pcmcia_device_id { - __u16 match_flags; - - __u16 manf_id; - __u16 card_id; - - __u8 func_id; - - /* for real multi-function devices */ - __u8 function; - - /* for pseudo multi-function devices */ - __u8 device_no; - - __u32 prod_id_hash[4]; - - /* not matched against in kernelspace */ - const char * prod_id[4]; - - /* not matched against */ - kernel_ulong_t driver_info; - char * cisfile; -}; - -#define PCMCIA_DEV_ID_MATCH_MANF_ID 0x0001 -#define PCMCIA_DEV_ID_MATCH_CARD_ID 0x0002 -#define PCMCIA_DEV_ID_MATCH_FUNC_ID 0x0004 -#define PCMCIA_DEV_ID_MATCH_FUNCTION 0x0008 -#define PCMCIA_DEV_ID_MATCH_PROD_ID1 0x0010 -#define PCMCIA_DEV_ID_MATCH_PROD_ID2 0x0020 -#define PCMCIA_DEV_ID_MATCH_PROD_ID3 0x0040 -#define PCMCIA_DEV_ID_MATCH_PROD_ID4 0x0080 -#define PCMCIA_DEV_ID_MATCH_DEVICE_NO 0x0100 -#define PCMCIA_DEV_ID_MATCH_FAKE_CIS 0x0200 -#define PCMCIA_DEV_ID_MATCH_ANONYMOUS 0x0400 - -/* Input */ -#define INPUT_DEVICE_ID_EV_MAX 0x1f -#define INPUT_DEVICE_ID_KEY_MIN_INTERESTING 0x71 -#define INPUT_DEVICE_ID_KEY_MAX 0x2ff -#define INPUT_DEVICE_ID_REL_MAX 0x0f -#define INPUT_DEVICE_ID_ABS_MAX 0x3f -#define INPUT_DEVICE_ID_MSC_MAX 0x07 -#define INPUT_DEVICE_ID_LED_MAX 0x0f -#define INPUT_DEVICE_ID_SND_MAX 0x07 -#define INPUT_DEVICE_ID_FF_MAX 0x7f -#define INPUT_DEVICE_ID_SW_MAX 0x11 -#define INPUT_DEVICE_ID_PROP_MAX 0x1f - -#define INPUT_DEVICE_ID_MATCH_BUS 1 -#define INPUT_DEVICE_ID_MATCH_VENDOR 2 -#define INPUT_DEVICE_ID_MATCH_PRODUCT 4 -#define INPUT_DEVICE_ID_MATCH_VERSION 8 - -#define INPUT_DEVICE_ID_MATCH_EVBIT 0x0010 -#define INPUT_DEVICE_ID_MATCH_KEYBIT 0x0020 -#define INPUT_DEVICE_ID_MATCH_RELBIT 0x0040 -#define INPUT_DEVICE_ID_MATCH_ABSBIT 0x0080 -#define INPUT_DEVICE_ID_MATCH_MSCIT 0x0100 -#define INPUT_DEVICE_ID_MATCH_LEDBIT 0x0200 -#define INPUT_DEVICE_ID_MATCH_SNDBIT 0x0400 -#define INPUT_DEVICE_ID_MATCH_FFBIT 0x0800 -#define INPUT_DEVICE_ID_MATCH_SWBIT 0x1000 -#define INPUT_DEVICE_ID_MATCH_PROPBIT 0x2000 - -struct input_device_id { - - kernel_ulong_t flags; - - __u16 bustype; - __u16 vendor; - __u16 product; - __u16 version; - - kernel_ulong_t evbit[INPUT_DEVICE_ID_EV_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t keybit[INPUT_DEVICE_ID_KEY_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t relbit[INPUT_DEVICE_ID_REL_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t absbit[INPUT_DEVICE_ID_ABS_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t mscbit[INPUT_DEVICE_ID_MSC_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t ledbit[INPUT_DEVICE_ID_LED_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t sndbit[INPUT_DEVICE_ID_SND_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t ffbit[INPUT_DEVICE_ID_FF_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t swbit[INPUT_DEVICE_ID_SW_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t propbit[INPUT_DEVICE_ID_PROP_MAX / BITS_PER_LONG + 1]; - - kernel_ulong_t driver_info; -}; - -/* EISA */ - -#define EISA_SIG_LEN 8 - -/* The EISA signature, in ASCII form, null terminated */ -struct eisa_device_id { - char sig[EISA_SIG_LEN]; - kernel_ulong_t driver_data; -}; - -#define EISA_DEVICE_MODALIAS_FMT "eisa:s%s" - -struct parisc_device_id { - __u8 hw_type; /* 5 bits used */ - __u8 hversion_rev; /* 4 bits */ - __u16 hversion; /* 12 bits */ - __u32 sversion; /* 20 bits */ -}; - -#define PA_HWTYPE_ANY_ID 0xff -#define PA_HVERSION_REV_ANY_ID 0xff -#define PA_HVERSION_ANY_ID 0xffff -#define PA_SVERSION_ANY_ID 0xffffffff - -/* SDIO */ - -#define SDIO_ANY_ID (~0) - -struct sdio_device_id { - __u8 class; /* Standard interface or SDIO_ANY_ID */ - __u16 vendor; /* Vendor or SDIO_ANY_ID */ - __u16 device; /* Device ID or SDIO_ANY_ID */ - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* SSB core, see drivers/ssb/ */ -struct ssb_device_id { - __u16 vendor; - __u16 coreid; - __u8 revision; - __u8 __pad; -} __attribute__((packed, aligned(2))); -#define SSB_DEVICE(_vendor, _coreid, _revision) \ - { .vendor = _vendor, .coreid = _coreid, .revision = _revision, } - -#define SSB_ANY_VENDOR 0xFFFF -#define SSB_ANY_ID 0xFFFF -#define SSB_ANY_REV 0xFF - -/* Broadcom's specific AMBA core, see drivers/bcma/ */ -struct bcma_device_id { - __u16 manuf; - __u16 id; - __u8 rev; - __u8 class; -} __attribute__((packed,aligned(2))); -#define BCMA_CORE(_manuf, _id, _rev, _class) \ - { .manuf = _manuf, .id = _id, .rev = _rev, .class = _class, } - -#define BCMA_ANY_MANUF 0xFFFF -#define BCMA_ANY_ID 0xFFFF -#define BCMA_ANY_REV 0xFF -#define BCMA_ANY_CLASS 0xFF - -struct virtio_device_id { - __u32 device; - __u32 vendor; -}; -#define VIRTIO_DEV_ANY_ID 0xffffffff - -/* - * For Hyper-V devices we use the device guid as the id. - */ -struct hv_vmbus_device_id { - guid_t guid; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* rpmsg */ - -#define RPMSG_NAME_SIZE 32 -#define RPMSG_DEVICE_MODALIAS_FMT "rpmsg:%s" - -struct rpmsg_device_id { - char name[RPMSG_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* i2c */ - -#define I2C_NAME_SIZE 20 -#define I2C_MODULE_PREFIX "i2c:" - -struct i2c_device_id { - char name[I2C_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* pci_epf */ - -#define PCI_EPF_NAME_SIZE 20 -#define PCI_EPF_MODULE_PREFIX "pci_epf:" - -struct pci_epf_device_id { - char name[PCI_EPF_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* i3c */ - -#define I3C_MATCH_DCR 0x1 -#define I3C_MATCH_MANUF 0x2 -#define I3C_MATCH_PART 0x4 -#define I3C_MATCH_EXTRA_INFO 0x8 - -struct i3c_device_id { - __u8 match_flags; - __u8 dcr; - __u16 manuf_id; - __u16 part_id; - __u16 extra_info; - - const void *data; -}; - -/* spi */ - -#define SPI_NAME_SIZE 32 -#define SPI_MODULE_PREFIX "spi:" - -struct spi_device_id { - char name[SPI_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* SLIMbus */ - -#define SLIMBUS_NAME_SIZE 32 -#define SLIMBUS_MODULE_PREFIX "slim:" - -struct slim_device_id { - __u16 manf_id, prod_code; - __u16 dev_index, instance; - - /* Data private to the driver */ - kernel_ulong_t driver_data; -}; - -#define APR_NAME_SIZE 32 -#define APR_MODULE_PREFIX "apr:" - -struct apr_device_id { - char name[APR_NAME_SIZE]; - __u32 domain_id; - __u32 svc_id; - __u32 svc_version; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -#define SPMI_NAME_SIZE 32 -#define SPMI_MODULE_PREFIX "spmi:" - -struct spmi_device_id { - char name[SPMI_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* dmi */ -enum dmi_field { - DMI_NONE, - DMI_BIOS_VENDOR, - DMI_BIOS_VERSION, - DMI_BIOS_DATE, - DMI_BIOS_RELEASE, - DMI_EC_FIRMWARE_RELEASE, - DMI_SYS_VENDOR, - DMI_PRODUCT_NAME, - DMI_PRODUCT_VERSION, - DMI_PRODUCT_SERIAL, - DMI_PRODUCT_UUID, - DMI_PRODUCT_SKU, - DMI_PRODUCT_FAMILY, - DMI_BOARD_VENDOR, - DMI_BOARD_NAME, - DMI_BOARD_VERSION, - DMI_BOARD_SERIAL, - DMI_BOARD_ASSET_TAG, - DMI_CHASSIS_VENDOR, - DMI_CHASSIS_TYPE, - DMI_CHASSIS_VERSION, - DMI_CHASSIS_SERIAL, - DMI_CHASSIS_ASSET_TAG, - DMI_STRING_MAX, - DMI_OEM_STRING, /* special case - will not be in dmi_ident */ -}; - -struct dmi_strmatch { - unsigned char slot:7; - unsigned char exact_match:1; - char substr[79]; -}; - -struct dmi_system_id { - int (*callback)(const struct dmi_system_id *); - const char *ident; - struct dmi_strmatch matches[4]; - void *driver_data; -}; -/* - * struct dmi_device_id appears during expansion of - * "MODULE_DEVICE_TABLE(dmi, x)". Compiler doesn't look inside it - * but this is enough for gcc 3.4.6 to error out: - * error: storage size of '__mod_dmi_device_table' isn't known - */ -#define dmi_device_id dmi_system_id - -#define DMI_MATCH(a, b) { .slot = a, .substr = b } -#define DMI_EXACT_MATCH(a, b) { .slot = a, .substr = b, .exact_match = 1 } - -#define PLATFORM_NAME_SIZE 24 -#define PLATFORM_MODULE_PREFIX "platform:" - -struct platform_device_id { - char name[PLATFORM_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -#define MDIO_MODULE_PREFIX "mdio:" - -#define MDIO_ID_FMT "%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u" -#define MDIO_ID_ARGS(_id) \ - ((_id)>>31) & 1, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \ - ((_id)>>27) & 1, ((_id)>>26) & 1, ((_id)>>25) & 1, ((_id)>>24) & 1, \ - ((_id)>>23) & 1, ((_id)>>22) & 1, ((_id)>>21) & 1, ((_id)>>20) & 1, \ - ((_id)>>19) & 1, ((_id)>>18) & 1, ((_id)>>17) & 1, ((_id)>>16) & 1, \ - ((_id)>>15) & 1, ((_id)>>14) & 1, ((_id)>>13) & 1, ((_id)>>12) & 1, \ - ((_id)>>11) & 1, ((_id)>>10) & 1, ((_id)>>9) & 1, ((_id)>>8) & 1, \ - ((_id)>>7) & 1, ((_id)>>6) & 1, ((_id)>>5) & 1, ((_id)>>4) & 1, \ - ((_id)>>3) & 1, ((_id)>>2) & 1, ((_id)>>1) & 1, (_id) & 1 - -/** - * struct mdio_device_id - identifies PHY devices on an MDIO/MII bus - * @phy_id: The result of - * (mdio_read(&MII_PHYSID1) << 16 | mdio_read(&MII_PHYSID2)) & @phy_id_mask - * for this PHY type - * @phy_id_mask: Defines the significant bits of @phy_id. A value of 0 - * is used to terminate an array of struct mdio_device_id. - */ -struct mdio_device_id { - __u32 phy_id; - __u32 phy_id_mask; -}; - -struct zorro_device_id { - __u32 id; /* Device ID or ZORRO_WILDCARD */ - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -#define ZORRO_WILDCARD (0xffffffff) /* not official */ - -#define ZORRO_DEVICE_MODALIAS_FMT "zorro:i%08X" - -#define ISAPNP_ANY_ID 0xffff -struct isapnp_device_id { - unsigned short card_vendor, card_device; - unsigned short vendor, function; - kernel_ulong_t driver_data; /* data private to the driver */ -}; - -/** - * struct amba_id - identifies a device on an AMBA bus - * @id: The significant bits if the hardware device ID - * @mask: Bitmask specifying which bits of the id field are significant when - * matching. A driver binds to a device when ((hardware device ID) & mask) - * == id. - * @data: Private data used by the driver. - */ -struct amba_id { - unsigned int id; - unsigned int mask; - void *data; -}; - -/** - * struct mips_cdmm_device_id - identifies devices in MIPS CDMM bus - * @type: Device type identifier. - */ -struct mips_cdmm_device_id { - __u8 type; -}; - -/* - * Match x86 CPUs for CPU specific drivers. - * See documentation of "x86_match_cpu" for details. - */ - -/* - * MODULE_DEVICE_TABLE expects this struct to be called x86cpu_device_id. - * Although gcc seems to ignore this error, clang fails without this define. - */ -#define x86cpu_device_id x86_cpu_id -struct x86_cpu_id { - __u16 vendor; - __u16 family; - __u16 model; - __u16 steppings; - __u16 feature; /* bit index */ - /* Solely for kernel-internal use: DO NOT EXPORT to userspace! */ - __u16 flags; - __u8 platform_mask; - __u8 type; - kernel_ulong_t driver_data; -}; - -/* Wild cards for x86_cpu_id::vendor, family, model and feature */ -#define X86_VENDOR_ANY 0xffff -#define X86_FAMILY_ANY 0 -#define X86_MODEL_ANY 0 -#define X86_STEPPING_ANY 0 -#define X86_STEP_MIN 0 -#define X86_STEP_MAX 0xf -#define X86_PLATFORM_ANY 0x0 -#define X86_FEATURE_ANY 0 /* Same as FPU, you can't test for that */ -#define X86_CPU_TYPE_ANY 0 +#include "device-id/acpi.h" +#include "device-id/amba.h" +#include "device-id/ap.h" +#include "device-id/apr.h" +#include "device-id/auxiliary.h" +#include "device-id/bcma.h" +#include "device-id/ccw.h" +#include "device-id/cdx.h" +#include "device-id/coreboot.h" +#include "device-id/css.h" +#include "device-id/dfl.h" +#include "device-id/dmi.h" +#include "device-id/eisa.h" +#include "device-id/fsl_mc.h" +#include "device-id/hda.h" +#include "device-id/hid.h" +#include "device-id/hv_vmbus.h" +#include "device-id/i2c.h" +#include "device-id/i3c.h" +#include "device-id/ieee1394.h" +#include "device-id/input.h" +#include "device-id/ipack.h" +#include "device-id/isapnp.h" +#include "device-id/ishtp.h" +#include "device-id/mcb.h" +#include "device-id/mdio.h" +#include "device-id/mei_cl.h" +#include "device-id/mhi.h" +#include "device-id/mips_cdmm.h" +#include "device-id/of.h" +#include "device-id/parisc.h" +#include "device-id/pci.h" +#include "device-id/pcmcia.h" +#include "device-id/platform.h" +#include "device-id/pnp.h" +#include "device-id/rio.h" +#include "device-id/rpmsg.h" +#include "device-id/sdio.h" +#include "device-id/sdw.h" +#include "device-id/serio.h" +#include "device-id/slim.h" +#include "device-id/spi.h" +#include "device-id/spmi.h" +#include "device-id/ssam.h" +#include "device-id/ssb.h" +#include "device-id/tb.h" +#include "device-id/tee_client.h" +#include "device-id/typec.h" +#include "device-id/ulpi.h" +#include "device-id/usb.h" +#include "device-id/vchiq.h" +#include "device-id/vio.h" +#include "device-id/virtio.h" +#include "device-id/wmi.h" +#include "device-id/x86_cpu.h" +#include "device-id/zorro.h" /* * Generic table type for matching CPU features. @@ -716,265 +78,4 @@ struct cpu_feature { __u16 feature; }; -#define IPACK_ANY_FORMAT 0xff -#define IPACK_ANY_ID (~0) -struct ipack_device_id { - __u8 format; /* Format version or IPACK_ANY_ID */ - __u32 vendor; /* Vendor ID or IPACK_ANY_ID */ - __u32 device; /* Device ID or IPACK_ANY_ID */ -}; - -#define MEI_CL_MODULE_PREFIX "mei:" -#define MEI_CL_NAME_SIZE 32 -#define MEI_CL_VERSION_ANY 0xff - -/** - * struct mei_cl_device_id - MEI client device identifier - * @name: helper name - * @uuid: client uuid - * @version: client protocol version - * @driver_info: information used by the driver. - * - * identifies mei client device by uuid and name - */ -struct mei_cl_device_id { - char name[MEI_CL_NAME_SIZE]; - uuid_le uuid; - __u8 version; - kernel_ulong_t driver_info; -}; - -/* RapidIO */ - -#define RIO_ANY_ID 0xffff - -/** - * struct rio_device_id - RIO device identifier - * @did: RapidIO device ID - * @vid: RapidIO vendor ID - * @asm_did: RapidIO assembly device ID - * @asm_vid: RapidIO assembly vendor ID - * - * Identifies a RapidIO device based on both the device/vendor IDs and - * the assembly device/vendor IDs. - */ -struct rio_device_id { - __u16 did, vid; - __u16 asm_did, asm_vid; -}; - -struct mcb_device_id { - __u16 device; - kernel_ulong_t driver_data; -}; - -struct ulpi_device_id { - __u16 vendor; - __u16 product; - kernel_ulong_t driver_data; -}; - -/** - * struct fsl_mc_device_id - MC object device identifier - * @vendor: vendor ID - * @obj_type: MC object type - * - * Type of entries in the "device Id" table for MC object devices supported by - * a MC object device driver. The last entry of the table has vendor set to 0x0 - */ -struct fsl_mc_device_id { - __u16 vendor; - const char obj_type[16]; -}; - -/** - * struct tb_service_id - Thunderbolt service identifiers - * @match_flags: Flags used to match the structure - * @protocol_key: Protocol key the service supports - * @protocol_id: Protocol id the service supports - * @protocol_version: Version of the protocol - * @protocol_revision: Revision of the protocol software - * @driver_data: Driver specific data - * - * Thunderbolt XDomain services are exposed as devices where each device - * carries the protocol information the service supports. Thunderbolt - * XDomain service drivers match against that information. - */ -struct tb_service_id { - __u32 match_flags; - char protocol_key[8 + 1]; - __u32 protocol_id; - __u32 protocol_version; - __u32 protocol_revision; - kernel_ulong_t driver_data; -}; - -#define TBSVC_MATCH_PROTOCOL_KEY 0x0001 -#define TBSVC_MATCH_PROTOCOL_ID 0x0002 -#define TBSVC_MATCH_PROTOCOL_VERSION 0x0004 -#define TBSVC_MATCH_PROTOCOL_REVISION 0x0008 - -/* USB Type-C Alternate Modes */ - -#define TYPEC_ANY_MODE 0x7 - -/** - * struct typec_device_id - USB Type-C alternate mode identifiers - * @svid: Standard or Vendor ID - * @mode: Mode index - * @driver_data: Driver specific data - */ -struct typec_device_id { - __u16 svid; - __u8 mode; - kernel_ulong_t driver_data; -}; - -/** - * struct tee_client_device_id - tee based device identifier - * @uuid: For TEE based client devices we use the device uuid as - * the identifier. - */ -struct tee_client_device_id { - uuid_t uuid; -}; - -/* WMI */ - -#define WMI_MODULE_PREFIX "wmi:" - -/** - * struct wmi_device_id - WMI device identifier - * @guid_string: 36 char string of the form fa50ff2b-f2e8-45de-83fa-65417f2f49ba - * @context: pointer to driver specific data - */ -struct wmi_device_id { - const char guid_string[UUID_STRING_LEN+1]; - const void *context; -}; - -#define MHI_DEVICE_MODALIAS_FMT "mhi:%s" -#define MHI_NAME_SIZE 32 - -#define MHI_EP_DEVICE_MODALIAS_FMT "mhi_ep:%s" - -/** - * struct mhi_device_id - MHI device identification - * @chan: MHI channel name - * @driver_data: driver data; - */ -struct mhi_device_id { - const char chan[MHI_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -#define AUXILIARY_NAME_SIZE 40 -#define AUXILIARY_MODULE_PREFIX "auxiliary:" - -struct auxiliary_device_id { - char name[AUXILIARY_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* Surface System Aggregator Module */ - -#define SSAM_MATCH_TARGET 0x1 -#define SSAM_MATCH_INSTANCE 0x2 -#define SSAM_MATCH_FUNCTION 0x4 - -struct ssam_device_id { - __u8 match_flags; - - __u8 domain; - __u8 category; - __u8 target; - __u8 instance; - __u8 function; - - kernel_ulong_t driver_data; -}; - -/* - * DFL (Device Feature List) - * - * DFL defines a linked list of feature headers within the device MMIO space to - * provide an extensible way of adding features. Software can walk through these - * predefined data structures to enumerate features. It is now used in the FPGA. - * See Documentation/fpga/dfl.rst for more information. - * - * The dfl bus type is introduced to match the individual feature devices (dfl - * devices) for specific dfl drivers. - */ - -/** - * struct dfl_device_id - dfl device identifier - * @type: DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - __u16 type; - __u16 feature_id; - kernel_ulong_t driver_data; -}; - -/* ISHTP (Integrated Sensor Hub Transport Protocol) */ - -#define ISHTP_MODULE_PREFIX "ishtp:" - -/** - * struct ishtp_device_id - ISHTP device identifier - * @guid: GUID of the device. - * @driver_data: pointer to driver specific data - */ -struct ishtp_device_id { - guid_t guid; - kernel_ulong_t driver_data; -}; - -#define CDX_ANY_ID (0xFFFF) - -enum { - CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1, -}; - -/** - * struct cdx_device_id - CDX device identifier - * @vendor: Vendor ID - * @device: Device ID - * @subvendor: Subsystem vendor ID (or CDX_ANY_ID) - * @subdevice: Subsystem device ID (or CDX_ANY_ID) - * @class: Device class - * Most drivers do not need to specify class/class_mask - * as vendor/device is normally sufficient. - * @class_mask: Limit which sub-fields of the class field are compared. - * @override_only: Match only when dev->driver_override is this driver. - * - * Type of entries in the "device Id" table for CDX devices supported by - * a CDX device driver. - */ -struct cdx_device_id { - __u16 vendor; - __u16 device; - __u16 subvendor; - __u16 subdevice; - __u32 class; - __u32 class_mask; - __u32 override_only; -}; - -struct vchiq_device_id { - char name[32]; -}; - -/** - * struct coreboot_device_id - Identifies a coreboot table entry - * @tag: tag ID - * @driver_data: driver specific data - */ -struct coreboot_device_id { - __u32 tag; - kernel_ulong_t driver_data; -}; - #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/include/linux/mtd/nand-qpic-common.h b/include/linux/mtd/nand-qpic-common.h index e8201d1b7cf9..006ca8c978a9 100644 --- a/include/linux/mtd/nand-qpic-common.h +++ b/include/linux/mtd/nand-qpic-common.h @@ -9,6 +9,8 @@ #ifndef __MTD_NAND_QPIC_COMMON_H__ #define __MTD_NAND_QPIC_COMMON_H__ +#include + /* NANDc reg offsets */ #define NAND_FLASH_CMD 0x00 #define NAND_ADDR0 0x04 @@ -394,7 +396,7 @@ struct qcom_nand_controller { const struct qcom_nandc_props *props; - struct nand_controller *controller; + struct nand_controller controller; struct qpic_spi_nand *qspi; struct list_head host_list; diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index cdcfe0fd2e7d..4b92494827b1 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -21,8 +21,8 @@ /* Flash opcodes. */ #define SPINOR_OP_WRDI 0x04 /* Write disable */ #define SPINOR_OP_WREN 0x06 /* Write enable */ -#define SPINOR_OP_RDSR 0x05 /* Read status register */ -#define SPINOR_OP_WRSR 0x01 /* Write status register 1 byte */ +#define SPINOR_OP_RDSR 0x05 /* Read status register 1 */ +#define SPINOR_OP_WRSR 0x01 /* Write status register 1 */ #define SPINOR_OP_RDSR2 0x3f /* Read status register 2 */ #define SPINOR_OP_WRSR2 0x3e /* Write status register 2 */ #define SPINOR_OP_READ 0x03 /* Read data bytes (low frequency) */ @@ -125,6 +125,7 @@ #define SR2_LB1 BIT(3) /* Security Register Lock Bit 1 */ #define SR2_LB2 BIT(4) /* Security Register Lock Bit 2 */ #define SR2_LB3 BIT(5) /* Security Register Lock Bit 3 */ +#define SR2_CMP_BIT6 BIT(6) #define SR2_QUAD_EN_BIT7 BIT(7) /* Supported SPI protocols */ @@ -371,6 +372,7 @@ struct spi_nor_flash_parameter; * @reg_proto: the SPI protocol for read_reg/write_reg/erase operations * @sfdp: the SFDP data of the flash * @debugfs_root: pointer to the debugfs directory + * @dfs_sr_cache: Status Register cached value for debugfs use only * @controller_ops: SPI NOR controller driver specific operations. * @params: [FLASH-SPECIFIC] SPI NOR flash parameters and settings. * The structure includes legacy flash parameters and @@ -409,6 +411,7 @@ struct spi_nor { enum spi_nor_cmd_ext cmd_ext_type; struct sfdp *sfdp; struct dentry *debugfs_root; + u8 dfs_sr_cache[2]; const struct spi_nor_controller_ops *controller_ops; diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index f53f5f0499b4..5f4c00ae72a7 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -437,6 +437,7 @@ extern const struct spinand_manufacturer esmt_c8_spinand_manufacturer; extern const struct spinand_manufacturer fmsh_spinand_manufacturer; extern const struct spinand_manufacturer foresee_spinand_manufacturer; extern const struct spinand_manufacturer gigadevice_spinand_manufacturer; +extern const struct spinand_manufacturer heyangtek_spinand_manufacturer; extern const struct spinand_manufacturer macronix_spinand_manufacturer; extern const struct spinand_manufacturer micron_spinand_manufacturer; extern const struct spinand_manufacturer paragon_spinand_manufacturer; @@ -583,6 +584,7 @@ enum spinand_bus_interface { * @op_variants.read_cache: variants of the read-cache operation * @op_variants.write_cache: variants of the write-cache operation * @op_variants.update_cache: variants of the update-cache operation + * @op_variants.cont_read_cache: variants of the continuous read-cache operation * @vendor_ops: vendor specific operations * @select_target: function used to select a target/die. Required only for * multi-die chips @@ -592,6 +594,7 @@ enum spinand_bus_interface { * @user_otp: SPI NAND user OTP info. * @read_retries: the number of read retry modes supported * @set_read_retry: enable/disable read retry for data recovery + * @set_randomizer: enable/disable randomizer support * * Each SPI NAND manufacturer driver should have a spinand_info table * describing all the chips supported by the driver. @@ -607,6 +610,7 @@ struct spinand_info { const struct spinand_op_variants *read_cache; const struct spinand_op_variants *write_cache; const struct spinand_op_variants *update_cache; + const struct spinand_op_variants *cont_read_cache; } op_variants; const struct spinand_op_variants *vendor_ops; int (*select_target)(struct spinand_device *spinand, @@ -620,6 +624,8 @@ struct spinand_info { unsigned int read_retries; int (*set_read_retry)(struct spinand_device *spinand, unsigned int read_retry); + int (*set_randomizer)(struct spinand_device *spinand, + bool enable); }; #define SPINAND_ID(__method, ...) \ @@ -636,6 +642,14 @@ struct spinand_info { .update_cache = __update, \ } +#define SPINAND_INFO_OP_VARIANTS_WITH_CONT(__read, __write, __update, __cont_read) \ + { \ + .read_cache = __read, \ + .write_cache = __write, \ + .update_cache = __update, \ + .cont_read_cache = __cont_read, \ + } + #define SPINAND_INFO_VENDOR_OPS(__ops) \ .vendor_ops = __ops @@ -676,6 +690,9 @@ struct spinand_info { .read_retries = __read_retries, \ .set_read_retry = __set_read_retry +#define SPINAND_RANDOMIZER(__set_randomizer) \ + .set_randomizer = __set_randomizer + #define SPINAND_INFO(__model, __id, __memorg, __eccreq, __op_variants, \ __flags, ...) \ { \ @@ -707,6 +724,7 @@ struct spinand_dirmap { * @read_cache: read cache op template * @write_cache: write cache op template * @update_cache: update cache op template + * @cont_read_cache: continuous read cache op template (optional) */ struct spinand_mem_ops { struct spi_mem_op reset; @@ -721,6 +739,7 @@ struct spinand_mem_ops { const struct spi_mem_op *read_cache; const struct spi_mem_op *write_cache; const struct spi_mem_op *update_cache; + const struct spi_mem_op *cont_read_cache; }; /** @@ -759,6 +778,7 @@ struct spinand_mem_ops { * @user_otp: SPI NAND user OTP info. * @read_retries: the number of read retry modes supported * @set_read_retry: Enable/disable the read retry feature + * @set_randomizer: Enable/disable the randomizer feature */ struct spinand_device { struct nand_device base; @@ -792,6 +812,8 @@ struct spinand_device { bool cont_read_possible; int (*set_cont_read)(struct spinand_device *spinand, bool enable); + int (*set_randomizer)(struct spinand_device *spinand, + bool enable); const struct spinand_fact_otp *fact_otp; const struct spinand_user_otp *user_otp; diff --git a/include/linux/net.h b/include/linux/net.h index f268f395ce47..3d82966e2243 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -47,6 +47,29 @@ typedef struct sockopt { int optlen; } sockopt_t; +/* + * Initialize a user-backed sockopt_t from the (optval, optlen) __user pair of + * a getsockopt() callback. Used by transitional __user getsockopt wrappers + * while the proto-layer callbacks are converted to take a sockopt_t; the + * caller writes opt->optlen back to the user optlen after the callback. + */ +static inline int sockopt_init_user(sockopt_t *opt, char __user *optval, + int __user *optlen) +{ + int len; + + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + iov_iter_ubuf(&opt->iter_out, ITER_DEST, optval, len); + iov_iter_ubuf(&opt->iter_in, ITER_SOURCE, optval, len); + opt->optlen = len; + + return 0; +} + struct poll_table_struct; struct pipe_inode_info; struct inode; @@ -285,6 +308,7 @@ int sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags); struct file *sock_alloc_file(struct socket *sock, int flags, const char *dname); struct socket *sockfd_lookup(int fd, int *err); struct socket *sock_from_file(struct file *file); +int sock_read_xattr(struct socket *sock, const char *name, void *value, size_t size); #define sockfd_put(sock) fput(sock->file) int net_ratelimit(void); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index b67a12541eac..987c3c87f14f 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1131,6 +1131,9 @@ struct netdev_net_notifier { * netdev_hw_addr_list_for_each(ha, uc). Return 0 on success or a * negative errno to request a retry via the core backoff. * + * void (*ndo_work)(struct net_device *dev, unsigned long events); + * Run deferred work scheduled with netdev_work_sched(@events). + * * int (*ndo_set_mac_address)(struct net_device *dev, void *addr); * This function is called when the Media Access Control address * needs to be changed. If this interface is not defined, the @@ -1460,6 +1463,8 @@ struct net_device_ops { struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc); + void (*ndo_work)(struct net_device *dev, + unsigned long events); int (*ndo_set_mac_address)(struct net_device *dev, void *addr); int (*ndo_validate_addr)(struct net_device *dev); @@ -1930,8 +1935,11 @@ enum netdev_reg_state { * has been enabled due to the need to listen to * additional unicast addresses in a device that * does not implement ndo_set_rx_mode() - * @rx_mode_node: List entry for rx_mode work processing - * @rx_mode_tracker: Refcount tracker for rx_mode work + * @work_node: List entry for async netdev_work processing + * @work_tracker: Refcount tracker for async netdev_work + * @work_pending: Driver-defined pending netdev_work, passed to + * ndo_work() (see netdev_work_sched()) + * @work_core_pending: Core-defined pending netdev_work (NETDEV_WORK_*) * @rx_mode_addr_cache: Recycled snapshot entries for rx_mode work * @rx_mode_retry_timer: Timer that re-queues rx_mode work after failure * @rx_mode_retry_count: Number of consecutive retries already scheduled @@ -2134,6 +2142,8 @@ enum netdev_reg_state { * * FIXME: cleanup struct net_device such that network protocol info * moves out. + * + * @netdev_trace_buffer_list: Linked list for debugging refcount leak. */ struct net_device { @@ -2290,6 +2300,9 @@ struct net_device { #if IS_ENABLED(CONFIG_TLS_DEVICE) const struct tlsdev_ops *tlsdev_ops; #endif +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + struct list_head netdev_trace_buffer_list; +#endif unsigned int operstate; unsigned char link_mode; @@ -2326,8 +2339,10 @@ struct net_device { unsigned int promiscuity; unsigned int allmulti; bool uc_promisc; - struct list_head rx_mode_node; - netdevice_tracker rx_mode_tracker; + struct list_head work_node; + netdevice_tracker work_tracker; + unsigned long work_pending; + unsigned long work_core_pending; struct netdev_hw_addr_list rx_mode_addr_cache; struct timer_list rx_mode_retry_timer; unsigned int rx_mode_retry_count; @@ -3238,6 +3253,7 @@ enum netdev_cmd { NETDEV_OFFLOAD_XSTATS_REPORT_USED, NETDEV_OFFLOAD_XSTATS_REPORT_DELTA, NETDEV_XDP_FEAT_CHANGE, + NETDEV_DEBUG_UNREGISTER, }; const char *netdev_cmd_to_name(enum netdev_cmd cmd); @@ -4459,9 +4475,15 @@ static inline bool dev_nit_active(const struct net_device *dev) void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev); +void save_netdev_trace_buffer(struct net_device *dev, int delta); +int trim_netdev_trace(unsigned long *entries, int nr_entries); + static inline void __dev_put(struct net_device *dev) { if (dev) { +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + save_netdev_trace_buffer(dev, -1); +#endif #ifdef CONFIG_PCPU_DEV_REFCNT this_cpu_dec(*dev->pcpu_refcnt); #else @@ -4473,6 +4495,9 @@ static inline void __dev_put(struct net_device *dev) static inline void __dev_hold(struct net_device *dev) { if (dev) { +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + save_netdev_trace_buffer(dev, 1); +#endif #ifdef CONFIG_PCPU_DEV_REFCNT this_cpu_inc(*dev->pcpu_refcnt); #else @@ -5176,6 +5201,9 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, const struct pcpu_sw_netstats __percpu *netstats); void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); +void netdev_work_sched(struct net_device *dev, unsigned long events); +unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask); + enum { NESTED_SYNC_IMM_BIT, NESTED_SYNC_TODO_BIT, diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index efbbfa770d66..e99afc1414cd 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -93,6 +93,7 @@ enum nf_hook_ops_type { NF_HOOK_OP_NF_TABLES, NF_HOOK_OP_BPF, NF_HOOK_OP_NFT_FT, + NF_HOOK_OP_NAT, }; struct nf_hook_ops { @@ -140,6 +141,12 @@ struct nf_hook_entries { */ }; +struct nf_nat_lookup_hook_priv { + struct nf_hook_entries __rcu *entries; + + struct rcu_head rcu_head; +}; + #ifdef CONFIG_NETFILTER static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e) { diff --git a/include/linux/netfilter/nf_conntrack_h323.h b/include/linux/netfilter/nf_conntrack_h323.h index 81286c499325..b15f37604cde 100644 --- a/include/linux/netfilter/nf_conntrack_h323.h +++ b/include/linux/netfilter/nf_conntrack_h323.h @@ -9,8 +9,6 @@ #include #include -#define RAS_PORT 1719 -#define Q931_PORT 1720 #define H323_RTP_CHANNEL_MAX 4 /* Audio, video, FAX and other */ /* This structure exists only once per master */ diff --git a/include/linux/netfilter/nf_conntrack_pptp.h b/include/linux/netfilter/nf_conntrack_pptp.h index c3bdb4370938..c0b305ce7c3c 100644 --- a/include/linux/netfilter/nf_conntrack_pptp.h +++ b/include/linux/netfilter/nf_conntrack_pptp.h @@ -50,8 +50,6 @@ struct nf_nat_pptp { __be16 pac_call_id; /* NAT'ed PAC call id */ }; -#define PPTP_CONTROL_PORT 1723 - #define PPTP_PACKET_CONTROL 1 #define PPTP_PACKET_MGMT 2 diff --git a/include/linux/netfilter/nf_conntrack_sane.h b/include/linux/netfilter/nf_conntrack_sane.h index 46c7acd1b4a7..8501035d7335 100644 --- a/include/linux/netfilter/nf_conntrack_sane.h +++ b/include/linux/netfilter/nf_conntrack_sane.h @@ -3,8 +3,6 @@ #define _NF_CONNTRACK_SANE_H /* SANE tracking. */ -#define SANE_PORT 6566 - enum sane_state { SANE_STATE_NORMAL, SANE_STATE_START_REQUESTED, diff --git a/include/linux/netfilter/nf_conntrack_tftp.h b/include/linux/netfilter/nf_conntrack_tftp.h index 90b334bbce3c..e3d1739c557d 100644 --- a/include/linux/netfilter/nf_conntrack_tftp.h +++ b/include/linux/netfilter/nf_conntrack_tftp.h @@ -2,8 +2,6 @@ #ifndef _NF_CONNTRACK_TFTP_H #define _NF_CONNTRACK_TFTP_H -#define TFTP_PORT 69 - #include #include #include diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h index 20d70dddbe50..25062f4a0dd5 100644 --- a/include/linux/netfilter/x_tables.h +++ b/include/linux/netfilter/x_tables.h @@ -18,7 +18,7 @@ * @match: the match extension * @target: the target extension * @matchinfo: per-match data - * @targetinfo: per-target data + * @targinfo: per-target data * @state: pointer to hook state this packet came from * @fragoff: packet is a fragment, this is the data offset * @thoff: position of transport header relative to skb->data @@ -77,7 +77,9 @@ static inline u_int8_t xt_family(const struct xt_action_param *par) * @match: struct xt_match through which this function was invoked * @matchinfo: per-match data * @hook_mask: via which hooks the new rule is reachable - * Other fields as above. + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) + * @nft_compat: running from the nft compat layer if true */ struct xt_mtchk_param { struct net *net; @@ -91,8 +93,13 @@ struct xt_mtchk_param { }; /** - * struct xt_mdtor_param - match destructor parameters - * Fields as above. + * struct xt_mtdtor_param - match destructor parameters + * + * @net: network namespace through which the check was invoked + * @match: struct xt_match through which this function was invoked + * @matchinfo: per-match data + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) */ struct xt_mtdtor_param { struct net *net; @@ -105,10 +112,16 @@ struct xt_mtdtor_param { * struct xt_tgchk_param - parameters for target extensions' * checkentry functions * + * @net: network namespace through which the check was invoked + * @table: table the rule is tried to be inserted into * @entryinfo: the family-specific rule data * (struct ipt_entry, ip6t_entry, arpt_entry, ebt_entry) - * - * Other fields see above. + * @target: the target extension + * @targinfo: per-target data + * @hook_mask: via which hooks the new rule is reachable + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) + * @nft_compat: running from the nft compat layer if true */ struct xt_tgchk_param { struct net *net; @@ -336,9 +349,9 @@ struct xt_table_info *xt_alloc_table_info(unsigned int size); void xt_free_table_info(struct xt_table_info *info); /** - * xt_recseq - recursive seqcount for netfilter use + * var xt_recseq - recursive seqcount for netfilter use * - * Packet processing changes the seqcount only if no recursion happened + * Packet processing changes the seqcount only if no recursion happened. * get_counters() can use read_seqcount_begin()/read_seqcount_retry(), * because we use the normal seqcount convention : * Low order bit set to 1 if a writer is active. diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 243c0f737938..1bc120d61c5b 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -61,14 +61,16 @@ struct netfs_inode { #if IS_ENABLED(CONFIG_FSCACHE) struct fscache_cookie *cache; #endif - struct mutex wb_lock; /* Writeback serialisation */ + struct list_head wb_queue; /* Queue of processes wanting to do writeback */ loff_t _remote_i_size; /* Size of the remote file */ loff_t _zero_point; /* Size after which we assume there's no data * on the server */ + spinlock_t lock; /* Lock covering wb_queue */ atomic_t io_count; /* Number of outstanding reqs */ unsigned long flags; #define NETFS_ICTX_ODIRECT 0 /* The file has DIO in progress */ #define NETFS_ICTX_UNBUFFERED 1 /* I/O should not use the pagecache */ +#define NETFS_ICTX_WB_LOCK 2 /* Writeback serialisation lock */ #define NETFS_ICTX_MODIFIED_ATTR 3 /* Indicate change in mtime/ctime */ #define NETFS_ICTX_SINGLE_NO_UPLOAD 4 /* Monolithic payload, cache but no upload */ }; @@ -462,6 +464,10 @@ int netfs_alloc_folioq_buffer(struct address_space *mapping, size_t *_cur_size, ssize_t size, gfp_t gfp); void netfs_free_folioq_buffer(struct folio_queue *fq); +/* Writeback exclusion API. */ +bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait); +void netfs_wb_end(struct netfs_inode *ictx); + /** * netfs_inode - Get the netfs inode context from the inode * @inode: The inode to query @@ -743,7 +749,8 @@ static inline void netfs_inode_init(struct netfs_inode *ctx, #if IS_ENABLED(CONFIG_FSCACHE) ctx->cache = NULL; #endif - mutex_init(&ctx->wb_lock); + INIT_LIST_HEAD(&ctx->wb_queue); + spin_lock_init(&ctx->lock); /* ->releasepage() drives zero_point */ if (use_zero_point) { ctx->_zero_point = ctx->_remote_i_size; @@ -753,7 +760,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx, /** * netfs_resize_file - Note that a file got resized - * @ctx: The netfs inode being resized + * @ictx: The netfs inode being resized * @new_i_size: The new file size * @changed_on_server: The change was applied to the server * diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 4623262da3c0..ec17e602c979 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -145,11 +145,6 @@ struct nfs4_xattr_cache; * nfs fs inode data in memory */ struct nfs_inode { - /* - * The 64bit 'inode number' - */ - __u64 fileid; - /* * NFS file handle */ @@ -394,16 +389,6 @@ static inline int NFS_STALE(const struct inode *inode) return test_bit(NFS_INO_STALE, &NFS_I(inode)->flags); } -static inline __u64 NFS_FILEID(const struct inode *inode) -{ - return NFS_I(inode)->fileid; -} - -static inline void set_nfs_fileid(struct inode *inode, __u64 fileid) -{ - NFS_I(inode)->fileid = fileid; -} - static inline void nfs_mark_for_revalidate(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); @@ -473,7 +458,6 @@ extern void nfs_file_set_open_context(struct file *filp, struct nfs_open_context extern void nfs_file_clear_open_context(struct file *flip); extern struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx); extern void nfs_put_lock_context(struct nfs_lock_context *l_ctx); -extern u64 nfs_compat_user_ino64(u64 fileid); extern void nfs_fattr_init(struct nfs_fattr *fattr); extern void nfs_fattr_set_barrier(struct nfs_fattr *fattr); extern unsigned long nfs_inc_attr_generation_counter(void); @@ -668,15 +652,6 @@ static inline loff_t nfs_size_to_loff_t(__u64 size) return min_t(u64, size, OFFSET_MAX); } -static inline ino_t -nfs_fileid_to_ino_t(u64 fileid) -{ - ino_t ino = (ino_t) fileid; - if (sizeof(ino_t) < sizeof(u64)) - ino ^= fileid >> (sizeof(u64)-sizeof(ino_t)) * 8; - return ino; -} - static inline void nfs_ooo_clear(struct nfs_inode *nfsi) { nfsi->cache_validity &= ~NFS_INO_DATA_INVAL_DEFER; diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index afe1d8f09d89..4b9a35dbc062 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -33,6 +33,7 @@ enum { PG_TEARDOWN, /* page group sync for destroy */ PG_UNLOCKPAGE, /* page group sync bit in read path */ PG_UPTODATE, /* page group sync bit in read path */ + PG_READ_FAILED, /* page group saw a read error */ PG_WB_END, /* page group sync bit in write path */ PG_REMOVE, /* page group sync bit in write path */ PG_CONTENDED1, /* Is someone waiting for a lock? */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 35ea18a40b66..11c5b31cfc7d 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -582,7 +582,6 @@ struct nfs_lock_args { struct nfs_lowner lock_owner; unsigned char block : 1; unsigned char reclaim : 1; - unsigned char new_lock : 1; unsigned char new_lock_owner : 1; }; diff --git a/include/linux/nmi.h b/include/linux/nmi.h index bc1162895f35..89c5465e2524 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -77,9 +77,9 @@ static inline void reset_hung_task_detector(void) { } * detectors are 'suspended' while 'watchdog_thresh' is equal zero. */ #define WATCHDOG_HARDLOCKUP_ENABLED_BIT 0 -#define WATCHDOG_SOFTOCKUP_ENABLED_BIT 1 +#define WATCHDOG_SOFTLOCKUP_ENABLED_BIT 1 #define WATCHDOG_HARDLOCKUP_ENABLED (1 << WATCHDOG_HARDLOCKUP_ENABLED_BIT) -#define WATCHDOG_SOFTOCKUP_ENABLED (1 << WATCHDOG_SOFTOCKUP_ENABLED_BIT) +#define WATCHDOG_SOFTLOCKUP_ENABLED (1 << WATCHDOG_SOFTLOCKUP_ENABLED_BIT) #if defined(CONFIG_HARDLOCKUP_DETECTOR) extern void hardlockup_detector_disable(void); diff --git a/include/linux/ntb.h b/include/linux/ntb.h index 879c3e89e026..ae72caa03859 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -567,7 +567,7 @@ void ntb_msg_event(struct ntb_dev *ntb); int ntb_default_port_number(struct ntb_dev *ntb); /** - * ntb_default_port_count() - get the default number of peer device ports + * ntb_default_peer_port_count() - get the default number of peer device ports * @ntb: NTB device context. * * By default hardware driver supports just one peer device. @@ -582,7 +582,7 @@ int ntb_default_peer_port_count(struct ntb_dev *ntb); /** * ntb_default_peer_port_number() - get the default peer port by given index * @ntb: NTB device context. - * @idx: Peer port index (should not differ from zero). + * @pidx: Peer port index (should not differ from zero). * * By default hardware driver supports just one peer device, so this method * shall return the corresponding value from enum ntb_default_port. diff --git a/include/linux/of.h b/include/linux/of.h index 20e4f752d5b6..b920aac6b975 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/linux/of_platform.h b/include/linux/of_platform.h index 17471ef8e092..181c438a9b0a 100644 --- a/include/linux/of_platform.h +++ b/include/linux/of_platform.h @@ -6,7 +6,9 @@ * */ -#include +#include +#include +#include struct device; struct device_node; diff --git a/include/linux/oom.h b/include/linux/oom.h index 7b02bc1d0a7e..00da05d227e6 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -30,7 +30,7 @@ struct oom_control { struct zonelist *zonelist; /* Used to determine mempolicy */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* Memory cgroup in which oom is invoked, or NULL for global oom */ struct mem_cgroup *memcg; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7223f6f4e2b4..7a863572adce 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -879,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped) #define PG_head_mask ((1UL << PG_head)) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * PageTransCompound returns true for both transparent huge pages - * and hugetlbfs pages, so it should only be called when it's known - * that hugetlbfs pages aren't involved. - */ -static inline int PageTransCompound(const struct page *page) -{ - return PageCompound(page); -} -#else -TESTPAGEFLAG_FALSE(TransCompound, transcompound) -#endif - #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) /* * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index 61e876e255e8..f23d4b218da0 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -120,14 +120,18 @@ struct page_ext_iter { * page_ext_iter_begin() - Prepare for iterating through page extensions. * @iter: page extension iterator. * @pfn: PFN of the page we're interested in. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no page_ext exists for this page. */ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, - unsigned long pfn) + unsigned long pfn, unsigned long count) { + if (!count) + return NULL; + iter->index = 0; iter->start_pfn = pfn; iter->page_ext = page_ext_lookup(pfn); @@ -138,19 +142,22 @@ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, /** * page_ext_iter_next() - Get next page extension * @iter: page extension iterator. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no next page_ext exists. */ -static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter) +static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter, + unsigned long count) { unsigned long pfn; if (WARN_ON_ONCE(!iter->page_ext)) return NULL; - iter->index++; + if (++iter->index >= count) + return NULL; pfn = iter->start_pfn + iter->index; if (page_ext_iter_next_fast_possible(pfn)) @@ -183,9 +190,9 @@ static inline struct page_ext *page_ext_iter_get(const struct page_ext_iter *ite * IMPORTANT: must be called with RCU read lock taken. */ #define for_each_page_ext(__page, __pgcount, __page_ext, __iter) \ - for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page));\ - __page_ext && __iter.index < __pgcount; \ - __page_ext = page_ext_iter_next(&__iter)) + for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page), __pgcount); \ + __page_ext; \ + __page_ext = page_ext_iter_next(&__iter, __pgcount)) #else /* !CONFIG_PAGE_EXTENSION */ struct page_ext; diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 3328357f6dba..9fe51dfccf26 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -3,6 +3,7 @@ #define __LINUX_PAGE_OWNER_H #include +#include #ifdef CONFIG_PAGE_OWNER extern struct static_key_false page_owner_inited; @@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page, extern void __split_page_owner(struct page *page, int old_order, int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); -extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason); +extern void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason); extern void __dump_page_owner(const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone); @@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) if (static_branch_unlikely(&page_owner_inited)) __folio_copy_owner(newfolio, old); } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { if (static_branch_unlikely(&page_owner_inited)) __folio_set_owner_migrate_reason(folio, reason); @@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order, static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) { } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { } static inline void dump_page_owner(const struct page *page) diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h index 9d4ca5c218a0..272b1274efdc 100644 --- a/include/linux/page_reporting.h +++ b/include/linux/page_reporting.h @@ -5,7 +5,6 @@ #include #include -/* This value should always be a power of 2, see page_reporting_cycle() */ #define PAGE_REPORTING_CAPACITY 32 #define PAGE_REPORTING_ORDER_UNSPECIFIED -1 @@ -22,6 +21,9 @@ struct page_reporting_dev_info { /* Minimal order of page reporting */ unsigned int order; + + /* Max pages per report batch; 0 (default) means PAGE_REPORTING_CAPACITY */ + unsigned int capacity; }; /* Tear-down and bring-up for page reporting devices */ diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 1f50991b43e3..2c3718d592d6 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -513,39 +513,37 @@ static inline bool mapping_large_folio_support(const struct address_space *mappi return mapping_max_folio_order(mapping) > 0; } -/* Return the maximum folio size for this pagecache mapping, in bytes. */ -static inline size_t mapping_max_folio_size(const struct address_space *mapping) +/** + * mapping_pmd_folio_support() - Check if a mapping supports PMD-sized folio + * @mapping: The address_space + * + * While some mappings support large folios, they might not support PMD-sized + * folios. This function checks whether a mapping supports PMD-sized folios. + * For example, khugepaged needs this information before attempting to + * collapsing THPs. + * + * Return: True if PMD-sized folios are supported, otherwise false. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool mapping_pmd_folio_support(const struct address_space *mapping) { - return PAGE_SIZE << mapping_max_folio_order(mapping); -} + /* AS_FOLIO_ORDER is only reasonable for pagecache folios */ + VM_WARN_ON_ONCE((unsigned long)mapping & FOLIO_MAPPING_ANON); -static inline int filemap_nr_thps(const struct address_space *mapping) -{ -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - return atomic_read(&mapping->nr_thps); -#else - return 0; -#endif + return mapping_min_folio_order(mapping) <= PMD_ORDER && + mapping_max_folio_order(mapping) >= PMD_ORDER; } - -static inline void filemap_nr_thps_inc(struct address_space *mapping) -{ -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - if (!mapping_large_folio_support(mapping)) - atomic_inc(&mapping->nr_thps); #else - WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0); -#endif +static inline bool mapping_pmd_folio_support(const struct address_space *mapping) +{ + return false; } +#endif -static inline void filemap_nr_thps_dec(struct address_space *mapping) +/* Return the maximum folio size for this pagecache mapping, in bytes. */ +static inline size_t mapping_max_folio_size(const struct address_space *mapping) { -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - if (!mapping_large_folio_support(mapping)) - atomic_dec(&mapping->nr_thps); -#else - WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0); -#endif + return PAGE_SIZE << mapping_max_folio_order(mapping); } struct address_space *folio_mapping(const struct folio *folio); @@ -1350,7 +1348,6 @@ struct readahead_control { struct file_ra_state *ra; /* private: use the readahead_* accessors instead */ pgoff_t _index; - pgoff_t _max_index; /* limit readahead to _max_index, inclusive */ unsigned int _nr_pages; unsigned int _batch_count; bool dropbehind; @@ -1364,7 +1361,6 @@ struct readahead_control { .mapping = m, \ .ra = r, \ ._index = i, \ - ._max_index = ULONG_MAX, \ } #define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 1eca1264815b..f247cf9bcf1a 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -61,6 +61,47 @@ struct pci_epc_map { void __iomem *virt_addr; }; +/** + * enum pci_epc_aux_resource_type - auxiliary resource type identifiers + * @PCI_EPC_AUX_DOORBELL_MMIO: Doorbell MMIO, that might be outside the DMA + * controller register window + * + * EPC backends may expose auxiliary blocks (e.g. DMA engines) by mapping their + * register windows and descriptor memories into BAR space. This enum + * identifies the type of each exposable resource. + */ +enum pci_epc_aux_resource_type { + PCI_EPC_AUX_DOORBELL_MMIO, +}; + +/** + * struct pci_epc_aux_resource - a physical auxiliary resource that may be + * exposed for peer use + * @type: resource type, see enum pci_epc_aux_resource_type + * @phys_addr: physical base address of the resource + * @size: size of the resource in bytes + * @bar: BAR number where this resource is already exposed to the RC + * (NO_BAR if not) + * @bar_offset: offset within @bar where the resource starts (valid iff + * @bar != NO_BAR) + * @u: type-specific metadata + */ +struct pci_epc_aux_resource { + enum pci_epc_aux_resource_type type; + phys_addr_t phys_addr; + resource_size_t size; + enum pci_barno bar; + resource_size_t bar_offset; + + union { + /* PCI_EPC_AUX_DOORBELL_MMIO */ + struct { + int irq; /* IRQ number for the doorbell handler */ + u32 data; /* write value to ring the doorbell */ + } db_mmio; + } u; +}; + /** * struct pci_epc_ops - set of function pointers for performing EPC operations * @write_header: ops to populate configuration space header @@ -84,6 +125,9 @@ struct pci_epc_map { * @start: ops to start the PCI link * @stop: ops to stop the PCI link * @get_features: ops to get the features supported by the EPC + * @get_aux_resources_count: ops to get the number of controller-owned + * auxiliary resources + * @get_aux_resources: ops to retrieve controller-owned auxiliary resources * @owner: the module owner containing the ops */ struct pci_epc_ops { @@ -115,6 +159,11 @@ struct pci_epc_ops { void (*stop)(struct pci_epc *epc); const struct pci_epc_features* (*get_features)(struct pci_epc *epc, u8 func_no, u8 vfunc_no); + int (*get_aux_resources_count)(struct pci_epc *epc, u8 func_no, + u8 vfunc_no); + int (*get_aux_resources)(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_aux_resource *resources, + int num_resources); struct module *owner; }; @@ -343,6 +392,11 @@ int pci_epc_start(struct pci_epc *epc); void pci_epc_stop(struct pci_epc *epc); const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no); +int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no, + u8 vfunc_no); +int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_aux_resource *resources, + int num_resources); enum pci_barno pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features); enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 7737a7c03260..704e1dc8b30a 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include @@ -152,14 +152,41 @@ struct pci_epf_bar { struct pci_epf_bar_submap *submap; }; +enum pci_epf_doorbell_type { + PCI_EPF_DOORBELL_MSI = 0, + PCI_EPF_DOORBELL_EMBEDDED, +}; + /** * struct pci_epf_doorbell_msg - represents doorbell message - * @msg: MSI message - * @virq: IRQ number of this doorbell MSI message + * @msg: Doorbell address/data pair to be mapped into BAR space. + * For MSI-backed doorbells this is the MSI message, while for + * "embedded" doorbells this represents an MMIO write that asserts + * an interrupt on the EP side. + * @virq: IRQ number of this doorbell message + * @irq_flags: Required flags for request_irq()/request_threaded_irq(). + * Callers may OR-in additional flags (e.g. IRQF_ONESHOT). + * @type: Doorbell type. + * @bar: BAR number where the doorbell target is already exposed to the RC + * (NO_BAR if not) + * @offset: offset within @bar for the doorbell target (valid iff + * @bar != NO_BAR) + * @iova_base: Internal: base DMA address returned by dma_map_resource() for the + * embedded doorbell MMIO window (used only for unmapping). Valid + * when @type is PCI_EPF_DOORBELL_EMBEDDED and @iova_size is + * non-zero. + * @iova_size: Internal: size of the dma_map_resource() mapping at @iova_base. + * Zero when no mapping was created (e.g. pre-exposed fixed BAR). */ struct pci_epf_doorbell_msg { struct msi_msg msg; int virq; + unsigned long irq_flags; + enum pci_epf_doorbell_type type; + enum pci_barno bar; + resource_size_t offset; + dma_addr_t iova_base; + size_t iova_size; }; /** diff --git a/include/linux/pci.h b/include/linux/pci.h index 2c4454583c11..64b308b6e61c 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -24,9 +24,10 @@ #define LINUX_PCI_H #include -#include +#include #include +#include #include #include #include @@ -508,15 +509,13 @@ struct pci_dev { unsigned int no_command_memory:1; /* No PCI_COMMAND_MEMORY */ unsigned int rom_bar_overlap:1; /* ROM BAR disable broken */ unsigned int rom_attr_enabled:1; /* Display of ROM attribute enabled? */ - unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ + unsigned int non_mappable_bars:1; /* BARs can't be mapped by CPU or peers */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ struct hlist_head saved_cap_space; - struct bin_attribute *res_attr[DEVICE_COUNT_RESOURCE]; /* sysfs file for resources */ - struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */ #ifdef CONFIG_HOTPLUG_PCI_PCIE unsigned int broken_cmd_compl:1; /* No compl for some cmds */ @@ -636,6 +635,7 @@ struct pci_host_bridge { int domain_nr; struct list_head windows; /* resource_entry */ struct list_head dma_ranges; /* dma ranges resource list */ + struct list_head ports; /* Root Port list (pci_host_port) */ #ifdef CONFIG_PCI_IDE u16 nr_ide_streams; /* Max streams possibly active in @ide_stream_ida */ struct ida ide_stream_ida; @@ -660,6 +660,8 @@ struct pci_host_bridge { unsigned int preserve_config:1; /* Preserve FW resource setup */ unsigned int size_windows:1; /* Enable root bus sizing */ unsigned int msi_domain:1; /* Bridge wants MSI domain */ + unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during + system suspend is broken */ /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev, @@ -727,8 +729,6 @@ struct pci_bus { pci_bus_flags_t bus_flags; /* Inherited by child buses */ struct device *bridge; struct device dev; - struct bin_attribute *legacy_io; /* Legacy I/O for this bus */ - struct bin_attribute *legacy_mem; /* Legacy mem */ unsigned int is_added:1; unsigned int unsafe_warn:1; /* warned about RW1C config write */ unsigned int flit_mode:1; /* Link in Flit mode */ @@ -1171,6 +1171,10 @@ enum { /* These external functions are only available when PCI support is enabled */ #ifdef CONFIG_PCI +/* PCI legacy I/O port and memory address space sizes. */ +#define PCI_LEGACY_IO_SIZE (SZ_64K - 1) +#define PCI_LEGACY_MEM_SIZE SZ_1M + extern unsigned int pci_flags; static inline void pci_set_flags(int flags) { pci_flags = flags; } @@ -2086,6 +2090,8 @@ pci_release_mem_regions(struct pci_dev *pdev) pci_select_bars(pdev, IORESOURCE_MEM)); } +bool pci_suspend_retains_context(struct pci_dev *pdev); + #else /* CONFIG_PCI is not enabled */ static inline void pci_set_flags(int flags) { } @@ -2244,6 +2250,11 @@ pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, static inline void pci_free_irq_vectors(struct pci_dev *dev) { } + +static inline bool pci_suspend_retains_context(struct pci_dev *pdev) +{ + return true; +} #endif /* CONFIG_PCI */ /* Include architecture-dependent settings and functions */ @@ -2300,6 +2311,31 @@ int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma); CONCATENATE(__pci_dev_for_each_res, COUNT_ARGS(__VA_ARGS__)) \ (dev, res, __VA_ARGS__) +/** + * pci_resource_is_io - check if a PCI resource is of I/O port type. + * @dev: PCI device to check. + * @resno: The resource number (BAR index) to check. + * + * Returns true if the resource type is I/O port. + */ +static inline bool pci_resource_is_io(const struct pci_dev *dev, int resno) +{ + return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_IO; +} + +/** + * pci_resource_is_mem - check if a PCI resource is of memory type. + * @dev: PCI device to check. + * @resno: The resource number (BAR index) to check. + * + * Returns true if the resource type is memory, including + * prefetchable memory. + */ +static inline bool pci_resource_is_mem(const struct pci_dev *dev, int resno) +{ + return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_MEM; +} + /* * Similar to the helpers above, these manipulate per-pci_dev * driver-specific data. They are really just a wrapper around @@ -2506,11 +2542,6 @@ int pcibios_alloc_irq(struct pci_dev *dev); void pcibios_free_irq(struct pci_dev *dev); resource_size_t pcibios_default_alignment(void); -#if !defined(HAVE_PCI_MMAP) && !defined(ARCH_GENERIC_PCI_MMAP_RESOURCE) -extern int pci_create_resource_files(struct pci_dev *dev); -extern void pci_remove_resource_files(struct pci_dev *dev); -#endif - #if defined(CONFIG_PCI_MMCONFIG) || defined(CONFIG_ACPI_MCFG) void __init pci_mmcfg_early_init(void); void __init pci_mmcfg_late_init(void); @@ -2540,7 +2571,7 @@ int pci_vfs_assigned(struct pci_dev *dev); int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs); int pci_sriov_get_totalvfs(struct pci_dev *dev); int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn); -resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno); +resource_size_t pci_iov_resource_size(const struct pci_dev *dev, int resno); int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size); u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs); void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe); @@ -2548,7 +2579,8 @@ void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe); /* Arch may override these (weak) */ int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs); int pcibios_sriov_disable(struct pci_dev *pdev); -resource_size_t pcibios_iov_resource_alignment(struct pci_dev *dev, int resno); +resource_size_t pcibios_iov_resource_alignment(const struct pci_dev *dev, + int resno); #else static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id) { @@ -2593,7 +2625,8 @@ static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) static inline int pci_sriov_get_totalvfs(struct pci_dev *dev) { return 0; } #define pci_sriov_configure_simple NULL -static inline resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) +static inline resource_size_t pci_iov_resource_size(const struct pci_dev *dev, + int resno) { return 0; } static inline int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size) { return -ENODEV; } diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 24cb42f66e4b..1c9d40e09107 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2732,6 +2732,9 @@ #define PCI_DEVICE_ID_INTEL_82815_MC 0x1130 #define PCI_DEVICE_ID_INTEL_82815_CGC 0x1132 #define PCI_DEVICE_ID_INTEL_SST_TNG 0x119a +#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb +#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 +#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 #define PCI_DEVICE_ID_INTEL_82092AA_0 0x1221 #define PCI_DEVICE_ID_INTEL_82437 0x122d #define PCI_DEVICE_ID_INTEL_82371FB_0 0x122e @@ -3052,6 +3055,11 @@ #define PCI_DEVICE_ID_INTEL_5400_FBD1 0x4036 #define PCI_DEVICE_ID_INTEL_HDA_TGL_H 0x43c8 #define PCI_DEVICE_ID_INTEL_HDA_DG1 0x490d +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 #define PCI_DEVICE_ID_INTEL_HDA_EHL_0 0x4b55 #define PCI_DEVICE_ID_INTEL_HDA_EHL_3 0x4b58 #define PCI_DEVICE_ID_INTEL_HDA_WCL 0x4d28 diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d73a1c08c3e3..1e3212e2c827 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -12,8 +12,8 @@ * start shutting down you call percpu_ref_kill() _before_ dropping the initial * refcount. * - * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less - * than an atomic_t - this is because of the way shutdown works, see + * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less + * than an atomic_long_t - this is because of the way shutdown works, see * percpu_ref_kill()/PERCPU_COUNT_BIAS. * * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the @@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) /** * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the * caller is responsible for taking RCU. + * @ref: percpu_ref to try-get * * This function is safe to call as long as @ref is between init and exit. */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea..5842552294c1 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1791,22 +1791,8 @@ static inline int perf_is_paranoid(void) } extern int perf_allow_kernel(void); - -static inline int perf_allow_cpu(void) -{ - if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) - return -EACCES; - - return security_perf_event_open(PERF_SECURITY_CPU); -} - -static inline int perf_allow_tracepoint(void) -{ - if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) - return -EPERM; - - return security_perf_event_open(PERF_SECURITY_TRACEPOINT); -} +extern int perf_allow_cpu(void); +extern int perf_allow_tracepoint(void); extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs); @@ -2023,6 +2009,19 @@ perf_event_pause(struct perf_event *event, bool reset) { return 0; } static inline int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) { return 0; } +static inline int perf_allow_kernel(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_cpu(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_tracepoint(void) +{ + return perfmon_capable() ? 0 : -EPERM; +} + #endif /* !CONFIG_PERF_EVENTS */ #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/include/linux/pfn.h b/include/linux/pfn.h index b90ca0b6c331..cfedf0f61bb3 100644 --- a/include/linux/pfn.h +++ b/include/linux/pfn.h @@ -2,7 +2,7 @@ #ifndef _LINUX_PFN_H_ #define _LINUX_PFN_H_ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #endif diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 2981e386da7b..e38f069c1c91 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -8,7 +8,7 @@ #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_MMU #include @@ -988,49 +988,30 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - -#ifndef clear_not_present_full_ptes /** - * clear_not_present_full_ptes - Clear multiple not present PTEs which are - * consecutive in the pgtable. + * clear_nonpresent_ptes - Clear multiple non-present PTEs which are + * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. - * @full: Whether we are clearing a full mm. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear_not_present_full(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */ -static inline void clear_not_present_full_ptes(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, unsigned int nr, int full) +static inline void clear_nonpresent_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) { + (void)addr; + for (;;) { - pte_clear_not_present_full(mm, addr, ptep, full); + pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; } } -#endif #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, @@ -1858,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; @@ -2320,7 +2301,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) #ifdef CONFIG_PHYS_ADDR_T_64BIT diff --git a/include/linux/phy.h b/include/linux/phy.h index 199a7aaa341b..fc680901275b 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/platform_data/apds990x.h b/include/linux/platform_data/apds990x.h deleted file mode 100644 index 37684f68c04f..000000000000 --- a/include/linux/platform_data/apds990x.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * This file is part of the APDS990x sensor driver. - * Chip is combined proximity and ambient light sensor. - * - * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). - * - * Contact: Samu Onkalo - */ - -#ifndef __APDS990X_H__ -#define __APDS990X_H__ - - -#define APDS_IRLED_CURR_12mA 0x3 -#define APDS_IRLED_CURR_25mA 0x2 -#define APDS_IRLED_CURR_50mA 0x1 -#define APDS_IRLED_CURR_100mA 0x0 - -/** - * struct apds990x_chip_factors - defines effect of the cover window - * @ga: Total glass attenuation - * @cf1: clear channel factor 1 for raw to lux conversion - * @irf1: IR channel factor 1 for raw to lux conversion - * @cf2: clear channel factor 2 for raw to lux conversion - * @irf2: IR channel factor 2 for raw to lux conversion - * @df: device factor for conversion formulas - * - * Structure for tuning ALS calculation to match with environment. - * Values depend on the material above the sensor and the sensor - * itself. If the GA is zero, driver will use uncovered sensor default values - * format: decimal value * APDS_PARAM_SCALE except df which is plain integer. - */ -struct apds990x_chip_factors { - int ga; - int cf1; - int irf1; - int cf2; - int irf2; - int df; -}; -#define APDS_PARAM_SCALE 4096 - -/** - * struct apds990x_platform_data - platform data for apsd990x.c driver - * @cf: chip factor data - * @pdrive: IR-led driving current - * @ppcount: number of IR pulses used for proximity estimation - * @setup_resources: interrupt line setup call back function - * @release_resources: interrupt line release call back function - * - * Proximity detection result depends heavily on correct ppcount, pdrive - * and cover window. - * - */ - -struct apds990x_platform_data { - struct apds990x_chip_factors cf; - u8 pdrive; - u8 ppcount; - int (*setup_resources)(void); - int (*release_resources)(void); -}; - -#endif diff --git a/include/linux/platform_data/dma-iop32x.h b/include/linux/platform_data/dma-iop32x.h deleted file mode 100644 index ac83cff89549..000000000000 --- a/include/linux/platform_data/dma-iop32x.h +++ /dev/null @@ -1,110 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright © 2006, Intel Corporation. - */ -#ifndef IOP_ADMA_H -#define IOP_ADMA_H -#include -#include -#include - -#define IOP_ADMA_SLOT_SIZE 32 -#define IOP_ADMA_THRESHOLD 4 -#ifdef DEBUG -#define IOP_PARANOIA 1 -#else -#define IOP_PARANOIA 0 -#endif -#define iop_paranoia(x) BUG_ON(IOP_PARANOIA && (x)) - -#define DMA0_ID 0 -#define DMA1_ID 1 -#define AAU_ID 2 - -/** - * struct iop_adma_device - internal representation of an ADMA device - * @pdev: Platform device - * @id: HW ADMA Device selector - * @dma_desc_pool: base of DMA descriptor region (DMA address) - * @dma_desc_pool_virt: base of DMA descriptor region (CPU address) - * @common: embedded struct dma_device - */ -struct iop_adma_device { - struct platform_device *pdev; - int id; - dma_addr_t dma_desc_pool; - void *dma_desc_pool_virt; - struct dma_device common; -}; - -/** - * struct iop_adma_chan - internal representation of an ADMA device - * @pending: allows batching of hardware operations - * @lock: serializes enqueue/dequeue operations to the slot pool - * @mmr_base: memory mapped register base - * @chain: device chain view of the descriptors - * @device: parent device - * @common: common dmaengine channel object members - * @last_used: place holder for allocation to continue from where it left off - * @all_slots: complete domain of slots usable by the channel - * @slots_allocated: records the actual size of the descriptor slot pool - * @irq_tasklet: bottom half where iop_adma_slot_cleanup runs - */ -struct iop_adma_chan { - int pending; - spinlock_t lock; /* protects the descriptor slot pool */ - void __iomem *mmr_base; - struct list_head chain; - struct iop_adma_device *device; - struct dma_chan common; - struct iop_adma_desc_slot *last_used; - struct list_head all_slots; - int slots_allocated; - struct tasklet_struct irq_tasklet; -}; - -/** - * struct iop_adma_desc_slot - IOP-ADMA software descriptor - * @slot_node: node on the iop_adma_chan.all_slots list - * @chain_node: node on the op_adma_chan.chain list - * @hw_desc: virtual address of the hardware descriptor chain - * @phys: hardware address of the hardware descriptor chain - * @group_head: first operation in a transaction - * @slot_cnt: total slots used in an transaction (group of operations) - * @slots_per_op: number of slots per operation - * @idx: pool index - * @tx_list: list of descriptors that are associated with one operation - * @async_tx: support for the async_tx api - * @group_list: list of slots that make up a multi-descriptor transaction - * for example transfer lengths larger than the supported hw max - * @xor_check_result: result of zero sum - * @crc32_result: result crc calculation - */ -struct iop_adma_desc_slot { - struct list_head slot_node; - struct list_head chain_node; - void *hw_desc; - struct iop_adma_desc_slot *group_head; - u16 slot_cnt; - u16 slots_per_op; - u16 idx; - struct list_head tx_list; - struct dma_async_tx_descriptor async_tx; - union { - u32 *xor_check_result; - u32 *crc32_result; - u32 *pq_check_result; - }; -}; - -struct iop_adma_platform_data { - int hw_id; - dma_cap_mask_t cap_mask; - size_t pool_size; -}; - -#define to_iop_sw_desc(addr_hw_desc) \ - container_of(addr_hw_desc, struct iop_adma_desc_slot, hw_desc) -#define iop_hw_desc_slot_idx(hw_desc, idx) \ - ( (void *) (((unsigned long) hw_desc) + ((idx) << 5)) ) -#endif diff --git a/include/linux/platform_data/pxa2xx_udc.h b/include/linux/platform_data/pxa2xx_udc.h index bc99cc6a3c5f..c1e4d03bae2c 100644 --- a/include/linux/platform_data/pxa2xx_udc.h +++ b/include/linux/platform_data/pxa2xx_udc.h @@ -10,21 +10,6 @@ #ifndef PXA2XX_UDC_H #define PXA2XX_UDC_H -struct pxa2xx_udc_mach_info { - int (*udc_is_connected)(void); /* do we see host? */ - void (*udc_command)(int cmd); -#define PXA2XX_UDC_CMD_CONNECT 0 /* let host see us */ -#define PXA2XX_UDC_CMD_DISCONNECT 1 /* so host won't see us */ - - /* Boards following the design guidelines in the developer's manual, - * with on-chip GPIOs not Lubbock's weird hardware, can have a sane - * VBUS IRQ and omit the methods above. Store the GPIO number - * here. Note that sometimes the signals go through inverters... - */ - bool gpio_pullup_inverted; - int gpio_pullup; /* high == pullup activated */ -}; - #ifdef CONFIG_PXA27x extern void pxa27x_clear_otgph(void); #else diff --git a/include/linux/platform_data/usb-xhci-prom21.h b/include/linux/platform_data/usb-xhci-prom21.h new file mode 100644 index 000000000000..ee672ad452a8 --- /dev/null +++ b/include/linux/platform_data/usb-xhci-prom21.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD Promontory 21 xHCI auxiliary device platform data. + * + * Copyright (C) 2026 Jihong Min + */ + +#ifndef _LINUX_PLATFORM_DATA_USB_XHCI_PROM21_H +#define _LINUX_PLATFORM_DATA_USB_XHCI_PROM21_H + +#include +#include + +struct pci_dev; + +struct prom21_xhci_pdata { + struct pci_dev *pdev; + void __iomem *regs; + resource_size_t rsrc_len; +}; + +#endif diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 554f41b827e1..b5ed8c83ace1 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -147,6 +147,13 @@ #define ASUS_WMI_DEVID_GPU_MUX 0x00090016 #define ASUS_WMI_DEVID_GPU_MUX_VIVO 0x00090026 +/* Keystone dongle insert/remove state. + * PRESENCE_BIT (0x00010000) encodes insert state: + * 0x00010000 = inserted, 0x00000000 = absent. STATUS_BIT is never set. + * 0xFFFFFFFE means no keystone slot on this machine. + */ +#define ASUS_WMI_DEVID_KEYSTONE 0x00120091 + /* TUF laptop RGB modes/colours */ #define ASUS_WMI_DEVID_TUF_RGB_MODE 0x00100056 #define ASUS_WMI_DEVID_TUF_RGB_MODE2 0x0010005A @@ -196,6 +203,7 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval); int asus_hid_register_listener(struct asus_hid_listener *cdev); void asus_hid_unregister_listener(struct asus_hid_listener *cdev); int asus_hid_event(enum asus_hid_event event); +bool asus_wmi_custom_fan_curve_is_enabled(void); #else static inline void set_ally_mcu_hack(enum asus_ally_mcu_hack status) { @@ -227,6 +235,10 @@ static inline int asus_hid_event(enum asus_hid_event event) { return -ENODEV; } +static inline bool asus_wmi_custom_fan_curve_is_enabled(void) +{ + return false; +} #endif #endif /* __PLATFORM_DATA_X86_ASUS_WMI_H */ diff --git a/include/linux/platform_data/x86/int3472.h b/include/linux/platform_data/x86/int3472.h index 93f1e1fe09b4..a73841dfae27 100644 --- a/include/linux/platform_data/x86/int3472.h +++ b/include/linux/platform_data/x86/int3472.h @@ -13,6 +13,7 @@ #include #include #include +#include #include /* FIXME drop this once the I2C_DEV_NAME_FORMAT macro has been added to include/linux/i2c.h */ @@ -72,7 +73,6 @@ container_of(clk, struct int3472_discrete_device, clock) struct acpi_device; -struct dmi_system_id; struct i2c_client; struct platform_device; diff --git a/include/linux/platform_data/x86/soc.h b/include/linux/platform_data/x86/soc.h index f981907a5cb0..a6a6b313dfa7 100644 --- a/include/linux/platform_data/x86/soc.h +++ b/include/linux/platform_data/x86/soc.h @@ -12,7 +12,7 @@ #if IS_ENABLED(CONFIG_X86) -#include +#include #include diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 26e6a43358e2..8c566f09d04e 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -11,6 +11,7 @@ #define _PLATFORM_DEVICE_H_ #include +#include #define PLATFORM_DEVID_NONE (-1) #define PLATFORM_DEVID_AUTO (-2) @@ -18,7 +19,6 @@ struct irq_affinity; struct mfd_cell; struct property_entry; -struct platform_device_id; struct platform_device { const char *name; diff --git a/include/linux/pnp.h b/include/linux/pnp.h index 23fe3eaf242d..e0c0d17eb7d8 100644 --- a/include/linux/pnp.h +++ b/include/linux/pnp.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #define PNP_NAME_LEN 50 diff --git a/include/linux/power/max17042_battery.h b/include/linux/power/max17042_battery.h index d5b08313cf11..13aeab1597c6 100644 --- a/include/linux/power/max17042_battery.h +++ b/include/linux/power/max17042_battery.h @@ -24,6 +24,8 @@ #define MAX17042_CHARACTERIZATION_DATA_SIZE 48 +#define MAX17055_MODELCFG_REFRESH_BIT BIT(15) + enum max17042_register { MAX17042_STATUS = 0x00, MAX17042_VALRT_Th = 0x01, @@ -198,16 +200,6 @@ enum max170xx_chip_type { MAXIM_DEVICE_TYPE_NUM }; -/* - * used for setting a register to a desired value - * addr : address for a register - * data : setting value for the register - */ -struct max17042_reg_data { - u8 addr; - u16 data; -}; - struct max17042_config_data { /* External current sense resistor value in milli-ohms */ u32 cur_sense_val; @@ -229,6 +221,7 @@ struct max17042_config_data { u16 full_soc_thresh; /* 0x13 */ u16 design_cap; /* 0x18 */ u16 ichgt_term; /* 0x1E */ + u16 model_cfg; /* 0xDB */ /* MG3 config */ u16 at_rate; /* 0x04 */ @@ -265,23 +258,4 @@ struct max17042_config_data { u16 cell_char_tbl[MAX17042_CHARACTERIZATION_DATA_SIZE]; } __packed; -struct max17042_platform_data { - struct max17042_reg_data *init_data; - struct max17042_config_data *config_data; - int num_init_data; /* Number of enties in init_data array */ - bool enable_current_sense; - bool enable_por_init; /* Use POR init from Maxim appnote */ - - /* - * R_sns in micro-ohms. - * default 10000 (if r_sns = 0) as it is the recommended value by - * the datasheet although it can be changed by board designers. - */ - unsigned int r_sns; - int vmin; /* in millivolts */ - int vmax; /* in millivolts */ - int temp_min; /* in tenths of degree Celsius */ - int temp_max; /* in tenths of degree Celsius */ -}; - #endif /* __MAX17042_BATTERY_H_ */ diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h index 9f088c9023b1..00b840970d56 100644 --- a/include/linux/pps_kernel.h +++ b/include/linux/pps_kernel.h @@ -60,6 +60,7 @@ struct pps_device { struct device dev; struct fasync_struct *async_queue; /* fasync method */ spinlock_t lock; + bool kc_removed; }; /* diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index ce16bbc0b308..03a79786df1d 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -129,6 +129,7 @@ enum sev_cmd { SEV_CMD_SNP_LAUNCH_FINISH = 0x0A2, SEV_CMD_SNP_DBG_DECRYPT = 0x0B0, SEV_CMD_SNP_DBG_ENCRYPT = 0x0B1, + SEV_CMD_SNP_VERIFY_MITIGATION = 0x0B2, SEV_CMD_SNP_PAGE_SWAP_OUT = 0x0C0, SEV_CMD_SNP_PAGE_SWAP_IN = 0x0C1, SEV_CMD_SNP_PAGE_MOVE = 0x0C2, @@ -898,10 +899,60 @@ struct snp_feature_info { #define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3) #define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4) #define SNP_CXL_ALLOW_POLICY_SUPPORTED BIT(5) +#define SNP_VERIFY_MITIGATION_SUPPORTED BIT(13) /* Feature bits in EBX */ #define SNP_SEV_TIO_SUPPORTED BIT(1) +#define SNP_MIT_SUBCMD_REQ_STATUS 0x0 +#define SNP_MIT_SUBCMD_REQ_VERIFY 0x1 + +/** + * struct sev_data_snp_verify_mitigation - SNP_VERIFY_MITIGATION command params + * + * @length: Length of the command buffer read by the PSP + * @subcommand: Mitigation sub-command for the firmware to execute. + * REQ_STATUS: 0x0 - Request status about currently supported and + * verified mitigations + * REQ_VERIFY: 0x1 - Request to initiate verification mitigation + * operation on a specific mitigation + * @rsvd: Reserved + * @vector: Bit specifying the vulnerability mitigation to process + * @dst_paddr_en: Destination paddr enabled + * @src_paddr_en: Source paddr enabled + * @rsvd1: Reserved + * @rsvd2: Reserved + * @src_paddr: Source address for optional input data + * @dst_paddr: Destination address to write the result + * @rsvd3: Reserved + */ +struct sev_data_snp_verify_mitigation { + u32 length; + u16 subcommand; + u16 rsvd; + u64 vector; + u32 dst_paddr_en : 1, + src_paddr_en : 1, + rsvd1 : 30; + u8 rsvd2[4]; + u64 src_paddr; + u64 dst_paddr; + u8 rsvd3[24]; +} __packed; + +/** + * struct sev_data_snp_verify_mitigation_dst - mitigation result vectors + * + * @mit_verified_vector: Bit vector of vulnerability mitigations verified + * @mit_supported_vector: Bit vector of vulnerability mitigations supported + * @mit_failure_status: Status of the verification operation + */ +struct sev_data_snp_verify_mitigation_dst { + u64 mit_verified_vector; /* OUT */ + u64 mit_supported_vector; /* OUT */ + u32 mit_failure_status; /* OUT */ +} __packed; + /** * struct sev_snp_tcb_version_genoa_milan * diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index eae67015ce51..057edc4cbb6e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -111,7 +111,7 @@ struct radix_tree_iter { }; /** - * Radix-tree synchronization + * DOC: Radix-tree synchronization * * The radix-tree API requires that users provide all synchronisation (with * specific exceptions, noted below). @@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot) /** * radix_tree_deref_slot_protected - dereference a slot with tree lock held * @slot: slot pointer, returned by radix_tree_lookup_slot + * @treelock: caller must hold this spinlock * * Similar to radix_tree_deref_slot. The caller does not hold the RCU read * lock but it must hold the tree lock to prevent parallel updates. @@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags); /** diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 2467b3be15c9..b3bf9339cd86 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -1,206 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* -*- linux-c -*- ------------------------------------------------------- * +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved * - * Copyright 2003 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -#ifndef LINUX_RAID_RAID6_H -#define LINUX_RAID_RAID6_H - -#ifdef __KERNEL__ - -#include -#include - -/* This should be const but the raid6 code is too convoluted for that. */ -static inline void *raid6_get_zero_page(void) -{ - return page_address(ZERO_PAGE(0)); -} - -#else /* ! __KERNEL__ */ -/* Used for testing in user space */ - -#include -#include -#include -#include -#include -#include -#include - -/* Not standard, but glibc defines it */ -#define BITS_PER_LONG __WORDSIZE - -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -#ifndef PAGE_SIZE -# define PAGE_SIZE 4096 -#endif -#ifndef PAGE_SHIFT -# define PAGE_SHIFT 12 -#endif -extern const char raid6_empty_zero_page[PAGE_SIZE]; - -#define __init -#define __exit -#ifndef __attribute_const__ -# define __attribute_const__ __attribute__((const)) -#endif -#define noinline __attribute__((noinline)) - -#define preempt_enable() -#define preempt_disable() -#define cpu_has_feature(x) 1 -#define enable_kernel_altivec() -#define disable_kernel_altivec() - -#undef EXPORT_SYMBOL -#define EXPORT_SYMBOL(sym) -#undef EXPORT_SYMBOL_GPL -#define EXPORT_SYMBOL_GPL(sym) -#define MODULE_LICENSE(licence) -#define MODULE_DESCRIPTION(desc) -#define subsys_initcall(x) -#define module_exit(x) - -#define IS_ENABLED(x) (x) -#define CONFIG_RAID6_PQ_BENCHMARK 1 -#endif /* __KERNEL__ */ - -/* Routine choices */ -struct raid6_calls { - void (*gen_syndrome)(int, size_t, void **); - void (*xor_syndrome)(int, int, int, size_t, void **); - int (*valid)(void); /* Returns 1 if this routine set is usable */ - const char *name; /* Name of this routine set */ - int priority; /* Relative priority ranking if non-zero */ -}; - -/* Selected algorithm */ -extern struct raid6_calls raid6_call; - -/* Various routine sets */ -extern const struct raid6_calls raid6_intx1; -extern const struct raid6_calls raid6_intx2; -extern const struct raid6_calls raid6_intx4; -extern const struct raid6_calls raid6_intx8; -extern const struct raid6_calls raid6_mmxx1; -extern const struct raid6_calls raid6_mmxx2; -extern const struct raid6_calls raid6_sse1x1; -extern const struct raid6_calls raid6_sse1x2; -extern const struct raid6_calls raid6_sse2x1; -extern const struct raid6_calls raid6_sse2x2; -extern const struct raid6_calls raid6_sse2x4; -extern const struct raid6_calls raid6_altivec1; -extern const struct raid6_calls raid6_altivec2; -extern const struct raid6_calls raid6_altivec4; -extern const struct raid6_calls raid6_altivec8; -extern const struct raid6_calls raid6_avx2x1; -extern const struct raid6_calls raid6_avx2x2; -extern const struct raid6_calls raid6_avx2x4; -extern const struct raid6_calls raid6_avx512x1; -extern const struct raid6_calls raid6_avx512x2; -extern const struct raid6_calls raid6_avx512x4; -extern const struct raid6_calls raid6_s390vx8; -extern const struct raid6_calls raid6_vpermxor1; -extern const struct raid6_calls raid6_vpermxor2; -extern const struct raid6_calls raid6_vpermxor4; -extern const struct raid6_calls raid6_vpermxor8; -extern const struct raid6_calls raid6_lsx; -extern const struct raid6_calls raid6_lasx; -extern const struct raid6_calls raid6_rvvx1; -extern const struct raid6_calls raid6_rvvx2; -extern const struct raid6_calls raid6_rvvx4; -extern const struct raid6_calls raid6_rvvx8; - -struct raid6_recov_calls { - void (*data2)(int, size_t, int, int, void **); - void (*datap)(int, size_t, int, void **); - int (*valid)(void); - const char *name; - int priority; -}; - -extern const struct raid6_recov_calls raid6_recov_intx1; -extern const struct raid6_recov_calls raid6_recov_ssse3; -extern const struct raid6_recov_calls raid6_recov_avx2; -extern const struct raid6_recov_calls raid6_recov_avx512; -extern const struct raid6_recov_calls raid6_recov_s390xc; -extern const struct raid6_recov_calls raid6_recov_neon; -extern const struct raid6_recov_calls raid6_recov_lsx; -extern const struct raid6_recov_calls raid6_recov_lasx; -extern const struct raid6_recov_calls raid6_recov_rvv; - -extern const struct raid6_calls raid6_neonx1; -extern const struct raid6_calls raid6_neonx2; -extern const struct raid6_calls raid6_neonx4; -extern const struct raid6_calls raid6_neonx8; - -/* Algorithm list */ -extern const struct raid6_calls * const raid6_algos[]; -extern const struct raid6_recov_calls *const raid6_recov_algos[]; -int raid6_select_algo(void); - -/* Return values from chk_syndrome */ -#define RAID6_OK 0 -#define RAID6_P_BAD 1 -#define RAID6_Q_BAD 2 -#define RAID6_PQ_BAD 3 - -/* Galois field tables */ -extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); -extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); -extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); -extern const u8 raid6_gflog[256] __attribute__((aligned(256))); -extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); -extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); - -/* Recovery routines */ -extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb, - void **ptrs); -extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila, - void **ptrs); -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, - void **ptrs); - -/* Some definitions to allow code to be compiled for testing in userspace */ -#ifndef __KERNEL__ - -# define jiffies raid6_jiffies() -# define printk printf -# define pr_err(format, ...) fprintf(stderr, format, ## __VA_ARGS__) -# define pr_info(format, ...) fprintf(stdout, format, ## __VA_ARGS__) -# define GFP_KERNEL 0 -# define __get_free_pages(x, y) ((unsigned long)mmap(NULL, PAGE_SIZE << (y), \ - PROT_READ|PROT_WRITE, \ - MAP_PRIVATE|MAP_ANONYMOUS,\ - 0, 0)) -# define free_pages(x, y) munmap((void *)(x), PAGE_SIZE << (y)) + * Public interface to the RAID6 P/Q calculation and recovery library. + */ +#ifndef LINUX_RAID_PQ_H +#define LINUX_RAID_PQ_H -static inline void cpu_relax(void) -{ - /* Nothing */ -} +#include -#undef HZ -#define HZ 1000 -static inline uint32_t raid6_jiffies(void) -{ - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec*1000 + tv.tv_usec/1000; -} +#define RAID6_MIN_DISKS 4 -static inline void *raid6_get_zero_page(void) -{ - return raid6_empty_zero_page; -} +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs); +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs); +bool raid6_can_xor_syndrome(void); -#endif /* ! __KERNEL__ */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs); +void raid6_recov_datap(int disks, size_t bytes, int faila, + void **ptrs); -#endif /* LINUX_RAID_RAID6_H */ +#endif /* LINUX_RAID_PQ_H */ diff --git a/include/linux/raid/pq_tables.h b/include/linux/raid/pq_tables.h new file mode 100644 index 000000000000..7b1ebe675677 --- /dev/null +++ b/include/linux/raid/pq_tables.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + * + * Galois field tables for the Linux RAID6 P/Q parity algorithm. + */ +#ifndef _LINUX_RAID_PQ_TABLES_H +#define _LINUX_RAID_PQ_TABLES_H + +#include + +extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); +extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); +extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); +extern const u8 raid6_gflog[256] __attribute__((aligned(256))); +extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); +extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); + +#endif /* _LINUX_RAID_PQ_TABLES_H */ diff --git a/include/linux/raspberrypi/vchiq_bus.h b/include/linux/raspberrypi/vchiq_bus.h index 9de179b39f85..e52291a3b247 100644 --- a/include/linux/raspberrypi/vchiq_bus.h +++ b/include/linux/raspberrypi/vchiq_bus.h @@ -7,7 +7,7 @@ #define _VCHIQ_DEVICE_H #include -#include +#include struct vchiq_drv_mgmt; diff --git a/include/linux/regmap.h b/include/linux/regmap.h index df44cb30f53b..370baa19db87 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -1770,6 +1770,8 @@ struct regmap_irq_chip { void *irq_drv_data); unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data, unsigned int base, int index); + int (*irq_reqres)(void *irq_drv_data, irq_hw_number_t hwirq); + void (*irq_relres)(void *irq_drv_data, irq_hw_number_t hwirq); void *irq_drv_data; }; diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index b4795698d8c2..7c1546d48008 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -42,274 +42,7 @@ #include #include #include - -/** - * struct resource_table - firmware resource table header - * @ver: version number - * @num: number of resource entries - * @reserved: reserved (must be zero) - * @offset: array of offsets pointing at the various resource entries - * - * A resource table is essentially a list of system resources required - * by the remote processor. It may also include configuration entries. - * If needed, the remote processor firmware should contain this table - * as a dedicated ".resource_table" ELF section. - * - * Some resources entries are mere announcements, where the host is informed - * of specific remoteproc configuration. Other entries require the host to - * do something (e.g. allocate a system resource). Sometimes a negotiation - * is expected, where the firmware requests a resource, and once allocated, - * the host should provide back its details (e.g. address of an allocated - * memory region). - * - * The header of the resource table, as expressed by this structure, - * contains a version number (should we need to change this format in the - * future), the number of available resource entries, and their offsets - * in the table. - * - * Immediately following this header are the resource entries themselves, - * each of which begins with a resource entry header (as described below). - */ -struct resource_table { - u32 ver; - u32 num; - u32 reserved[2]; - u32 offset[]; -} __packed; - -/** - * struct fw_rsc_hdr - firmware resource entry header - * @type: resource type - * @data: resource data - * - * Every resource entry begins with a 'struct fw_rsc_hdr' header providing - * its @type. The content of the entry itself will immediately follow - * this header, and it should be parsed according to the resource type. - */ -struct fw_rsc_hdr { - u32 type; - u8 data[]; -} __packed; - -/** - * enum fw_resource_type - types of resource entries - * - * @RSC_CARVEOUT: request for allocation of a physically contiguous - * memory region. - * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. - * @RSC_TRACE: announces the availability of a trace buffer into which - * the remote processor will be writing logs. - * @RSC_VDEV: declare support for a virtio device, and serve as its - * virtio header. - * @RSC_LAST: just keep this one at the end of standard resources - * @RSC_VENDOR_START: start of the vendor specific resource types range - * @RSC_VENDOR_END: end of the vendor specific resource types range - * - * For more details regarding a specific resource type, please see its - * dedicated structure below. - * - * Please note that these values are used as indices to the rproc_handle_rsc - * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to - * check the validity of an index before the lookup table is accessed, so - * please update it as needed. - */ -enum fw_resource_type { - RSC_CARVEOUT = 0, - RSC_DEVMEM = 1, - RSC_TRACE = 2, - RSC_VDEV = 3, - RSC_LAST = 4, - RSC_VENDOR_START = 128, - RSC_VENDOR_END = 512, -}; - -#define FW_RSC_ADDR_ANY (-1) - -/** - * struct fw_rsc_carveout - physically contiguous memory request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested memory region - * - * This resource entry requests the host to allocate a physically contiguous - * memory region. - * - * These request entries should precede other firmware resource entries, - * as other entries might request placing other data objects inside - * these memory regions (e.g. data/code segments, trace resource entries, ...). - * - * Allocating memory this way helps utilizing the reserved physical memory - * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries - * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB - * pressure is important; it may have a substantial impact on performance. - * - * If the firmware is compiled with static addresses, then @da should specify - * the expected device address of this memory region. If @da is set to - * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then - * overwrite @da with the dynamically allocated address. - * - * We will always use @da to negotiate the device addresses, even if it - * isn't using an iommu. In that case, though, it will obviously contain - * physical addresses. - * - * Some remote processors needs to know the allocated physical address - * even if they do use an iommu. This is needed, e.g., if they control - * hardware accelerators which access the physical memory directly (this - * is the case with OMAP4 for instance). In that case, the host will - * overwrite @pa with the dynamically allocated physical address. - * Generally we don't want to expose physical addresses if we don't have to - * (remote processors are generally _not_ trusted), so we might want to - * change this to happen _only_ when explicitly required by the hardware. - * - * @flags is used to provide IOMMU protection flags, and @name should - * (optionally) contain a human readable name of this carveout region - * (mainly for debugging purposes). - */ -struct fw_rsc_carveout { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_devmem - iommu mapping request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested region to be mapped - * - * This resource entry requests the host to iommu map a physically contiguous - * memory region. This is needed in case the remote processor requires - * access to certain memory-based peripherals; _never_ use it to access - * regular memory. - * - * This is obviously only needed if the remote processor is accessing memory - * via an iommu. - * - * @da should specify the required device address, @pa should specify - * the physical address we want to map, @len should specify the size of - * the mapping and @flags is the IOMMU protection flags. As always, @name may - * (optionally) contain a human readable name of this mapping (mainly for - * debugging purposes). - * - * Note: at this point we just "trust" those devmem entries to contain valid - * physical addresses, but this isn't safe and will be changed: eventually we - * want remoteproc implementations to provide us ranges of physical addresses - * the firmware is allowed to request, and not allow firmwares to request - * access to physical addresses that are outside those ranges. - */ -struct fw_rsc_devmem { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_trace - trace buffer declaration - * @da: device address - * @len: length (in bytes) - * @reserved: reserved (must be zero) - * @name: human-readable name of the trace buffer - * - * This resource entry provides the host information about a trace buffer - * into which the remote processor will write log messages. - * - * @da specifies the device address of the buffer, @len specifies - * its size, and @name may contain a human readable name of the trace buffer. - * - * After booting the remote processor, the trace buffers are exposed to the - * user via debugfs entries (called trace0, trace1, etc..). - */ -struct fw_rsc_trace { - u32 da; - u32 len; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_vdev_vring - vring descriptor entry - * @da: device address - * @align: the alignment between the consumer and producer parts of the vring - * @num: num of buffers supported by this vring (must be power of two) - * @notifyid: a unique rproc-wide notify index for this vring. This notify - * index is used when kicking a remote processor, to let it know that this - * vring is triggered. - * @pa: physical address - * - * This descriptor is not a resource entry by itself; it is part of the - * vdev resource type (see below). - * - * Note that @da should either contain the device address where - * the remote processor is expecting the vring, or indicate that - * dynamically allocation of the vring's device address is supported. - */ -struct fw_rsc_vdev_vring { - u32 da; - u32 align; - u32 num; - u32 notifyid; - u32 pa; -} __packed; - -/** - * struct fw_rsc_vdev - virtio device header - * @id: virtio device id (as in virtio_ids.h) - * @notifyid: a unique rproc-wide notify index for this vdev. This notify - * index is used when kicking a remote processor, to let it know that the - * status/features of this vdev have changes. - * @dfeatures: specifies the virtio device features supported by the firmware - * @gfeatures: a place holder used by the host to write back the - * negotiated features that are supported by both sides. - * @config_len: the size of the virtio config space of this vdev. The config - * space lies in the resource table immediate after this vdev header. - * @status: a place holder where the host will indicate its virtio progress. - * @num_of_vrings: indicates how many vrings are described in this vdev header - * @reserved: reserved (must be zero) - * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. - * - * This resource is a virtio device header: it provides information about - * the vdev, and is then used by the host and its peer remote processors - * to negotiate and share certain virtio properties. - * - * By providing this resource entry, the firmware essentially asks remoteproc - * to statically allocate a vdev upon registration of the rproc (dynamic vdev - * allocation is not yet supported). - * - * Note: - * 1. unlike virtualization systems, the term 'host' here means - * the Linux side which is running remoteproc to control the remote - * processors. We use the name 'gfeatures' to comply with virtio's terms, - * though there isn't really any virtualized guest OS here: it's the host - * which is responsible for negotiating the final features. - * Yeah, it's a bit confusing. - * - * 2. immediately following this structure is the virtio config space for - * this vdev (which is specific to the vdev; for more info, read the virtio - * spec). The size of the config space is specified by @config_len. - */ -struct fw_rsc_vdev { - u32 id; - u32 notifyid; - u32 dfeatures; - u32 gfeatures; - u32 config_len; - u8 status; - u8 num_of_vrings; - u8 reserved[2]; - struct fw_rsc_vdev_vring vring[]; -} __packed; +#include struct rproc; diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 79f83b6eec27..57a2a29bef0e 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include diff --git a/include/linux/rio.h b/include/linux/rio.h index 2c29f21ba9e5..f42379775ce8 100644 --- a/include/linux/rio.h +++ b/include/linux/rio.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #ifdef CONFIG_RAPIDIO_DMA_ENGINE #include #endif diff --git a/include/linux/rpmsg.h b/include/linux/rpmsg.h index 2e40eb54155e..0171c490339c 100644 --- a/include/linux/rpmsg.h +++ b/include/linux/rpmsg.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/rsc_table.h b/include/linux/rsc_table.h new file mode 100644 index 000000000000..c6d6d553d8f1 --- /dev/null +++ b/include/linux/rsc_table.h @@ -0,0 +1,359 @@ +/* + * Resource table and its types data structure + * + * Copyright(c) 2011 Texas Instruments, Inc. + * Copyright(c) 2011 Google, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name Texas Instruments nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RSC_TABLE_H +#define RSC_TABLE_H + +/** + * struct resource_table - firmware resource table header + * @ver: version number + * @num: number of resource entries + * @reserved: reserved (must be zero) + * @offset: array of offsets pointing at the various resource entries + * + * A resource table is essentially a list of system resources required + * by the remote processor. It may also include configuration entries. + * If needed, the remote processor firmware should contain this table + * as a dedicated ".resource_table" ELF section. + * + * Some resources entries are mere announcements, where the host is informed + * of specific remoteproc configuration. Other entries require the host to + * do something (e.g. allocate a system resource). Sometimes a negotiation + * is expected, where the firmware requests a resource, and once allocated, + * the host should provide back its details (e.g. address of an allocated + * memory region). + * + * The header of the resource table, as expressed by this structure, + * contains a version number (should we need to change this format in the + * future), the number of available resource entries, and their offsets + * in the table. + * + * Immediately following this header are the resource entries themselves, + * each of which begins with a resource entry header (as described below). + */ +struct resource_table { + u32 ver; + u32 num; + u32 reserved[2]; + u32 offset[]; +} __packed; + +/** + * struct fw_rsc_hdr - firmware resource entry header + * @type: resource type + * @data: resource data + * + * Every resource entry begins with a 'struct fw_rsc_hdr' header providing + * its @type. The content of the entry itself will immediately follow + * this header, and it should be parsed according to the resource type. + */ +struct fw_rsc_hdr { + u32 type; + u8 data[]; +} __packed; + +/** + * enum fw_resource_type - types of resource entries + * + * @RSC_CARVEOUT: request for allocation of a physically contiguous + * memory region. + * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. + * @RSC_TRACE: announces the availability of a trace buffer into which + * the remote processor will be writing logs. + * @RSC_VDEV: declare support for a virtio device, and serve as its + * virtio header. + * @RSC_LAST: just keep this one at the end of standard resources + * @RSC_VENDOR_START: start of the vendor specific resource types range + * @RSC_VENDOR_END: end of the vendor specific resource types range + * + * For more details regarding a specific resource type, please see its + * dedicated structure below. + * + * Please note that these values are used as indices to the rproc_handle_rsc + * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to + * check the validity of an index before the lookup table is accessed, so + * please update it as needed. + */ +enum fw_resource_type { + RSC_CARVEOUT = 0, + RSC_DEVMEM = 1, + RSC_TRACE = 2, + RSC_VDEV = 3, + RSC_LAST = 4, + RSC_VENDOR_START = 128, + RSC_VENDOR_END = 512, +}; + +#define FW_RSC_ADDR_ANY (-1) + +/** + * struct fw_rsc_carveout - physically contiguous memory request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested memory region + * + * This resource entry requests the host to allocate a physically contiguous + * memory region. + * + * These request entries should precede other firmware resource entries, + * as other entries might request placing other data objects inside + * these memory regions (e.g. data/code segments, trace resource entries, ...). + * + * Allocating memory this way helps utilizing the reserved physical memory + * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries + * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB + * pressure is important; it may have a substantial impact on performance. + * + * If the firmware is compiled with static addresses, then @da should specify + * the expected device address of this memory region. If @da is set to + * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then + * overwrite @da with the dynamically allocated address. + * + * We will always use @da to negotiate the device addresses, even if it + * isn't using an iommu. In that case, though, it will obviously contain + * physical addresses. + * + * Some remote processors needs to know the allocated physical address + * even if they do use an iommu. This is needed, e.g., if they control + * hardware accelerators which access the physical memory directly (this + * is the case with OMAP4 for instance). In that case, the host will + * overwrite @pa with the dynamically allocated physical address. + * Generally we don't want to expose physical addresses if we don't have to + * (remote processors are generally _not_ trusted), so we might want to + * change this to happen _only_ when explicitly required by the hardware. + * + * @flags is used to provide IOMMU protection flags, and @name should + * (optionally) contain a human readable name of this carveout region + * (mainly for debugging purposes). + */ +struct fw_rsc_carveout { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_devmem - iommu mapping request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested region to be mapped + * + * This resource entry requests the host to iommu map a physically contiguous + * memory region. This is needed in case the remote processor requires + * access to certain memory-based peripherals; _never_ use it to access + * regular memory. + * + * This is obviously only needed if the remote processor is accessing memory + * via an iommu. + * + * @da should specify the required device address, @pa should specify + * the physical address we want to map, @len should specify the size of + * the mapping and @flags is the IOMMU protection flags. As always, @name may + * (optionally) contain a human readable name of this mapping (mainly for + * debugging purposes). + * + * Note: at this point we just "trust" those devmem entries to contain valid + * physical addresses, but this isn't safe and will be changed: eventually we + * want remoteproc implementations to provide us ranges of physical addresses + * the firmware is allowed to request, and not allow firmwares to request + * access to physical addresses that are outside those ranges. + */ +struct fw_rsc_devmem { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_trace - trace buffer declaration + * @da: device address + * @len: length (in bytes) + * @reserved: reserved (must be zero) + * @name: human-readable name of the trace buffer + * + * This resource entry provides the host information about a trace buffer + * into which the remote processor will write log messages. + * + * @da specifies the device address of the buffer, @len specifies + * its size, and @name may contain a human readable name of the trace buffer. + * + * After booting the remote processor, the trace buffers are exposed to the + * user via debugfs entries (called trace0, trace1, etc..). + */ +struct fw_rsc_trace { + u32 da; + u32 len; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_vdev_vring - vring descriptor entry + * @da: device address + * @align: the alignment between the consumer and producer parts of the vring + * @num: num of buffers supported by this vring (must be power of two) + * @notifyid: a unique rproc-wide notify index for this vring. This notify + * index is used when kicking a remote processor, to let it know that this + * vring is triggered. + * @pa: physical address + * + * This descriptor is not a resource entry by itself; it is part of the + * vdev resource type (see below). + * + * Note that @da should either contain the device address where + * the remote processor is expecting the vring, or indicate that + * dynamically allocation of the vring's device address is supported. + */ +struct fw_rsc_vdev_vring { + u32 da; + u32 align; + u32 num; + u32 notifyid; + u32 pa; +} __packed; + +/** + * struct fw_rsc_vdev - virtio device header + * @id: virtio device id (as in virtio_ids.h) + * @notifyid: a unique rproc-wide notify index for this vdev. This notify + * index is used when kicking a remote processor, to let it know that the + * status/features of this vdev have changes. + * @dfeatures: specifies the virtio device features supported by the firmware + * @gfeatures: a place holder used by the host to write back the + * negotiated features that are supported by both sides. + * @config_len: the size of the virtio config space of this vdev. The config + * space lies in the resource table immediate after this vdev header. + * @status: a place holder where the host will indicate its virtio progress. + * @num_of_vrings: indicates how many vrings are described in this vdev header + * @reserved: reserved (must be zero) + * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. + * + * This resource is a virtio device header: it provides information about + * the vdev, and is then used by the host and its peer remote processors + * to negotiate and share certain virtio properties. + * + * By providing this resource entry, the firmware essentially asks remoteproc + * to statically allocate a vdev upon registration of the rproc (dynamic vdev + * allocation is not yet supported). + * + * Note: + * 1. unlike virtualization systems, the term 'host' here means + * the Linux side which is running remoteproc to control the remote + * processors. We use the name 'gfeatures' to comply with virtio's terms, + * though there isn't really any virtualized guest OS here: it's the host + * which is responsible for negotiating the final features. + * Yeah, it's a bit confusing. + * + * 2. immediately following this structure is the virtio config space for + * this vdev (which is specific to the vdev; for more info, read the virtio + * spec). The size of the config space is specified by @config_len. + */ +struct fw_rsc_vdev { + u32 id; + u32 notifyid; + u32 dfeatures; + u32 gfeatures; + u32 config_len; + u8 status; + u8 num_of_vrings; + u8 reserved[2]; + struct fw_rsc_vdev_vring vring[]; +} __packed; + +/** + * rsc_table_for_each_entry() - iterate over all entries in a resource table + * @table: pointer to the resource table + * @table_sz: total size of the table buffer in bytes + * @dev: device used for error logging + * @cb: callback invoked for each entry: + * @type - value from enum fw_resource_type + * @rsc - pointer to the entry payload (past struct fw_rsc_hdr) + * @offset - byte offset of the payload within the table; callers + * that write back into the table (e.g. to record a + * dynamically allocated address) use this to locate the + * entry for later update + * @avail - bytes available in the payload + * @data - caller-supplied private pointer + * Return 0 to continue iteration, non-zero to stop. + * @data: private pointer forwarded to @cb on every call + * + * Iterates over every resource entry in @table, performing the standard + * truncation check, and invokes @cb for each one. Iteration stops on the + * first non-zero return from @cb or on a malformed table. + * + * Returns 0 after a complete iteration, -EINVAL if the table is truncated, + * or the first non-zero value returned by @cb. + */ +static inline int rsc_table_for_each_entry(struct resource_table *table, + size_t table_sz, + struct device *dev, + int (*cb)(u32 type, void *rsc, + int offset, int avail, + void *data), + void *data) { + int i, ret; + + for (i = 0; i < table->num; i++) { + int offset = table->offset[i]; + struct fw_rsc_hdr *hdr = (void *)table + offset; + int avail = table_sz - offset - sizeof(*hdr); + int rsc_offset = offset + sizeof(*hdr); + void *rsc = (void *)hdr + sizeof(*hdr); + + if (avail < 0) { + dev_err(dev, "rsc table is truncated\n"); + return -EINVAL; + } + + ret = cb(hdr->type, rsc, rsc_offset, avail, data); + if (ret) + return ret; + } + + return 0; +} + +#endif /* RSC_TABLE_H */ diff --git a/include/linux/rtc.h b/include/linux/rtc.h index 95da051fb155..c09fc22819d0 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -190,6 +190,8 @@ extern int rtc_set_time(struct rtc_device *rtc, struct rtc_time *tm); int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm); extern int rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alrm); +extern int rtc_read_next_alarm(struct rtc_device *rtc, + struct rtc_wkalrm *alrm); extern int rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alrm); extern int rtc_initialize_alarm(struct rtc_device *rtc, diff --git a/include/linux/sched.h b/include/linux/sched.h index b3204a15d512..bcd101db428e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -550,7 +550,6 @@ struct sched_statistics { s64 exec_max; u64 slice_max; - u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; @@ -563,8 +562,6 @@ struct sched_statistics { u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; - u64 nr_wakeups_passive; - u64 nr_wakeups_idle; #ifdef CONFIG_SCHED_CORE u64 core_forceidle_sum; @@ -575,6 +572,7 @@ struct sched_statistics { struct sched_entity { /* For load-balancing: */ struct load_weight load; + struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; @@ -1191,6 +1189,7 @@ struct task_struct { unsigned long last_switch_time; #endif /* Filesystem information: */ + struct fs_struct *real_fs; struct fs_struct *fs; /* Open file information: */ @@ -1288,6 +1287,7 @@ struct task_struct { u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; + unsigned int lockdep_seq; struct held_lock held_locks[MAX_LOCK_DEPTH]; #endif @@ -1532,6 +1532,9 @@ struct task_struct { /* KCOV descriptor wired with this task or NULL: */ struct kcov *kcov; + /* KCOV descriptor for remote coverage collection from other tasks: */ + struct kcov *kcov_remote; + /* KCOV common handle for remote coverage collection: */ u64 kcov_handle; diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 20b2343aa344..75cb8b119fb7 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -263,7 +263,7 @@ void sched_ext_dead(struct task_struct *p); void print_scx_info(const char *log_lvl, struct task_struct *p); void scx_softlockup(u32 dur_s); bool scx_hardlockup(int cpu); -bool scx_rcu_cpu_stall(void); +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask); #else /* !CONFIG_SCHED_CLASS_EXT */ @@ -271,7 +271,7 @@ static inline void sched_ext_dead(struct task_struct *p) {} static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {} static inline void scx_softlockup(u32 dur_s) {} static inline bool scx_hardlockup(int cpu) { return false; } -static inline bool scx_rcu_cpu_stall(void) { return false; } +static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; } #endif /* CONFIG_SCHED_CLASS_EXT */ diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 41ed884cffc9..e0c1ca8c6a18 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -31,6 +31,7 @@ struct kernel_clone_args { u32 io_thread:1; u32 user_worker:1; u32 no_files:1; + u32 umh:1; unsigned long stack; unsigned long stack_size; unsigned long tls; diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 110ad4e2aef9..bdc214386e4a 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -533,6 +533,7 @@ struct uart_port { #define UPF_HARD_FLOW ((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS)) /* Port has hardware-assisted s/w flow control */ #define UPF_SOFT_FLOW ((__force upf_t) BIT_ULL(22)) +/* Deprecated: use uart_set_cons_flow_enabled()/uart_cons_flow_enabled() instead. */ #define UPF_CONS_FLOW ((__force upf_t) BIT_ULL(23)) #define UPF_SHARE_IRQ ((__force upf_t) BIT_ULL(24)) #define UPF_EXAR_EFR ((__force upf_t) BIT_ULL(25)) @@ -567,6 +568,7 @@ struct uart_port { #define UPSTAT_SYNC_FIFO ((__force upstat_t) (1 << 5)) bool hw_stopped; /* sw-assisted CTS flow state */ + bool cons_flow; /* user specified console flow control */ unsigned int mctrl; /* current modem ctrl settings */ unsigned int frame_time; /* frame timing in ns */ unsigned int type; /* port type */ @@ -1163,6 +1165,24 @@ static inline bool uart_softcts_mode(struct uart_port *uport) return ((uport->status & mask) == UPSTAT_CTS_ENABLE); } +static inline void uart_set_cons_flow_enabled(struct uart_port *uport, bool enabled) +{ + uport->cons_flow = enabled; +} + +static inline bool uart_cons_flow_enabled(const struct uart_port *uport) +{ + return uport->cons_flow; +} + +static inline bool uart_console_hwflow_active(struct uart_port *uport) +{ + return uart_console(uport) && + !(uport->rs485.flags & SER_RS485_ENABLED) && + uart_cons_flow_enabled(uport) && + uart_cts_enabled(uport); +} + /* * The following are helper functions for the low level drivers. */ diff --git a/include/linux/serial_s3c.h b/include/linux/serial_s3c.h index 102aa33d956c..f54cb6e23f85 100644 --- a/include/linux/serial_s3c.h +++ b/include/linux/serial_s3c.h @@ -269,7 +269,7 @@ #define APPLE_S5L_UTRSTAT_RXTO BIT(9) #define APPLE_S5L_UTRSTAT_ALL_FLAGS GENMASK(9, 3) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -294,7 +294,7 @@ struct s3c2410_uartcfg { unsigned long ufcon; /* value of ufcon for port */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_ARM_REGS_SERIAL_H */ diff --git a/include/linux/serial_sci.h b/include/linux/serial_sci.h index 0f2f50b8d28e..36c795d61f7e 100644 --- a/include/linux/serial_sci.h +++ b/include/linux/serial_sci.h @@ -51,7 +51,6 @@ struct plat_sci_port_ops { */ struct plat_sci_port { unsigned int type; /* SCI / SCIF / IRDA / HSCIF */ - upf_t flags; /* UPF_* flags */ unsigned int sampling_rate; unsigned int scscr; /* SCSCR initialization */ diff --git a/include/linux/serio.h b/include/linux/serio.h index 69a47674af65..98be7084412c 100644 --- a/include/linux/serio.h +++ b/include/linux/serio.h @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include extern const struct bus_type serio_bus; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 69b0177da156..e729b9b0e38d 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -128,7 +128,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SHMEM) unsigned long shmem_allowable_huge_orders(struct inode *inode, struct vm_area_struct *vma, pgoff_t index, loff_t write_end, bool shmem_huge_force); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 115db8c44db2..22eda1d54a0e 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1084,9 +1084,7 @@ struct sk_buff { __u16 network_header; __u16 mac_header; -#ifdef CONFIG_KCOV - u64 kcov_handle; -#endif + struct kcov_common_handle_id kcov_handle; ); /* end headers group */ @@ -5440,20 +5438,14 @@ static inline void skb_reset_csum_not_inet(struct sk_buff *skb) } static inline void skb_set_kcov_handle(struct sk_buff *skb, - const u64 kcov_handle) + struct kcov_common_handle_id kcov_handle) { -#ifdef CONFIG_KCOV skb->kcov_handle = kcov_handle; -#endif } -static inline u64 skb_get_kcov_handle(struct sk_buff *skb) +static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *skb) { -#ifdef CONFIG_KCOV return skb->kcov_handle; -#else - return 0; -#endif } static inline void skb_mark_for_recycle(struct sk_buff *skb) diff --git a/include/linux/slab.h b/include/linux/slab.h index d4a873a16289..51f03f18c9a7 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1039,8 +1039,9 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in /** * kmalloc_nolock - Allocate an object of given size from any context. * @size: size to allocate - * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT - * allowed. + * @gfp_flags: GFP flags. Only __GFP_ACCOUNT and __GFP_ZERO allowed. Also + * __GFP_NOWARN and __GFP_NOMEMALLOC are allowed but added internally thus not + * necessary. * @node: node number of the target node. * * Return: pointer to the new object or NULL in case of error. @@ -1093,7 +1094,7 @@ void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); /** * kmalloc_obj - Allocate a single instance of the given type * @VAR_OR_TYPE: Variable or type to allocate. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL * on failure. @@ -1105,7 +1106,7 @@ void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); * kmalloc_objs - Allocate an array of the given type * @VAR_OR_TYPE: Variable or type to allocate an array of. * @COUNT: How many elements in the array. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, * or NULL on failure. @@ -1118,7 +1119,7 @@ void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). * @FAM: The name of the flexible array member of the structure. * @COUNT: How many flexible array member elements are desired. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on * failure. If @FAM has been annotated with __counted_by(), the allocation @@ -1155,8 +1156,11 @@ void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); #define kmem_buckets_alloc(_b, _size, _flags) \ alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE)) -#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ - alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE, _RET_IP_)) +#define kmem_buckets_alloc_node_track_caller(_b, _size, _flags, _node) \ + alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, _node, _RET_IP_)) + +#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ + kmem_buckets_alloc_node_track_caller(_b, _size, _flags, NUMA_NO_NODE) static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token) { diff --git a/include/linux/slimbus.h b/include/linux/slimbus.h index a4608d9a9684..ca6f1da4bdf3 100644 --- a/include/linux/slimbus.h +++ b/include/linux/slimbus.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include extern const struct bus_type slimbus_bus; diff --git a/include/linux/soc/qcom/apr.h b/include/linux/soc/qcom/apr.h index 58fa1df96347..909e84f84e0c 100644 --- a/include/linux/soc/qcom/apr.h +++ b/include/linux/soc/qcom/apr.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 83d2c2a7116c..a7372d9c25fb 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -13,20 +13,6 @@ struct qcom_ubwc_cfg_data { u32 ubwc_enc_version; - /* Can be read from MDSS_BASE + 0x58 */ - u32 ubwc_dec_version; - - /** - * @ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling. - * - * UBWC 1.0 always enables all three levels. - * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3. - * UBWC 4.0 adds the optional ability to disable levels 2 & 3. - */ - u32 ubwc_swizzle; -#define UBWC_SWIZZLE_ENABLE_LVL1 BIT(0) -#define UBWC_SWIZZLE_ENABLE_LVL2 BIT(1) -#define UBWC_SWIZZLE_ENABLE_LVL3 BIT(2) /** * @highest_bank_bit: Highest Bank Bit @@ -35,16 +21,10 @@ struct qcom_ubwc_cfg_data { * DDR bank. This should ideally use DRAM type detection. */ int highest_bank_bit; - bool ubwc_bank_spread; - /** - * @macrotile_mode: Macrotile Mode - * - * Whether to use 4-channel macrotiling mode or the newer - * 8-channel macrotiling mode introduced in UBWC 3.1. 0 is - * 4-channel and 1 is 8-channel. - */ - bool macrotile_mode; + unsigned int flags; +#define UBWC_FLAG_DISABLE_SWIZZLE_LVL2 BIT(0) +#define UBWC_FLAG_DISABLE_SWIZZLE_LVL3 BIT(1) }; #define UBWC_1_0 0x10000000 @@ -67,37 +47,62 @@ static inline const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void) static inline bool qcom_ubwc_get_ubwc_mode(const struct qcom_ubwc_cfg_data *cfg) { - bool ret = cfg->ubwc_enc_version == UBWC_1_0; - - if (ret && !(cfg->ubwc_swizzle & UBWC_SWIZZLE_ENABLE_LVL1)) - pr_err("UBWC config discrepancy - level 1 swizzling disabled on UBWC 1.0\n"); - - return ret; + return cfg->ubwc_enc_version == UBWC_1_0; } -/* - * This is the best guess, based on the MDSS driver, which worked so far. - */ static inline bool qcom_ubwc_min_acc_length_64b(const struct qcom_ubwc_cfg_data *cfg) { - return cfg->ubwc_enc_version == UBWC_1_0 && - (cfg->ubwc_dec_version == UBWC_2_0 || - cfg->ubwc_dec_version == UBWC_3_0); + return cfg->ubwc_enc_version == UBWC_1_0; } +/* + * @qcom_ubwc_macrotile_mode: whether to use 4-channel or 8-channel macrotiling + * + * The 8-channel macrotiling mode was introduced in UBWC 3.1. + * + * Returns: false for the 4-channel and true for 8-channel. + */ static inline bool qcom_ubwc_macrotile_mode(const struct qcom_ubwc_cfg_data *cfg) { - return cfg->macrotile_mode; + return cfg->ubwc_enc_version >= UBWC_3_1; } static inline bool qcom_ubwc_bank_spread(const struct qcom_ubwc_cfg_data *cfg) { - return cfg->ubwc_bank_spread; + return true; } +#define UBWC_SWIZZLE_ENABLE_LVL1 BIT(0) +#define UBWC_SWIZZLE_ENABLE_LVL2 BIT(1) +#define UBWC_SWIZZLE_ENABLE_LVL3 BIT(2) + +/** + * @qcom_ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling. + * + * UBWC 1.0 always enables all three levels. + * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3. + * UBWC 4.0 adds the optional ability to disable levels 2 & 3. + */ static inline u32 qcom_ubwc_swizzle(const struct qcom_ubwc_cfg_data *cfg) { - return cfg->ubwc_swizzle; + if (cfg->ubwc_enc_version == 0) + return 0; + + if (cfg->ubwc_enc_version == UBWC_1_0) + return UBWC_SWIZZLE_ENABLE_LVL1 | + UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3; + + u32 ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3; + + if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL2) + ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL2; + + if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL3) + ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL3; + + return ubwc_swizzle; } static inline u32 qcom_ubwc_version_tag(const struct qcom_ubwc_cfg_data *cfg) diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index a46cbaec5949..79dd44922fbc 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -11,8 +11,9 @@ #include #include #include +#include #include -#include +#include #include #include #include @@ -1093,8 +1094,6 @@ int sdw_slave_get_current_bank(struct sdw_slave *sdev); int sdw_slave_get_scale_index(struct sdw_slave *slave, u8 *base); -int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms); - /* messaging and data APIs */ int sdw_read(struct sdw_slave *slave, u32 addr); int sdw_write(struct sdw_slave *slave, u32 addr, u8 value); @@ -1138,12 +1137,6 @@ static inline int sdw_slave_get_current_bank(struct sdw_slave *sdev) return -EINVAL; } -static inline int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms) -{ - WARN_ONCE(1, "SoundWire API is disabled"); - return -EINVAL; -} - /* messaging and data APIs */ static inline int sdw_read(struct sdw_slave *slave, u32 addr) { @@ -1207,4 +1200,33 @@ static inline int sdw_update_no_pm(struct sdw_slave *slave, u32 addr, u8 mask, u #endif /* CONFIG_SOUNDWIRE */ +/** + * sdw_slave_wait_for_init - Wait for device initialisation + * @slave: Pointer to the SoundWire peripheral. + * @timeout_ms: Timeout in milliseconds. + * + * Wait for a peripheral device to enumerate and be initialised by the + * SoundWire core. + * + * Return: Zero on success, and a negative error code on failure. + */ +static inline int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms) +{ + unsigned long time; + + if (!slave) + return 0; + + time = wait_for_completion_timeout(&slave->initialization_complete, + msecs_to_jiffies(timeout_ms)); + if (!time) { + dev_err(&slave->dev, "Initialization not complete\n"); + return -ETIMEDOUT; + } + + slave->unattach_request = 0; + + return 0; +} + #endif /* __SOUNDWIRE_H */ diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index f6ed93eff00b..4c285d3ede1d 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -12,7 +12,9 @@ #include #include #include -#include +#include +#include +#include #include #include #include diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 28e8c8bd3944..4eb9564a7fb3 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -76,6 +76,7 @@ void spmi_device_remove(struct spmi_device *sdev); * @cmd: sends a non-data command sequence on the SPMI bus. * @read_cmd: sends a register read command sequence on the SPMI bus. * @write_cmd: sends a register write command sequence on the SPMI bus. + * @priv: array of private data. */ struct spmi_controller { struct device dev; @@ -85,6 +86,7 @@ struct spmi_controller { u8 sid, u16 addr, u8 *buf, size_t len); int (*write_cmd)(struct spmi_controller *ctrl, u8 opcode, u8 sid, u16 addr, const u8 *buf, size_t len); + u8 priv[]; }; static inline struct spmi_controller *to_spmi_controller(struct device *d) @@ -109,7 +111,7 @@ struct spmi_controller *spmi_controller_alloc(struct device *parent, /** * spmi_controller_put() - decrement controller refcount - * @ctrl SPMI controller. + * @ctrl: SPMI controller. */ static inline void spmi_controller_put(struct spmi_controller *ctrl) { @@ -129,6 +131,7 @@ int devm_spmi_controller_add(struct device *parent, struct spmi_controller *ctrl * this structure. * @probe: binds this driver to a SPMI device. * @remove: unbinds this driver from the SPMI device. + * @shutdown: shuts down this driver. * * If PM runtime support is desired for a slave, a device driver can call * pm_runtime_put() from their probe() routine (and a balancing diff --git a/include/linux/ssb/ssb.h b/include/linux/ssb/ssb.h index e1fb11e0f12c..7fee9afa9458 100644 --- a/include/linux/ssb/ssb.h +++ b/include/linux/ssb/ssb.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h index 98939cb664cf..59d0cc889beb 100644 --- a/include/linux/sunrpc/bc_xprt.h +++ b/include/linux/sunrpc/bc_xprt.h @@ -32,6 +32,7 @@ int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs); void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs); void xprt_free_bc_rqst(struct rpc_rqst *req); unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt); +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt); void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv); /* @@ -71,6 +72,10 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req) { } +static inline void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ +} + static inline void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { svc_destroy(serv); diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 4be6204f6630..3a0152d926fb 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -469,6 +469,7 @@ int svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool, unsigned int min_threads, unsigned int max_threads); int svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, unsigned int nrservs); +unsigned int svc_serv_maxthreads(const struct svc_serv *serv); int svc_pool_stats_open(struct svc_info *si, struct file *file); void svc_process(struct svc_rqst *rqstp); void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp); diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h index 7516ad0fae80..6346d8cf2587 100644 --- a/include/linux/sunrpc/svc_rdma_pcl.h +++ b/include/linux/sunrpc/svc_rdma_pcl.h @@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk) */ #define pcl_for_each_segment(pos, chunk) \ for (pos = &(chunk)->ch_segments[0]; \ - pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \ + pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \ pos++) /** @@ -119,6 +119,8 @@ extern bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, struct svc_rdma_pcl *pcl, __be32 *p); +extern bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt, + unsigned int inline_len); extern int pcl_process_nonpayloads(const struct svc_rdma_pcl *pcl, const struct xdr_buf *xdr, int (*actor)(const struct xdr_buf *, diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index 8cd8c38cf3f3..ed6b271e5a73 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -14,7 +14,7 @@ #define _LINUX_SURFACE_AGGREGATOR_DEVICE_H #include -#include +#include #include #include diff --git a/include/linux/swap.h b/include/linux/swap.h index 8f0f68e245ba..3f31b6a56788 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -16,10 +16,6 @@ #include #include -struct notifier_block; - -struct bio; - #define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */ @@ -29,7 +25,6 @@ struct bio; #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) -#define SWAP_BATCH 64 static inline int current_is_kswapd(void) { @@ -175,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages) struct address_space; struct sysinfo; -struct writeback_control; struct zone; /* @@ -351,7 +345,7 @@ extern void swap_setup(void); /* linux/mm/vmscan.c */ extern unsigned long zone_reclaimable_pages(struct zone *zone); extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *mask); + gfp_t gfp_mask, const nodemask_t *mask); unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) @@ -442,7 +436,6 @@ extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); -struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..705a84154d28 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); @@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index cdb58d61c152..724da6c08bf7 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -42,6 +42,7 @@ enum switchtec_gen { SWITCHTEC_GEN3, SWITCHTEC_GEN4, SWITCHTEC_GEN5, + SWITCHTEC_GEN6, }; struct mrpc_regs { diff --git a/include/linux/synclink.h b/include/linux/synclink.h deleted file mode 100644 index f1405b1c71ba..000000000000 --- a/include/linux/synclink.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SyncLink Multiprotocol Serial Adapter Driver - * - * $Id: synclink.h,v 3.14 2006/07/17 20:15:43 paulkf Exp $ - * - * Copyright (C) 1998-2000 by Microgate Corporation - * - * Redistribution of this file is permitted under - * the terms of the GNU Public License (GPL) - */ -#ifndef _SYNCLINK_H_ -#define _SYNCLINK_H_ - -#include - -/* provide 32 bit ioctl compatibility on 64 bit systems */ -#ifdef CONFIG_COMPAT -#include -struct MGSL_PARAMS32 { - compat_ulong_t mode; - unsigned char loopback; - unsigned short flags; - unsigned char encoding; - compat_ulong_t clock_speed; - unsigned char addr_filter; - unsigned short crc_type; - unsigned char preamble_length; - unsigned char preamble; - compat_ulong_t data_rate; - unsigned char data_bits; - unsigned char stop_bits; - unsigned char parity; -}; -#define MGSL_IOCSPARAMS32 _IOW(MGSL_MAGIC_IOC,0,struct MGSL_PARAMS32) -#define MGSL_IOCGPARAMS32 _IOR(MGSL_MAGIC_IOC,1,struct MGSL_PARAMS32) -#endif -#endif /* _SYNCLINK_H_ */ diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h index 5226efde9ad4..ed23d6516223 100644 --- a/include/linux/sysfb.h +++ b/include/linux/sysfb.h @@ -118,7 +118,7 @@ struct platform_device *sysfb_create_simplefb(const struct screen_info *si, const struct simplefb_platform_data *mode, struct device *parent); -#else /* CONFIG_SYSFB_SIMPLE */ +#else /* CONFIG_SYSFB_SIMPLEFB */ static inline bool sysfb_parse_mode(const struct screen_info *si, struct simplefb_platform_data *mode) @@ -133,6 +133,6 @@ static inline struct platform_device *sysfb_create_simplefb(const struct screen_ return ERR_PTR(-EINVAL); } -#endif /* CONFIG_SYSFB_SIMPLE */ +#endif /* CONFIG_SYSFB_SIMPLEFB */ #endif /* _LINUX_SYSFB_H */ diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h index e561a26f537a..f3c5e106d853 100644 --- a/include/linux/tee_drv.h +++ b/include/linux/tee_drv.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 0ba112175bb3..56aa4d620279 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -13,6 +13,7 @@ #include +struct config_group; struct fwnode_handle; struct device; @@ -22,7 +23,7 @@ struct device; #include #include #include -#include +#include #include #include #include @@ -153,6 +154,9 @@ struct tb_property_dir *tb_property_parse_dir(const u32 *block, ssize_t tb_property_format_dir(const struct tb_property_dir *dir, u32 *block, size_t block_len); struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir); +int tb_property_merge_dir(struct tb_property_dir *parent, + const struct tb_property_dir *dir, + bool replace); struct tb_property_dir *tb_property_create_dir(const uuid_t *uuid); void tb_property_free_dir(struct tb_property_dir *dir); int tb_property_add_immediate(struct tb_property_dir *parent, const char *key, @@ -209,6 +213,8 @@ enum tb_link_width { * @link_width: Width of the downstream facing link * @link_usb4: Downstream link is USB4 * @is_unplugged: The XDomain is unplugged + * @removing: Set by tb_xdomain_remove() under @lock to prevent + * concurrent delayed work queueing * @needs_uuid: If the XDomain does not have @remote_uuid it will be * queried first * @service_ids: Used to generate IDs for the services @@ -228,6 +234,7 @@ enum tb_link_width { * changed notification * @bonding_possible: True if lane bonding is possible on local side * @target_link_width: Target link width from the remote host + * @ntunnels: Keeps track of how many tunnels go through this XDomain * @link: Root switch link the remote domain is connected (ICM only) * @depth: Depth in the chain the remote domain is connected (ICM only) * @@ -257,6 +264,7 @@ struct tb_xdomain { enum tb_link_width link_width; bool link_usb4; bool is_unplugged; + bool removing; bool needs_uuid; struct ida service_ids; struct ida in_hopids; @@ -273,6 +281,7 @@ struct tb_xdomain { int properties_changed_retries; bool bonding_possible; u8 target_link_width; + atomic_t ntunnels; u8 link; u8 depth; }; @@ -392,6 +401,10 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler); * @prtcvers: Protocol version from the properties directory * @prtcrevs: Protocol software revision from the properties directory * @prtcstns: Protocol settings mask from the properties directory + * @lock: Protects this structure + * @local_properties: Properties owned by the service driver + * @remote_properties: Properties read from the remote service. These + * are read-only. * @debugfs_dir: Pointer to the service debugfs directory. Always created * when debugfs is enabled. Can be used by service drivers to * add their own entries under the service. @@ -399,6 +412,9 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler); * Each domain exposes set of services it supports as collection of * properties. For each service there will be one corresponding * &struct tb_service. Service drivers are bound to these. + * + * Service drivers can add their own dynamic properties to + * @local_properties but whenever they do so @lock must be held. */ struct tb_service { struct device dev; @@ -408,6 +424,9 @@ struct tb_service { u32 prtcvers; u32 prtcrevs; u32 prtcstns; + struct mutex lock; + struct tb_property_dir *local_properties; + struct tb_property_dir *remote_properties; struct dentry *debugfs_dir; }; @@ -446,7 +465,7 @@ static inline struct tb_service *tb_to_service(struct device *dev) */ struct tb_service_driver { struct device_driver driver; - int (*probe)(struct tb_service *svc, const struct tb_service_id *id); + int (*probe)(struct tb_service *svc); void (*remove)(struct tb_service *svc); void (*shutdown)(struct tb_service *svc); const struct tb_service_id *id_table; @@ -476,16 +495,17 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc) return tb_to_xdomain(svc->dev.parent); } +void tb_service_properties_changed(struct tb_service *svc); + /** * struct tb_nhi - thunderbolt native host interface * @lock: Must be held during ring creation/destruction. Is acquired by * interrupt_work when dispatching interrupts to individual rings. - * @pdev: Pointer to the PCI device + * @dev: Device associated with this NHI instance * @ops: NHI specific optional ops * @iobase: MMIO space of the NHI * @tx_rings: All Tx rings available on this host controller * @rx_rings: All Rx rings available on this host controller - * @msix_ida: Used to allocate MSI-X vectors for rings * @going_away: The host controller device is about to disappear so when * this flag is set, avoid touching the hardware anymore. * @iommu_dma_protection: An IOMMU will isolate external-facing ports. @@ -493,20 +513,27 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc) * MSI-X is used. * @hop_count: Number of rings (end point hops) supported by NHI. * @quirks: NHI specific quirks if any + * @domain_released: Completed when domain has been fully released + * @host_reset: Host router was reset on driver load, or forced on system + * shutdown/reboot. When set, tb_stop() asserts DPR on connected + * downstream ports to signal disconnect before tearing down the + * router tree. Only Thunderbolt 3 devices are reset; USB4 + * routers are skipped. */ struct tb_nhi { spinlock_t lock; - struct pci_dev *pdev; + struct device *dev; const struct tb_nhi_ops *ops; void __iomem *iobase; struct tb_ring **tx_rings; struct tb_ring **rx_rings; - struct ida msix_ida; bool going_away; bool iommu_dma_protection; struct work_struct interrupt_work; u32 hop_count; unsigned long quirks; + struct completion domain_released; + bool host_reset; }; /** @@ -535,6 +562,9 @@ struct tb_nhi { * @start_poll: Called when ring interrupt is triggered to start * polling. Passing %NULL keeps the ring in interrupt mode. * @poll_data: Data passed to @start_poll + * @interval_nsec: Interval counter if interrupt throttling is to be + * used with this ring (in ns) + * @wait: Used to signal that the ring may be empty now */ struct tb_ring { spinlock_t lock; @@ -558,6 +588,8 @@ struct tb_ring { u16 eof_mask; void (*start_poll)(void *data); void *poll_data; + unsigned int interval_nsec; + wait_queue_head_t wait; }; /* Leave ring interrupt enabled on suspend */ @@ -609,7 +641,20 @@ struct ring_frame { }; /* Minimum size for ring_rx */ -#define TB_FRAME_SIZE 0x100 +#define TB_FRAME_SIZE 256 +#define TB_MAX_FRAME_SIZE 4096 + +static inline size_t tb_ring_frame_size(const struct ring_frame *frame) +{ + if (frame->size) + return frame->size; + return TB_MAX_FRAME_SIZE; +} + +static inline size_t tb_ring_size(const struct tb_ring *ring) +{ + return ring->size; +} struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size, unsigned int flags); @@ -618,6 +663,7 @@ struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size, u16 sof_mask, u16 eof_mask, void (*start_poll)(void *), void *poll_data); void tb_ring_start(struct tb_ring *ring); +bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec); void tb_ring_stop(struct tb_ring *ring); void tb_ring_free(struct tb_ring *ring); @@ -670,6 +716,8 @@ static inline int tb_ring_tx(struct tb_ring *ring, struct ring_frame *frame) struct ring_frame *tb_ring_poll(struct tb_ring *ring); void tb_ring_poll_complete(struct tb_ring *ring); +int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec); + /** * tb_ring_dma_device() - Return device used for DMA mapping * @ring: Ring whose DMA device is retrieved @@ -681,12 +729,17 @@ void tb_ring_poll_complete(struct tb_ring *ring); */ static inline struct device *tb_ring_dma_device(struct tb_ring *ring) { - return &ring->nhi->pdev->dev; + return ring->nhi->dev; } bool usb4_usb3_port_match(struct device *usb4_port_dev, const struct fwnode_handle *usb3_port_fwnode); +#if IS_REACHABLE(CONFIG_CONFIGFS_FS) +int tb_configfs_register_group(struct config_group *group); +void tb_configfs_unregister_group(struct config_group *group); +#endif + #else /* CONFIG_USB4 */ static inline bool usb4_usb3_port_match(struct device *usb4_port_dev, const struct fwnode_handle *usb3_port_fwnode) diff --git a/include/linux/types.h b/include/linux/types.h index 608050dbca6a..bc5dda2a3d86 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -163,6 +163,8 @@ typedef u32 dma_addr_t; typedef unsigned int __bitwise gfp_t; typedef unsigned int __bitwise slab_flags_t; typedef unsigned int __bitwise fmode_t; +typedef unsigned int __bitwise blk_mode_t; +typedef unsigned int __bitwise fop_flags_t; #ifdef CONFIG_PHYS_ADDR_T_64BIT typedef u64 phys_addr_t; @@ -224,6 +226,12 @@ struct ustat { char f_fpack[6]; }; +struct kcov_common_handle_id { +#ifdef CONFIG_KCOV + u64 val; +#endif +}; + /** * struct callback_head - callback structure for use with RCU and task_work * @next: next update requests in a list diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 8a264662b242..eddbbb65ccc4 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -84,7 +84,7 @@ * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) * that are used instead. Out of those, __... ones are inlined. Plain * copy_{to,from}_user() might or might not be inlined. If you want them - * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. + * inlined, have asm/uaccess.h define INLINE_COPY_USER. * * NOTE: only copy_from_user() zero-pads the destination in case of short copy. * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything @@ -157,7 +157,7 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) } /* - * Architectures that #define INLINE_COPY_TO_USER use this function + * Architectures that #define INLINE_COPY_USER use this function * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. @@ -190,10 +190,6 @@ _inline_copy_from_user(void *to, const void __user *from, unsigned long n) memset(to + (n - res), 0, res); return res; } -#ifndef INLINE_COPY_FROM_USER -extern __must_check unsigned long -_copy_from_user(void *, const void __user *, unsigned long); -#endif static inline __must_check unsigned long _inline_copy_to_user(void __user *to, const void *from, unsigned long n) @@ -207,7 +203,13 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n) } return n; } -#ifndef INLINE_COPY_TO_USER +#ifdef INLINE_COPY_USER +# define _copy_to_user _inline_copy_to_user +# define _copy_from_user _inline_copy_from_user +#else +extern __must_check unsigned long +_copy_from_user(void *, const void __user *, unsigned long); + extern __must_check unsigned long _copy_to_user(void __user *, const void *, unsigned long); #endif @@ -217,11 +219,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n) { if (!check_copy_size(to, n, false)) return n; -#ifdef INLINE_COPY_FROM_USER - return _inline_copy_from_user(to, from, n); -#else return _copy_from_user(to, from, n); -#endif } static __always_inline unsigned long __must_check @@ -229,12 +227,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n) { if (!check_copy_size(from, n, true)) return n; - -#ifdef INLINE_COPY_TO_USER - return _inline_copy_to_user(to, from, n); -#else return _copy_to_user(to, from, n); -#endif } #ifndef copy_mc_to_kernel diff --git a/include/linux/ulpi/driver.h b/include/linux/ulpi/driver.h index a8cb617a3028..c668d9c8d876 100644 --- a/include/linux/ulpi/driver.h +++ b/include/linux/ulpi/driver.h @@ -2,7 +2,7 @@ #ifndef __LINUX_ULPI_DRIVER_H #define __LINUX_ULPI_DRIVER_H -#include +#include #include diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 5e6b212a2aed..64fa44fe180c 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -66,9 +66,6 @@ int utf8_strncasecmp_folded(const struct unicode_map *um, const struct qstr *cf, const struct qstr *s1); -int utf8_normalize(const struct unicode_map *um, const struct qstr *str, - unsigned char *dest, size_t dlen); - int utf8_casefold(const struct unicode_map *um, const struct qstr *str, unsigned char *dest, size_t dlen); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec..18be159bbc34 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -186,9 +186,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +235,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/linux/usb.h b/include/linux/usb.h index 25a203ac7a7e..1da4ad1610bc 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -2,7 +2,7 @@ #ifndef __LINUX_USB_H #define __LINUX_USB_H -#include +#include #include #define USB_MAJOR 180 diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 75b2b763f1ba..534e6650e2aa 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -13,6 +13,7 @@ #include #include #include +#include /* The maximum number of ports one device can grab at once */ #define MAX_NUM_PORTS 16 diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index b90cc5cfff8d..ef21ead551be 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -3,7 +3,7 @@ #ifndef __USB_TYPEC_ALTMODE_H #define __USB_TYPEC_ALTMODE_H -#include +#include #include #include diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 5fc6ce4dd786..9a1674c152aa 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -101,6 +101,9 @@ struct vfio_pci_core_device { const struct vfio_pci_device_ops *pci_ops; void __iomem *barmap[PCI_STD_NUM_BARS]; bool bar_mmap_supported[PCI_STD_NUM_BARS]; + /* Flags modified at runtime - dedicated storage unit */ + bool virq_disabled; + bool bardirty; u8 *pci_config_map; u8 *vconfig; struct perm_bits *msi_perm; @@ -115,18 +118,21 @@ struct vfio_pci_core_device { u16 msix_size; u32 msix_offset; u32 rbar[7]; + /* Flags only modified on setup/release - bitfield ok */ bool has_dyn_msix:1; bool pci_2_3:1; - bool virq_disabled:1; bool reset_works:1; bool extended_caps:1; - bool bardirty:1; bool has_vga:1; - bool needs_reset:1; bool nointx:1; bool needs_pm_restore:1; - bool pm_intx_masked:1; - bool pm_runtime_engaged:1; + bool disable_idle_d3:1; + bool nointxmask:1; + bool disable_vga:1; + /* Flags modified at runtime - dedicated storage unit */ + bool needs_reset; + bool pm_intx_masked; + bool pm_runtime_engaged; bool sriov_active; struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; @@ -157,8 +163,6 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data); -void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, - bool is_disable_idle_d3); void vfio_pci_core_close_device(struct vfio_device *core_vdev); int vfio_pci_core_init_dev(struct vfio_device *core_vdev); void vfio_pci_core_release_dev(struct vfio_device *core_vdev); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index bf089e51970e..93e573c56563 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index d87dc7f77f4e..e4d8d0a9f30f 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -310,14 +310,14 @@ static inline void set_vm_flush_reset_perms(void *addr) {} #if defined(CONFIG_MMU) && defined(CONFIG_SMP) struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align); + size_t align, gfp_t gfp); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); # else static inline struct vm_struct ** pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { return NULL; } diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h index faecd5522401..b4d13457bc2a 100644 --- a/include/linux/vmpressure.h +++ b/include/linux/vmpressure.h @@ -13,18 +13,31 @@ struct vmpressure { unsigned long scanned; unsigned long reclaimed; + /* The lock is used to keep the scanned/reclaimed in sync. */ + spinlock_t sr_lock; +#ifdef CONFIG_MEMCG_V1 + /* + * tree=true accumulators feed the v1 userspace eventfd interface + * (memory.pressure_level). Drained by @work. v2 has no equivalent + * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds. + */ unsigned long tree_scanned; unsigned long tree_reclaimed; - /* The lock is used to keep the scanned/reclaimed above in sync. */ - spinlock_t sr_lock; - /* The list of vmpressure_event structs. */ struct list_head events; /* Have to grab the lock on events traversal or modifications. */ struct mutex events_lock; struct work_struct work; +#endif +}; + +enum vmpressure_levels { + VMPRESSURE_LOW = 0, + VMPRESSURE_MEDIUM, + VMPRESSURE_CRITICAL, + VMPRESSURE_NUM_LEVELS, }; struct mem_cgroup; @@ -32,18 +45,41 @@ struct mem_cgroup; #ifdef CONFIG_MEMCG void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed); -extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); - extern void vmpressure_init(struct vmpressure *vmpr); extern void vmpressure_cleanup(struct vmpressure *vmpr); extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg); extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr); + +/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */ +extern const unsigned long vmpressure_win; +extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed); + +#ifdef CONFIG_MEMCG_V1 +extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); extern int vmpressure_register_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); extern void vmpressure_unregister_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); + +/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */ +extern void vmpressure_v1_init(struct vmpressure *vmpr); +extern void vmpressure_v1_cleanup(struct vmpressure *vmpr); +extern void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed); #else +static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, + int prio) {} +static inline void vmpressure_v1_init(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) {} +#endif /* CONFIG_MEMCG_V1 */ + +#else /* !CONFIG_MEMCG */ static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, unsigned long scanned, unsigned long reclaimed) {} diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 9dc25b04a119..82825e775499 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -42,9 +42,15 @@ extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset); extern void vtime_account_softirq(struct task_struct *tsk); extern void vtime_account_hardirq(struct task_struct *tsk); extern void vtime_flush(struct task_struct *tsk); +#ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE +static inline void vtime_reset(void) { } +static inline void vtime_dyntick_start(void) { } +static inline void vtime_dyntick_stop(void) { } +#else extern void vtime_reset(void); extern void vtime_dyntick_start(void); extern void vtime_dyntick_stop(void); +#endif #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static inline void vtime_account_irq(struct task_struct *tsk, unsigned int offset) { } static inline void vtime_account_softirq(struct task_struct *tsk) { } diff --git a/include/linux/wmi.h b/include/linux/wmi.h index d723e4b1cafb..defcb624a7e2 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include /** diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 62552a2ce5b9..b749a9a5a5ee 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, struct inode *inode); /** - * wbc_init_bio - writeback specific initializtion of bio + * wbc_init_bio - writeback specific initialization of bio * @wbc: writeback_control for the writeback in progress * @bio: bio to be initialized * diff --git a/include/linux/zorro.h b/include/linux/zorro.h index f36c8d39553d..4514c3109deb 100644 --- a/include/linux/zorro.h +++ b/include/linux/zorro.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include diff --git a/include/media/media-entity.h b/include/media/media-entity.h index d9b72cd87d52..fa393e840669 100644 --- a/include/media/media-entity.h +++ b/include/media/media-entity.h @@ -1387,6 +1387,7 @@ void media_remove_intf_links(struct media_interface *intf); * @entity: entity where the @operation will be called * @operation: type of the operation. Should be the name of a member of * struct &media_entity_operations. + * @args: arguments for the operation. * * This helper function will check if @operation is not %NULL. On such case, * it will issue a call to @operation\(@entity, @args\). diff --git a/include/media/rc-map.h b/include/media/rc-map.h index d90e4611b066..d95ed3e96de2 100644 --- a/include/media/rc-map.h +++ b/include/media/rc-map.h @@ -262,7 +262,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_GENIUS_TVGO_A11MCE "rc-genius-tvgo-a11mce" #define RC_MAP_GOTVIEW7135 "rc-gotview7135" #define RC_MAP_HAUPPAUGE "rc-hauppauge" -#define RC_MAP_HAUPPAUGE_NEW "rc-hauppauge" #define RC_MAP_HISI_POPLAR "rc-hisi-poplar" #define RC_MAP_HISI_TV_DEMO "rc-hisi-tv-demo" #define RC_MAP_IMON_MCE "rc-imon-mce" @@ -309,7 +308,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_PROTEUS_2309 "rc-proteus-2309" #define RC_MAP_PURPLETV "rc-purpletv" #define RC_MAP_PV951 "rc-pv951" -#define RC_MAP_RC5_TV "rc-rc5-tv" #define RC_MAP_RC6_MCE "rc-rc6-mce" #define RC_MAP_REAL_AUDIO_220_32_KEYS "rc-real-audio-220-32-keys" #define RC_MAP_REDDO "rc-reddo" diff --git a/include/net/9p/client.h b/include/net/9p/client.h index 838a94218b59..55c6cb54bd25 100644 --- a/include/net/9p/client.h +++ b/include/net/9p/client.h @@ -192,6 +192,7 @@ struct p9_rdma_opts { * @dfltgid: default numeric groupid to mount hierarchy as * @uid: if %V9FS_ACCESS_SINGLE, the numeric uid which mounted the hierarchy * @session_lock_timeout: retry interval for blocking locks + * @ndentry_timeout_ms: Negative dentry lookup cache retention time in ms * * This strucure holds options which are parsed and will be transferred * to the v9fs_session_info structure when mounted, and therefore largely @@ -203,6 +204,7 @@ struct p9_session_opts { unsigned short debug; unsigned int afid; unsigned int cache; + unsigned int ndentry_timeout_ms; #ifdef CONFIG_9P_FSCACHE char *cachetag; #endif diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 38186a245f14..50f0eef71fb1 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -3413,8 +3413,9 @@ static inline struct hci_iso_hdr *hci_iso_hdr(const struct sk_buff *skb) #define hci_iso_flags_pack(pb, ts) ((pb & 0x03) | ((ts & 0x01) << 2)) /* ISO data length and flags pack/unpack */ -#define hci_iso_data_len_pack(h, f) ((__u16) ((h) | ((f) << 14))) -#define hci_iso_data_len(h) ((h) & 0x3fff) +#define hci_iso_data_len_pack(h, f) ((__u16) (((h) & 0x0fff) | \ + (((f) & 0x3) << 14))) +#define hci_iso_data_len(h) ((h) & 0x0fff) #define hci_iso_data_flags(h) ((h) >> 14) /* codec transport types */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 7e15da47fe3a..4ca09298e11a 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -985,6 +985,7 @@ enum { HCI_CONN_AUTH_FAILURE, HCI_CONN_PER_ADV, HCI_CONN_BIG_CREATED, + HCI_CONN_CREATE, HCI_CONN_CREATE_CIS, HCI_CONN_CREATE_BIG_SYNC, HCI_CONN_BIG_SYNC, diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 73e494b2591d..0756d6fe77d4 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -59,6 +59,7 @@ int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param, u8 event, u32 timeout, struct sock *sk); +int __hci_reset_sync(struct hci_dev *hdev); int hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param, u32 timeout); @@ -84,9 +85,6 @@ void hci_cmd_sync_cancel_entry(struct hci_dev *hdev, struct hci_cmd_sync_work_entry *entry); bool hci_cmd_sync_dequeue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, void *data, hci_cmd_sync_work_destroy_t destroy); -bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev, - hci_cmd_sync_work_func_t func, void *data, - hci_cmd_sync_work_destroy_t destroy); int hci_update_eir_sync(struct hci_dev *hdev); int hci_update_class_sync(struct hci_dev *hdev); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 1640cc9bf83a..ef6ce1c20a4f 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -617,7 +617,8 @@ struct l2cap_chan { struct l2cap_ops { char *name; - struct l2cap_chan *(*new_connection) (struct l2cap_chan *chan); + int (*new_connection)(struct l2cap_chan *chan, + struct l2cap_chan *new_chan); int (*recv) (struct l2cap_chan * chan, struct sk_buff *skb); void (*teardown) (struct l2cap_chan *chan, int err); @@ -882,9 +883,10 @@ static inline __u16 __next_seq(struct l2cap_chan *chan, __u16 seq) return (seq + 1) % (chan->tx_win_max + 1); } -static inline struct l2cap_chan *l2cap_chan_no_new_connection(struct l2cap_chan *chan) +static inline int l2cap_chan_no_new_connection(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - return NULL; + return -EOPNOTSUPP; } static inline int l2cap_chan_no_recv(struct l2cap_chan *chan, struct sk_buff *skb) @@ -961,7 +963,7 @@ int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len, void l2cap_chan_busy(struct l2cap_chan *chan, int busy); void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail); int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator); -void l2cap_chan_set_defaults(struct l2cap_chan *chan); +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan); int l2cap_ertm_init(struct l2cap_chan *chan); void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index 05572c19e14b..ef667dff2972 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -302,8 +302,8 @@ void bond_3ad_state_machine_handler(struct work_struct *); void bond_3ad_initiate_agg_selection(struct bonding *bond, int timeout); void bond_3ad_adapter_speed_duplex_changed(struct slave *slave); void bond_3ad_handle_link_change(struct slave *slave, char link); -int bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info); -int __bond_3ad_get_active_agg_info(struct bonding *bond, +int bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info); +int __bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info); int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond, struct slave *slave); diff --git a/include/net/bonding.h b/include/net/bonding.h index 2c54a36a8477..598d56b1bc97 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -345,14 +345,14 @@ static inline bool bond_mode_uses_primary(int mode) mode == BOND_MODE_ALB; } -static inline bool bond_uses_primary(struct bonding *bond) +static inline bool bond_uses_primary(const struct bonding *bond) { return bond_mode_uses_primary(BOND_MODE(bond)); } -static inline struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond) +static inline struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond) { - struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave); + const struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave); return bond_uses_primary(bond) && slave ? slave->dev : NULL; } @@ -703,7 +703,7 @@ void bond_setup(struct net_device *bond_dev); unsigned int bond_get_num_tx_queues(void); int bond_netlink_init(void); void bond_netlink_fini(void); -struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond); +struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond); const char *bond_slave_link_status(s8 link); struct bond_vlan_tag *bond_verify_device_path(struct net_device *start_dev, struct net_device *end_dev, diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 8188ad200de5..697657ed3319 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -1236,6 +1236,26 @@ ieee80211_chandef_max_power(struct cfg80211_chan_def *chandef) return chandef->chan->max_power; } +/** + * cfg80211_chandef_s1g_pri_width - return S1G primary width in MHz + * + * An S1G interface may have a primary channel width of either 1 + * or 2MHz depending on whether chandef::s1g_primary_2mhz is set. + * + * Note: There is _always_ a 1MHz primary subchannel, regardless + * of the primary width. So chandef::chan always points to this + * 1MHz primary channel. + * + * @chandef: the chandef to use + * + * Returns: width in MHz of the S1G primary channel in use + */ +static inline int +cfg80211_chandef_s1g_pri_width(struct cfg80211_chan_def *chandef) +{ + return chandef->s1g_primary_2mhz ? 2 : 1; +} + /** * cfg80211_any_usable_channels - check for usable channels * @wiphy: the wiphy to check for @@ -5086,8 +5106,8 @@ struct mgmt_frame_regs { * @tdls_mgmt: Transmit a TDLS management frame. * @tdls_oper: Perform a high-level TDLS operation (e.g. TDLS link setup). * - * @probe_client: probe an associated client, must return a cookie that it - * later passes to cfg80211_probe_status(). + * @probe_peer: probe a connected peer (AP: STA MAC required; STA: no MAC), + * must return a cookie that is later passed to cfg80211_probe_status(). * * @set_noack_map: Set the NoAck Map for the TIDs. * @@ -5488,8 +5508,8 @@ struct cfg80211_ops { int (*tdls_oper)(struct wiphy *wiphy, struct net_device *dev, const u8 *peer, enum nl80211_tdls_operation oper); - int (*probe_client)(struct wiphy *wiphy, struct net_device *dev, - const u8 *peer, u64 *cookie); + int (*probe_peer)(struct wiphy *wiphy, struct net_device *dev, + const u8 *peer, u64 *cookie); int (*set_noack_map)(struct wiphy *wiphy, struct net_device *dev, @@ -5690,7 +5710,6 @@ struct cfg80211_ops { * set this flag to update channels on beacon hints. * @WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY: support connection to non-primary link * of an NSTR mobile AP MLD. - * @WIPHY_FLAG_DISABLE_WEXT: disable wireless extensions for this device */ enum wiphy_flags { WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0), @@ -5702,7 +5721,7 @@ enum wiphy_flags { WIPHY_FLAG_4ADDR_STATION = BIT(6), WIPHY_FLAG_CONTROL_PORT_PROTOCOL = BIT(7), WIPHY_FLAG_IBSS_RSN = BIT(8), - WIPHY_FLAG_DISABLE_WEXT = BIT(9), + /* reuse bit 9 */ WIPHY_FLAG_MESH_AUTH = BIT(10), WIPHY_FLAG_SUPPORTS_EXT_KCK_32 = BIT(11), WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY = BIT(12), @@ -7265,7 +7284,7 @@ struct wireless_dev { struct list_head pmsr_list; spinlock_t pmsr_lock; - struct work_struct pmsr_free_wk; + struct wiphy_work pmsr_free_wk; unsigned long unprot_beacon_reported; @@ -9846,15 +9865,17 @@ bool cfg80211_rx_unexpected_4addr_frame(struct net_device *dev, const u8 *addr, /** * cfg80211_probe_status - notify userspace about probe status * @dev: the device the probe was sent on - * @addr: the address of the peer - * @cookie: the cookie filled in @probe_client previously + * @peer: The peer MAC address (or MLD address for MLO) or %NULL if not + * applicable (e.g. for STA/P2P-client) + * @cookie: the cookie filled in @probe_peer previously + * @link_id: The link ID on which the probe was sent (or -1 for non-MLO) * @acked: indicates whether probe was acked or not * @ack_signal: signal strength (in dBm) of the ACK frame. * @is_valid_ack_signal: indicates the ack_signal is valid or not. * @gfp: allocation flags */ -void cfg80211_probe_status(struct net_device *dev, const u8 *addr, - u64 cookie, bool acked, s32 ack_signal, +void cfg80211_probe_status(struct net_device *dev, const u8 *peer, u64 cookie, + int link_id, bool acked, s32 ack_signal, bool is_valid_ack_signal, gfp_t gfp); /** diff --git a/include/net/dst.h b/include/net/dst.h index 307073eae7f8..10b73fb45117 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -95,8 +95,19 @@ struct dst_entry { #ifdef CONFIG_64BIT struct lwtunnel_state *lwtstate; #endif +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + atomic_t dst_trace_seq; +#endif }; +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +void save_dst_trace_buffer(struct dst_entry *dst, int delta); +void dump_dst_trace_buffer(const struct net_device *dev); +#else +static inline void save_dst_trace_buffer(struct dst_entry *dst, int delta) { }; +static inline void dump_dst_trace_buffer(const struct net_device *dev) { }; +#endif + struct dst_metrics { u32 metrics[RTAX_MAX]; refcount_t refcnt; @@ -244,7 +255,10 @@ static inline void dst_hold(struct dst_entry *dst) * the placement of __rcuref in struct dst_entry */ BUILD_BUG_ON(offsetof(struct dst_entry, __rcuref) & 63); - WARN_ON(!rcuref_get(&dst->__rcuref)); + if (!rcuref_get(&dst->__rcuref)) + WARN_ON(1); + else + save_dst_trace_buffer(dst, 1); } static inline void dst_use_noref(struct dst_entry *dst, unsigned long time) @@ -308,7 +322,11 @@ static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb */ static inline bool dst_hold_safe(struct dst_entry *dst) { - return rcuref_get(&dst->__rcuref); + const bool ret = rcuref_get(&dst->__rcuref); + + if (ret) + save_dst_trace_buffer(dst, 1); + return ret; } /** diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h index 1fc2fb03ce3f..f45d1e3163f0 100644 --- a/include/net/dst_metadata.h +++ b/include/net/dst_metadata.h @@ -164,8 +164,11 @@ static inline struct metadata_dst *tun_dst_unclone(struct sk_buff *skb) if (!new_md) return ERR_PTR(-ENOMEM); - memcpy(&new_md->u.tun_info, &md_dst->u.tun_info, - sizeof(struct ip_tunnel_info) + md_size); + /* Copy in two stages to keep the __counted_by happy. */ + new_md->u.tun_info = md_dst->u.tun_info; + memcpy(ip_tunnel_info_opts(&new_md->u.tun_info), + ip_tunnel_info_opts(&md_dst->u.tun_info), md_size); + #ifdef CONFIG_DST_CACHE /* Unclone the dst cache if there is one */ if (new_md->u.tun_info.dst_cache.cache) { diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h index 7dee0ae616e3..c6b94790fa81 100644 --- a/include/net/fib_rules.h +++ b/include/net/fib_rules.h @@ -82,7 +82,7 @@ struct fib_rules_ops { struct fib_rule_hdr *, struct nlattr **, struct netlink_ext_ack *); - int (*delete)(struct fib_rule *); + void (*delete)(struct fib_rule *); int (*compare)(struct fib_rule *, struct fib_rule_hdr *, struct nlattr **); @@ -93,11 +93,13 @@ struct fib_rules_ops { /* Called after modifications to the rules set, must flush * the route cache if one exists. */ void (*flush_cache)(struct fib_rules_ops *ops); + bool (*need_rtnl)(struct net *net); int nlgroup; struct list_head rules_list; struct module *owner; struct net *fro_net; + struct mutex lock; struct rcu_head rcu; }; diff --git a/include/net/gue.h b/include/net/gue.h index dfca298bec9c..caefd6da8693 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -80,7 +80,7 @@ static inline size_t guehdr_flags_len(__be16 flags) static inline size_t guehdr_priv_flags_len(__be32 flags) { - return 0; + return (flags & GUE_PFLAG_REMCSUM) ? GUE_PLEN_REMCSUM : 0; } /* Validate standard and private flags. Returns non-zero (meaning invalid) diff --git a/include/net/ieee80211_radiotap.h b/include/net/ieee80211_radiotap.h index c60867e7e43c..8bbaf77da7cf 100644 --- a/include/net/ieee80211_radiotap.h +++ b/include/net/ieee80211_radiotap.h @@ -95,6 +95,8 @@ enum ieee80211_radiotap_presence { IEEE80211_RADIOTAP_EXT = 31, IEEE80211_RADIOTAP_EHT_USIG = 33, IEEE80211_RADIOTAP_EHT = 34, + IEEE80211_RADIOTAP_UHR_ELR = 37, + IEEE80211_RADIOTAP_UHR = 38, }; /* for IEEE80211_RADIOTAP_FLAGS */ @@ -602,6 +604,194 @@ enum ieee80211_radiotap_eht_usig_tb { IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL = 0xfc000000, }; +/* + * ieee80211_radiotap_uhr_elr - content of UHR-ELR TLV (type 37) + * see https://www.radiotap.org/fields/UHR-ELR for details + */ +struct ieee80211_radiotap_uhr_elr { + __le32 known; + __le32 sig1, sig2, mark; +} __packed; + +enum ieee80211_radiotap_uhr_elr_known { + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_VERSION_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_UL_DL = 0x00000002, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MCS = 0x00000004, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_CODING = 0x00000008, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LENGTH = 0x00000010, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LDPC_EXTRA_OFDM_SYM = 0x00000020, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC = 0x00000040, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_TAIL = 0x00000080, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_STA_ID = 0x00000100, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_DISREGARD = 0x00000200, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC = 0x00000400, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_TAIL = 0x00000800, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC_CHECKED = 0x00001000, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC_CHECKED = 0x00002000, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MARK_BSS_COLOR = 0x00010000, +}; + +enum ieee80211_radiotap_uhr_elr_sig1 { + IEEE80211_RADIOTAP_UHR_ELR_SIG1_VERSION_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_UL_DL = 0x00000002, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_MCS = 0x00000004, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CODING = 0x00000008, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_LENGTH = 0x00001FF0, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_LDPC_EXTRA_OFDM_SYM = 0x00002000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC = 0x0003C000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_TAIL = 0x00FC0000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC_VALID = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_elr_sig2 { + IEEE80211_RADIOTAP_UHR_ELR_SIG2_STA_ID = 0x000007FF, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_DISREGARD = 0x00003800, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC = 0x0003C000, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_TAIL = 0x00FC0000, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC_VALID = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_elr_mark { + IEEE80211_RADIOTAP_UHR_ELR_MARK_BSS_COLOR = 0x0000003F, +}; + +/* + * ieee80211_radiotap_uhr - content of UHR TLV (type 38) + * see https://www.radiotap.org/fields/UHR for details + */ +struct ieee80211_radiotap_uhr { + __le32 known; + __le32 data[9]; + struct { + __le32 known, info; + } user[]; +} __packed; + +enum ieee80211_radiotap_uhr_known { + IEEE80211_RADIOTAP_UHR_KNOWN_SPATIAL_REUSE = 0x00000001, + IEEE80211_RADIOTAP_UHR_KNOWN_GI_LTF_SIZE = 0x00000002, + IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_UHR_LTF_SYMBOLS = 0x00000004, + IEEE80211_RADIOTAP_UHR_KNOWN_LDPC_EXTRA_SYMBOL_SEGMENT = 0x00000008, + IEEE80211_RADIOTAP_UHR_KNOWN_PRE_FEC_PADDING_FACTOR = 0x00000010, + IEEE80211_RADIOTAP_UHR_KNOWN_PE_DISAMBIGUITY = 0x00000020, + IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_OFDMA = 0x00000040, + IEEE80211_RADIOTAP_UHR_KNOWN_CRC1 = 0x00000080, + IEEE80211_RADIOTAP_UHR_KNOWN_TAIL1 = 0x00000100, + IEEE80211_RADIOTAP_UHR_KNOWN_CRC2 = 0x00000200, + IEEE80211_RADIOTAP_UHR_KNOWN_TAIL2 = 0x00000400, + IEEE80211_RADIOTAP_UHR_KNOWN_INTERFERENCE_MITIGATION = 0x00000800, + IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_NON_OFDMA = 0x00001000, + IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_NON_OFDMA_USERS = 0x00002000, + IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_CRC = 0x00004000, + IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_TAIL = 0x00008000, + IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_DRU_SIZE = 0x00010000, + IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_INDEX = 0x00020000, + IEEE80211_RADIOTAP_UHR_KNOWN_DRU_RRU_ALLOC_TB_FMT = 0x00040000, + IEEE80211_RADIOTAP_UHR_KNOWN_PRI80_CHAN_POS = 0x00080000, +}; + +enum ieee80211_radiotap_uhr_data { + /* data[0] */ + IEEE80211_RADIOTAP_UHR_DATA0_SPATIAL_REUSE = 0x0000000F, + IEEE80211_RADIOTAP_UHR_DATA0_GI_LTF_SIZE = 0x00000030, + IEEE80211_RADIOTAP_UHR_DATA0_NUMBER_OF_LTF_SYMBOLS = 0x00000700, + IEEE80211_RADIOTAP_UHR_DATA0_LDPC_EXTRA_SYMBOL_SEGMENT = 0x00000800, + IEEE80211_RADIOTAP_UHR_DATA0_PRE_FEC_PADDING_FACTOR = 0x00003000, + IEEE80211_RADIOTAP_UHR_DATA0_PE_DISAMBIGUITY = 0x00004000, + IEEE80211_RADIOTAP_UHR_DATA0_DISREGARD_OFDMA = 0x00078000, + IEEE80211_RADIOTAP_UHR_DATA0_CRC1 = 0x00780000, + IEEE80211_RADIOTAP_UHR_DATA0_TAIL1 = 0x1f800000, + /* data[1] */ + IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_DRU_SIZE = 0x0000001f, + IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_INDEX = 0x00001fe0, + IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1 = 0x003fe000, + IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1_KNOWN = 0x00400000, + IEEE80211_RADIOTAP_UHR_DATA1_PRI80_CHAN_POS = 0xc0000000, + /* data[2] */ + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2_KNOWN = 0x20000000, + /* data[3] */ + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2_KNOWN = 0x20000000, + /* data[4] */ + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3_KNOWN = 0x20000000, + /* data[5] */ + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5_KNOWN = 0x20000000, + /* data[6] */ + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6_KNOWN = 0x20000000, + /* data[7] */ + IEEE80211_RADIOTAP_UHR_DATA7_CRC2 = 0x0000000f, + IEEE80211_RADIOTAP_UHR_DATA7_TAIL2 = 0x000003f0, + IEEE80211_RADIOTAP_UHR_DATA7_INTERFERENCE_MITIGATION = 0x00000400, + IEEE80211_RADIOTAP_UHR_DATA7_DISREGARD_NON_OFDMA = 0x00001800, + IEEE80211_RADIOTAP_UHR_DATA7_NUMBER_OF_NON_OFDMA_USERS = 0x0000e000, + IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_CRC = 0x000f0000, + IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_TAIL = 0x03f00000, + /* data[8] */ + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_PS_160= 0x00000001, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B0 = 0x00000002, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B7_B1 = 0x000001fc, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_INDICATION = 0x00000200, +}; + +enum ieee80211_radiotap_uhr_user_known { + IEEE80211_RADIOTAP_UHR_USER_KNOWN_STA_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_MCS = 0x00000002, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_NSS = 0x00000004, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM = 0x00000008, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_BF = 0x00000010, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_CODING = 0x00000020, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM_PATTERN = 0x00000040, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_2X_LDPC = 0x00000080, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_SPATIAL_CONFIG = 0x00000100, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DISREGARD = 0x00000200, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_BSS_COLOR_INDICATION = 0x00000400, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_CRC = 0x00000800, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_TAIL = 0x00001000, + /* really 'known' but actual data */ + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_CRC = 0x000f0000, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_TAIL = 0x03f00000, + /* indicates this user was captured */ + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USER_CAPTURED = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_user_info { + IEEE80211_RADIOTAP_UHR_USER_INFO_STA_ID = 0x000007ff, + IEEE80211_RADIOTAP_UHR_USER_INFO_MCS = 0x0000f800, + IEEE80211_RADIOTAP_UHR_USER_INFO_NSS = 0x00070000, + IEEE80211_RADIOTAP_UHR_USER_INFO_SPATIAL_CONFIG = 0x000f0000, + IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM = 0x00100000, + IEEE80211_RADIOTAP_UHR_USER_INFO_DISREGARD = 0x00100000, + IEEE80211_RADIOTAP_UHR_USER_INFO_BF = 0x00200000, + IEEE80211_RADIOTAP_UHR_USER_INFO_BSS_COLOR_INDICATION = 0x00200000, + IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM_PATTERN = 0x00c00000, + IEEE80211_RADIOTAP_UHR_USER_INFO_CODING = 0x01000000, + IEEE80211_RADIOTAP_UHR_USER_INFO_2X_LDPC = 0x02000000, +}; + /** * ieee80211_get_radiotap_len - get radiotap header length * @data: pointer to the header diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index a71a98505650..0a35355fb0f3 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -302,7 +302,8 @@ static inline struct fib_table *fib_get_table(struct net *net, u32 id) &net->ipv4.fib_table_hash[TABLE_LOCAL_INDEX] : &net->ipv4.fib_table_hash[TABLE_MAIN_INDEX]; - tb_hlist = rcu_dereference_rtnl(hlist_first_rcu(ptr)); + /* Only fib4_rules_init() adds fib_table. */ + tb_hlist = rcu_dereference_protected(hlist_first_rcu(ptr), true); return hlist_entry(tb_hlist, struct fib_table, tb_hlist); } @@ -374,7 +375,7 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, struct fib_result *res, unsigned int flags) { struct fib_table *tb; - int err = -ENETUNREACH; + int err = -EAGAIN; flags |= FIB_LOOKUP_NOREF; if (net->ipv4.fib_has_custom_rules) @@ -388,17 +389,16 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, if (tb) err = fib_table_lookup(tb, flp, res, flags); - if (!err) + if (err != -EAGAIN) goto out; tb = rcu_dereference_rtnl(net->ipv4.fib_default); if (tb) err = fib_table_lookup(tb, flp, res, flags); -out: if (err == -EAGAIN) err = -ENETUNREACH; - +out: rcu_read_unlock(); return err; diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 49297fec448a..ed2e9bc1bb4e 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -2123,7 +2123,7 @@ void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, int outin); int ip_vs_confirm_conntrack(struct sk_buff *skb); void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, - struct ip_vs_conn *cp, u_int8_t proto, + struct ip_vs_conn *cp, u8 proto, const __be16 port, int from_rs); void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index 80f50fd0f7ad..c024345c9bd8 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -26,6 +26,7 @@ struct nf_conntrack_expect { possible_net_t net; /* We expect this tuple, with the following mask */ + struct nf_conntrack_tuple master_tuple; struct nf_conntrack_tuple tuple; struct nf_conntrack_tuple_mask mask; @@ -54,8 +55,8 @@ struct nf_conntrack_expect { /* The conntrack of the master connection */ struct nf_conn *master; - /* Timer function; deletes the expectation. */ - struct timer_list timeout; + /* jiffies32 when this expectation expires */ + u32 timeout; #if IS_ENABLED(CONFIG_NF_NAT) union nf_inet_addr saved_addr; @@ -69,6 +70,14 @@ struct nf_conntrack_expect { struct rcu_head rcu; }; +static inline bool nf_ct_exp_is_expired(const struct nf_conntrack_expect *exp) +{ + if (READ_ONCE(exp->flags) & NF_CT_EXPECT_DEAD) + return true; + + return (__s32)(READ_ONCE(exp->timeout) - nfct_time_stamp) <= 0; +} + static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) { return read_pnet(&exp->net); @@ -130,7 +139,6 @@ static inline void nf_ct_unlink_expect(struct nf_conntrack_expect *exp) void nf_ct_remove_expectations(struct nf_conn *ct); void nf_ct_unexpect_related(struct nf_conntrack_expect *exp); -bool nf_ct_remove_expect(struct nf_conntrack_expect *exp); void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, void *data), void *data); void nf_ct_expect_iterate_net(struct net *net, @@ -153,5 +161,8 @@ static inline int nf_ct_expect_related(struct nf_conntrack_expect *expect, return nf_ct_expect_related_report(expect, 0, 0, flags); } +struct nf_conn_help; +void nf_ct_expectation_gc(struct nf_conn_help *master_help); + #endif /*_NF_CONNTRACK_EXPECT_H*/ diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h index 81025101f86d..bc5427d239f4 100644 --- a/include/net/netfilter/nf_conntrack_helper.h +++ b/include/net/netfilter/nf_conntrack_helper.h @@ -43,11 +43,10 @@ struct nf_conntrack_helper { refcount_t ct_refcnt; - /* Tuple of things we will help (compared against server response) */ - struct nf_conntrack_tuple tuple; + u8 nfproto; /* NFPROTO_*, can be NFPROTO_UNSPEC */ + u8 l4proto; /* IPPROTO_UDP/TCP */ - /* Function to call when data passes; return verdict, or -1 to - invalidate. */ + /* Function to call when data passes; return verdict */ int __rcu (*help)(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct, enum ip_conntrack_info conntrackinfo); @@ -94,8 +93,7 @@ struct nf_conntrack_helper *nf_conntrack_helper_try_module_get(const char *name, void nf_conntrack_helper_put(struct nf_conntrack_helper *helper); void nf_ct_helper_init(struct nf_conntrack_helper *helper, - u16 l3num, u16 protonum, const char *name, - u16 default_port, u16 spec_port, u32 id, + u8 l3num, u16 protonum, const char *name, const struct nf_conntrack_expect_policy *exp_pol, u32 expect_class_max, int (*help)(struct sk_buff *skb, unsigned int protoff, @@ -114,6 +112,10 @@ int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int, void nf_conntrack_helpers_unregister(struct nf_conntrack_helper **, unsigned int); +#define nf_conntrack_helper_deprecated(name) \ + pr_warn("The %s conntrack helper is scheduled for removal.\n" \ + "Please contact the netfilter-devel mailing list if you still need this.\n", name) + struct nf_conn_help *nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp); int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 3978c3174cdb..fc3e81c07364 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -18,6 +18,7 @@ struct nf_queue_entry { unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + struct net_device *bridge_dev; struct net_device *physin; struct net_device *physout; #endif diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index f74e63290603..6cf1d910bbf8 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -40,6 +40,8 @@ void nft_meta_set_eval(const struct nft_expr *expr, void nft_meta_set_destroy(const struct nft_ctx *ctx, const struct nft_expr *expr); +int nft_meta_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr); int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 6e27c56514df..59506320558a 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -127,6 +127,7 @@ struct netns_ipv4 { atomic_t fib_num_tclassid_users; #endif struct hlist_head *fib_table_hash; + spinlock_t fib_table_hash_lock; struct sock *fibnl; struct hlist_head *fib_info_hash; unsigned int fib_info_hash_bits; diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h index ec65a8cebb99..2bff41aacc98 100644 --- a/include/net/rtnetlink.h +++ b/include/net/rtnetlink.h @@ -256,6 +256,8 @@ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm, int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer, struct netlink_ext_ack *exterr); struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net); #define MODULE_ALIAS_RTNL_LINK(kind) MODULE_ALIAS("rtnl-link-" kind) diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h index 60b073fd3ed8..d50c27812504 100644 --- a/include/net/sctp/sctp.h +++ b/include/net/sctp/sctp.h @@ -111,7 +111,8 @@ int sctp_transport_lookup_process(sctp_callback_t cb, struct net *net, const union sctp_addr *paddr, void *p, int dif); int sctp_transport_traverse_process(sctp_callback_t cb, sctp_callback_t cb_done, struct net *net, int *pos, void *p); -int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), void *p); +int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), + struct net *net, int *pos, void *p); int sctp_get_sctp_info(struct sock *sk, struct sctp_association *asoc, struct sctp_info *info); diff --git a/include/net/sock.h b/include/net/sock.h index 51185222aac2..a043fd422089 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2206,8 +2206,12 @@ sk_dst_get(const struct sock *sk) rcu_read_lock(); dst = rcu_dereference(sk->sk_dst_cache); - if (dst && !rcuref_get(&dst->__rcuref)) - dst = NULL; + if (dst) { + if (!rcuref_get(&dst->__rcuref)) + dst = NULL; + else + save_dst_trace_buffer(dst, 1); + } rcu_read_unlock(); return dst; } diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h index cb7b82f2cbc7..97754ea0a827 100644 --- a/include/net/tc_act/tc_pedit.h +++ b/include/net/tc_act/tc_pedit.h @@ -37,17 +37,15 @@ static inline bool is_tcf_pedit(const struct tc_action *a) return false; } -static inline int tcf_pedit_nkeys(const struct tc_action *a) +/* Must be called with act->tcfa_lock held to ensure consistency of parallel + * reads of the same action's pedit keys (e.g. flow_offload count vs fill). + * Note, this is only used for pedit offload. + */ +static inline int tcf_pedit_nkeys_locked(const struct tc_action *a) { - struct tcf_pedit_parms *parms; - int nkeys; - - rcu_read_lock(); - parms = to_pedit_parms(a); - nkeys = parms->tcfp_nkeys; - rcu_read_unlock(); - - return nkeys; + lockdep_assert_held(&a->tcfa_lock); + return rcu_dereference_protected(to_pedit(a)->parms, + lockdep_is_held(&a->tcfa_lock))->tcfp_nkeys; } static inline u32 tcf_pedit_htype(const struct tc_action *a, int index) diff --git a/include/net/tcp_ao.h b/include/net/tcp_ao.h index 29fd7b735afa..9a2333e62e99 100644 --- a/include/net/tcp_ao.h +++ b/include/net/tcp_ao.h @@ -145,6 +145,7 @@ struct tcp_ao_info { u32 snd_sne; u32 rcv_sne; refcount_t refcnt; /* Protects twsk destruction */ + struct rcu_head rcu; }; #ifdef CONFIG_TCP_MD5SIG diff --git a/include/net/udp.h b/include/net/udp.h index 8262e2b215b4..1fee17274745 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -430,7 +430,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, netdev_features_t features, bool is_ipv6); int udp_lib_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen); + sockopt_t *opt); int udp_lib_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen, int (*push_pending_frames)(struct sock *)); diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 46c1e499e955..a6d69aaa6cd2 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -162,6 +162,8 @@ struct xfrm_dev_offload { */ struct net_device *real_dev; unsigned long offload_handle; + /* Snapshot the attached device index for dump paths. */ + int ifindex; u8 dir : 2; u8 type : 2; u8 flags : 2; @@ -953,6 +955,9 @@ static inline bool addr_match(const void *token1, const void *token2, unsigned int pdw; unsigned int pbi; + if (prefixlen > 128) + return false; + pdw = prefixlen >> 5; /* num of whole u32 in prefix */ pbi = prefixlen & 0x1f; /* num of bits in incomplete u32 in prefix */ @@ -977,6 +982,10 @@ static inline bool addr4_match(__be32 a1, __be32 a2, u8 prefixlen) /* C99 6.5.7 (3): u32 << 32 is undefined behaviour */ if (sizeof(long) == 4 && prefixlen == 0) return true; + + if (prefixlen > 32) + return false; + return !((a1 ^ a2) & htonl(~0UL << (32 - prefixlen))); } @@ -1260,8 +1269,8 @@ int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb, static inline bool __xfrm_check_nopolicy(struct net *net, struct sk_buff *skb, int dir) { - if (!net->xfrm.policy_count[dir] && !secpath_exists(skb)) - return net->xfrm.policy_default[dir] == XFRM_USERPOLICY_ACCEPT; + if (!READ_ONCE(net->xfrm.policy_count[dir]) && !secpath_exists(skb)) + return READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_ACCEPT; return false; } @@ -1361,8 +1370,8 @@ static inline int xfrm_route_forward(struct sk_buff *skb, unsigned short family) { struct net *net = dev_net(skb->dev); - if (!net->xfrm.policy_count[XFRM_POLICY_OUT] && - net->xfrm.policy_default[XFRM_POLICY_OUT] == XFRM_USERPOLICY_ACCEPT) + if (!READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]) && + READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]) == XFRM_USERPOLICY_ACCEPT) return true; return (skb_dst(skb)->flags & DST_NOXFRM) || diff --git a/include/pcmcia/ds.h b/include/pcmcia/ds.h index b7a8de88b3c0..8ae92602e12f 100644 --- a/include/pcmcia/ds.h +++ b/include/pcmcia/ds.h @@ -14,7 +14,7 @@ #define _LINUX_DS_H #ifdef __KERNEL__ -#include +#include #endif #include diff --git a/include/scsi/scsi_common.h b/include/scsi/scsi_common.h index fb58715fac86..00c8a16d3cd2 100644 --- a/include/scsi/scsi_common.h +++ b/include/scsi/scsi_common.h @@ -10,6 +10,14 @@ #include #include +/* From the standard INQUIRY data description in SPC-6. */ +#define INQUIRY_VENDOR_OFFSET 8 +#define INQUIRY_VENDOR_LEN 8 +#define INQUIRY_MODEL_OFFSET 16 +#define INQUIRY_MODEL_LEN 16 +#define INQUIRY_REVISION_OFFSET 32 +#define INQUIRY_REVISION_LEN 4 + enum scsi_pr_type { SCSI_PR_WRITE_EXCLUSIVE = 0x01, SCSI_PR_EXCLUSIVE_ACCESS = 0x03, diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 9c2a7bbe5891..8694eeadd753 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -137,9 +138,9 @@ struct scsi_device { struct mutex inquiry_mutex; unsigned char inquiry_len; /* valid bytes in 'inquiry' */ unsigned char * inquiry; /* INQUIRY response data */ - const char * vendor; /* [back_compat] point into 'inquiry' ... */ - const char * model; /* ... after scan; point to static string */ - const char * rev; /* ... "nullnullnullnull" before scan */ + char vendor[INQUIRY_VENDOR_LEN + 1]; + char model[INQUIRY_MODEL_LEN + 1]; + char rev[INQUIRY_REVISION_LEN + 1]; #define SCSI_DEFAULT_VPD_LEN 255 /* default SCSI VPD page size (max) */ struct scsi_vpd __rcu *vpd_pg0; @@ -407,7 +408,6 @@ void scsi_attach_vpd(struct scsi_device *sdev); void scsi_cdl_check(struct scsi_device *sdev); int scsi_cdl_enable(struct scsi_device *sdev, bool enable); -extern struct scsi_device *scsi_device_from_queue(struct request_queue *q); extern int __must_check scsi_device_get(struct scsi_device *); extern void scsi_device_put(struct scsi_device *); extern struct scsi_device *scsi_device_lookup(struct Scsi_Host *, diff --git a/include/soc/fsl/phy-fsl-lynx.h b/include/soc/fsl/phy-fsl-lynx.h new file mode 100644 index 000000000000..ff5a7d1835b5 --- /dev/null +++ b/include/soc/fsl/phy-fsl-lynx.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright 2023-2026 NXP */ + +#ifndef __PHY_FSL_LYNX_H_ +#define __PHY_FSL_LYNX_H_ + +enum lynx_lane_mode { + LANE_MODE_UNKNOWN, + LANE_MODE_1000BASEX_SGMII, + LANE_MODE_2500BASEX, + LANE_MODE_QSGMII, + LANE_MODE_10G_QXGMII, + LANE_MODE_10GBASER, + LANE_MODE_USXGMII, + LANE_MODE_25GBASER, + LANE_MODE_MAX, +}; + +static inline bool lynx_lane_mode_uses_gmii_mac(enum lynx_lane_mode mode) +{ + switch (mode) { + case LANE_MODE_1000BASEX_SGMII: + case LANE_MODE_2500BASEX: + case LANE_MODE_QSGMII: + case LANE_MODE_10G_QXGMII: + return true; + default: + return false; + } +} + +static inline bool lynx_lane_mode_uses_xgmii_mac(enum lynx_lane_mode mode) +{ + switch (mode) { + case LANE_MODE_10GBASER: + case LANE_MODE_USXGMII: + return true; + default: + return false; + } +} + +#endif /* __PHY_FSL_LYNX_H_ */ diff --git a/include/soc/rockchip/rk3588_grf.h b/include/soc/rockchip/rk3588_grf.h index 02a7b2432d99..db0092fc66ad 100644 --- a/include/soc/rockchip/rk3588_grf.h +++ b/include/soc/rockchip/rk3588_grf.h @@ -19,4 +19,6 @@ /* Whether the LPDDR5 is in 2:1 (= 0) or 4:1 (= 1) CKR a.k.a. DQS mode */ #define RK3588_PMUGRF_OS_REG6_LP5_CKR BIT(0) +#define RK3588_SYSGRF_SOC_CON6 0x0318 + #endif /* __SOC_RK3588_GRF_H */ diff --git a/include/soc/spacemit/k3-syscon.h b/include/soc/spacemit/k3-syscon.h index 0299bea065a0..a68255dd641f 100644 --- a/include/soc/spacemit/k3-syscon.h +++ b/include/soc/spacemit/k3-syscon.h @@ -168,8 +168,8 @@ #define APMU_CPU_C2_CLK_CTRL 0x394 #define APMU_CPU_C3_CLK_CTRL 0x208 #define APMU_PCIE_CLK_RES_CTRL_A 0x1f0 -#define APMU_PCIE_CLK_RES_CTRL_B 0x1c8 -#define APMU_PCIE_CLK_RES_CTRL_C 0x1d0 +#define APMU_PCIE_CLK_RES_CTRL_B 0x1d0 +#define APMU_PCIE_CLK_RES_CTRL_C 0x1c8 #define APMU_PCIE_CLK_RES_CTRL_D 0x1e0 #define APMU_PCIE_CLK_RES_CTRL_E 0x1e8 #define APMU_EMAC0_CLK_RES_CTRL 0x3e4 diff --git a/include/soc/tegra/pmc.h b/include/soc/tegra/pmc.h index 4bcbf19d75ac..53f6d02889db 100644 --- a/include/soc/tegra/pmc.h +++ b/include/soc/tegra/pmc.h @@ -210,7 +210,6 @@ tegra_pmc_io_pad_power_disable(struct tegra_pmc *pmc, enum tegra_io_pad id) bool tegra_pmc_cpu_is_powered(unsigned int cpuid); int tegra_pmc_cpu_power_on(unsigned int cpuid); int tegra_pmc_cpu_remove_clamping(unsigned int cpuid); -bool tegra_pmc_core_domain_state_synced(void); #if defined(CONFIG_SOC_TEGRA_PMC) && defined(CONFIG_PM_SLEEP) enum tegra_suspend_mode tegra_pmc_get_suspend_mode(void); @@ -230,6 +229,10 @@ static inline void tegra_pmc_enter_suspend_mode(enum tegra_suspend_mode mode) { } #endif +#endif + +#if defined(CONFIG_ARM) && defined(CONFIG_SOC_TEGRA_PMC) +bool tegra_pmc_core_domain_state_synced(void); #else /* needed for COMPILE_TEST */ static inline bool tegra_pmc_core_domain_state_synced(void) diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 17945ab5e6e2..1d05f991f2ce 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -9,7 +9,7 @@ #define __SOUND_HDA_CODEC_H #include -#include +#include #include #include #include @@ -256,6 +256,7 @@ struct hda_codec { unsigned int link_down_at_suspend:1; /* link down at runtime suspend */ unsigned int relaxed_resume:1; /* don't resume forcibly for jack */ unsigned int forced_resume:1; /* forced resume for jack */ + unsigned int acomp_requested_resume:1; /* resume requested by acomp */ unsigned int no_stream_clean_at_suspend:1; /* do not clean streams at suspend */ unsigned int ctl_dev_id:1; /* old control element id build behaviour */ unsigned int eld_jack_detect:1; /* Machine jack-detection by ELD */ diff --git a/include/sound/hdaudio.h b/include/sound/hdaudio.h index f11bfc6b9f42..aa994d6e6d35 100644 --- a/include/sound/hdaudio.h +++ b/include/sound/hdaudio.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/include/sound/soc-component.h b/include/sound/soc-component.h index 11bc9527653f..aa423865dbe7 100644 --- a/include/sound/soc-component.h +++ b/include/sound/soc-component.h @@ -10,6 +10,8 @@ #include +struct device_link; + /* * Component probe and remove ordering levels for components with runtime * dependencies. @@ -216,6 +218,8 @@ struct snd_soc_component { struct list_head card_aux_list; /* for auxiliary bound components */ struct list_head card_list; + struct device_link *card_device_link; + const struct snd_soc_component_driver *driver; struct list_head dai_list; diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h index 79c21966220b..9b28e9aef4f1 100644 --- a/include/sound/soc_sdw_utils.h +++ b/include/sound/soc_sdw_utils.h @@ -44,6 +44,18 @@ struct asoc_sdw_codec_info; +struct asoc_sdw_mc_private { + struct snd_soc_card card; + struct snd_soc_jack sdw_headset; + struct device *headset_codec_dev; /* only one headset per card */ + struct device *amp_dev1, *amp_dev2; + bool append_dai_type; + bool ignore_internal_dmic; + void *private; + unsigned long mc_quirk; + int codec_info_list_count; +}; + struct asoc_sdw_dai_info { const bool direction[2]; /* playback & capture support */ const char *codec_name; @@ -88,25 +100,13 @@ struct asoc_sdw_codec_info { int (*codec_card_late_probe)(struct snd_soc_card *card); - int (*count_sidecar)(struct snd_soc_card *card, + int (*count_sidecar)(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs); int (*add_sidecar)(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, struct snd_soc_codec_conf **codec_conf); }; -struct asoc_sdw_mc_private { - struct snd_soc_card card; - struct snd_soc_jack sdw_headset; - struct device *headset_codec_dev; /* only one headset per card */ - struct device *amp_dev1, *amp_dev2; - bool append_dai_type; - bool ignore_internal_dmic; - void *private; - unsigned long mc_quirk; - int codec_info_list_count; -}; - struct asoc_sdw_endpoint { struct list_head list; @@ -182,7 +182,8 @@ struct asoc_sdw_dailink *asoc_sdw_find_dailink(struct asoc_sdw_dailink *dailinks const struct snd_soc_acpi_endpoint *new); int asoc_sdw_get_dai_type(u32 type); -int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, +int asoc_sdw_parse_sdw_endpoints(struct device *dev, + struct asoc_sdw_mc_private *ctx, struct snd_soc_aux_dev *soc_aux, struct asoc_sdw_dailink *soc_dais, struct asoc_sdw_endpoint *soc_ends, @@ -235,7 +236,7 @@ int asoc_sdw_es9356_amp_init(struct snd_soc_card *card, int asoc_sdw_es9356_exit(struct snd_soc_card *card, struct snd_soc_dai_link *dai_link); /* CS AMP support */ -int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card, +int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs); int asoc_sdw_bridge_cs35l56_add_sidecar(struct snd_soc_card *card, struct snd_soc_dai_link **dai_links, diff --git a/include/sound/sof/dai-amd.h b/include/sound/sof/dai-amd.h index 59cd014392c1..e2107da6558f 100644 --- a/include/sound/sof/dai-amd.h +++ b/include/sound/sof/dai-amd.h @@ -18,6 +18,7 @@ struct sof_ipc_dai_acp_params { uint32_t fsync_rate; /* FSYNC frequency in Hz */ uint32_t tdm_slots; uint32_t tdm_mode; + uint32_t format; } __packed; /* ACPDMIC Configuration Request - SOF_IPC_DAI_AMD_CONFIG */ diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h index 36809f712723..e3fe492e78f5 100644 --- a/include/sound/sof/dai.h +++ b/include/sound/sof/dai.h @@ -90,6 +90,8 @@ enum sof_ipc_dai_type { SOF_DAI_AMD_HS_VIRTUAL, /**< AMD ACP HS VIRTUAL */ SOF_DAI_IMX_MICFIL, /** < i.MX MICFIL PDM */ SOF_DAI_AMD_SDW, /**< AMD ACP SDW */ + SOF_DAI_INTEL_UAOL, /**< Intel UAOL */ + SOF_DAI_AMD_I2S, /**< AMD ACP I2S */ }; /* general purpose DAI configuration */ @@ -121,6 +123,7 @@ struct sof_ipc_dai_config { struct sof_ipc_dai_mtk_afe_params afe; struct sof_ipc_dai_micfil_params micfil; struct sof_ipc_dai_acp_sdw_params acp_sdw; + struct sof_ipc_dai_acp_params acp_i2s; }; } __packed; diff --git a/include/sound/tas2781.h b/include/sound/tas2781.h index d45f805b5d18..b763da613769 100644 --- a/include/sound/tas2781.h +++ b/include/sound/tas2781.h @@ -59,9 +59,9 @@ #define TASDEVICE_CHECKSUM_REG TASDEVICE_REG(0x0, 0x0, 0x7e) /* XM_340 */ -#define TASDEVICE_XM_A1_REG TASDEVICE_REG(0x64, 0x63, 0x3c) +#define TASDEVICE_XM_A1_REG TASDEVICE_REG(0x64, 0x02, 0x4c) /* XM_341 */ -#define TASDEVICE_XM_A2_REG TASDEVICE_REG(0x64, 0x63, 0x38) +#define TASDEVICE_XM_A2_REG TASDEVICE_REG(0x64, 0x02, 0x64) /* Volume control */ #define TAS2563_DVC_LVL TASDEVICE_REG(0x00, 0x02, 0x0c) diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h index 9a0e9f9e1ec4..002b0fc57587 100644 --- a/include/target/target_core_base.h +++ b/include/target/target_core_base.h @@ -8,6 +8,7 @@ #include #include /* struct semaphore */ #include +#include #define TARGET_CORE_VERSION "v5.0" @@ -46,10 +47,6 @@ /* Used by transport_get_inquiry_vpd_device_ident() */ #define INQUIRY_VPD_DEVICE_IDENTIFIER_LEN 254 -#define INQUIRY_VENDOR_LEN 8 -#define INQUIRY_MODEL_LEN 16 -#define INQUIRY_REVISION_LEN 4 - /* Attempts before moving from SHORT to LONG */ #define PYX_TRANSPORT_WINDOW_CLOSED_THRESHOLD 3 #define PYX_TRANSPORT_WINDOW_CLOSED_WAIT_SHORT 3 /* In milliseconds */ diff --git a/include/trace/events/ceph.h b/include/trace/events/ceph.h index 08cb0659fbfc..1b990632f62b 100644 --- a/include/trace/events/ceph.h +++ b/include/trace/events/ceph.h @@ -226,6 +226,73 @@ TRACE_EVENT(ceph_handle_caps, __entry->mseq) ); +/* + * Client reset tracepoints - identify the client by its monitor- + * assigned global_id so traces remain meaningful when kernel pointer + * hashing is enabled. + */ +TRACE_EVENT(ceph_client_reset_schedule, + TP_PROTO(const struct ceph_mds_client *mdsc, const char *reason), + TP_ARGS(mdsc, reason), + TP_STRUCT__entry( + __field(u64, client_id) + __string(reason, reason ? reason : "") + ), + TP_fast_assign( + __entry->client_id = mdsc->fsc->client->monc.auth ? + mdsc->fsc->client->monc.auth->global_id : 0; + __assign_str(reason); + ), + TP_printk("client_id=%llu reason=%s", + __entry->client_id, __get_str(reason)) +); + +TRACE_EVENT(ceph_client_reset_complete, + TP_PROTO(const struct ceph_mds_client *mdsc, int ret), + TP_ARGS(mdsc, ret), + TP_STRUCT__entry( + __field(u64, client_id) + __field(int, ret) + ), + TP_fast_assign( + __entry->client_id = mdsc->fsc->client->monc.auth ? + mdsc->fsc->client->monc.auth->global_id : 0; + __entry->ret = ret; + ), + TP_printk("client_id=%llu ret=%d", __entry->client_id, __entry->ret) +); + +TRACE_EVENT(ceph_client_reset_blocked, + TP_PROTO(const struct ceph_mds_client *mdsc, int blocked_count), + TP_ARGS(mdsc, blocked_count), + TP_STRUCT__entry( + __field(u64, client_id) + __field(int, blocked_count) + ), + TP_fast_assign( + __entry->client_id = mdsc->fsc->client->monc.auth ? + mdsc->fsc->client->monc.auth->global_id : 0; + __entry->blocked_count = blocked_count; + ), + TP_printk("client_id=%llu blocked_count=%d", __entry->client_id, + __entry->blocked_count) +); + +TRACE_EVENT(ceph_client_reset_unblocked, + TP_PROTO(const struct ceph_mds_client *mdsc, int ret), + TP_ARGS(mdsc, ret), + TP_STRUCT__entry( + __field(u64, client_id) + __field(int, ret) + ), + TP_fast_assign( + __entry->client_id = mdsc->fsc->client->monc.auth ? + mdsc->fsc->client->monc.auth->global_id : 0; + __entry->ret = ret; + ), + TP_printk("client_id=%llu ret=%d", __entry->client_id, __entry->ret) +); + #undef EM #undef E_ #endif /* _TRACE_CEPH_H */ diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 78388538acf4..8851727ae162 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -78,9 +78,11 @@ TRACE_EVENT_CONDITION(damos_before_apply, TP_PROTO(unsigned int context_idx, unsigned int scheme_idx, unsigned int target_idx, struct damon_region *r, - unsigned int nr_regions, bool do_trace), + unsigned int nr_accesses, unsigned int nr_regions, + bool do_trace), - TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_regions, do_trace), + TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_accesses, + nr_regions, do_trace), TP_CONDITION(do_trace), @@ -101,7 +103,7 @@ TRACE_EVENT_CONDITION(damos_before_apply, __entry->target_idx = target_idx; __entry->start = r->ar.start; __entry->end = r->ar.end; - __entry->nr_accesses = r->nr_accesses_bp / 10000; + __entry->nr_accesses = nr_accesses; __entry->age = r->age; __entry->nr_regions = nr_regions; ), diff --git a/include/trace/events/erofs.h b/include/trace/events/erofs.h index cd0e3fd8c23f..0a178cb10fb1 100644 --- a/include/trace/events/erofs.h +++ b/include/trace/events/erofs.h @@ -90,7 +90,7 @@ TRACE_EVENT(erofs_read_folio, __field(erofs_nid_t, nid ) __field(int, dir ) __field(pgoff_t, index ) - __field(int, uptodate) + __field(unsigned int, order ) __field(bool, raw ) ), @@ -99,16 +99,15 @@ TRACE_EVENT(erofs_read_folio, __entry->nid = EROFS_I(inode)->nid; __entry->dir = S_ISDIR(inode->i_mode); __entry->index = folio->index; - __entry->uptodate = folio_test_uptodate(folio); + __entry->order = folio_order(folio); __entry->raw = raw; ), - TP_printk("dev = (%d,%d), nid = %llu, %s, index = %lu, uptodate = %d " - "raw = %d", + TP_printk("dev = (%d,%d), nid = %llu, %s, index = %lu, order = %u, raw = %d", show_dev_nid(__entry), show_file_type(__entry->dir), (unsigned long)__entry->index, - __entry->uptodate, + __entry->order, __entry->raw) ); diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index b5188d2671d7..270c1a2c24c4 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -2595,6 +2595,34 @@ DEFINE_EVENT(f2fs_priority_update, f2fs_priority_restore, TP_ARGS(sbi, lock_name, is_write, p, orig_prio, new_prio) ); +TRACE_EVENT(f2fs_fault_report, + + TP_PROTO(struct super_block *sb, unsigned int err_code, + const char *func, unsigned int data), + + TP_ARGS(sb, err_code, func, data), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, err_code) + __string(func, func) + __field(unsigned int, data) + ), + + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->err_code = err_code; + __assign_str(func); + __entry->data = data; + ), + + TP_printk("dev = (%d,%d), err_code = %u, func = %s, data = %u", + show_dev(__entry->dev), + __entry->err_code, + __get_str(func), + __entry->data) +); + #endif /* _TRACE_F2FS_H */ /* This part must be outside protection */ diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index bcdc57eea270..291fae364c62 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -89,40 +89,44 @@ TRACE_EVENT(mm_khugepaged_scan_pmd, TRACE_EVENT(mm_collapse_huge_page, - TP_PROTO(struct mm_struct *mm, int isolated, int status), + TP_PROTO(struct mm_struct *mm, int isolated, int status, unsigned int order), - TP_ARGS(mm, isolated, status), + TP_ARGS(mm, isolated, status, order), TP_STRUCT__entry( __field(struct mm_struct *, mm) __field(int, isolated) __field(int, status) + __field(unsigned int, order) ), TP_fast_assign( __entry->mm = mm; __entry->isolated = isolated; __entry->status = status; + __entry->order = order; ), - TP_printk("mm=%p, isolated=%d, status=%s", + TP_printk("mm=%p, isolated=%d, status=%s, order=%u", __entry->mm, __entry->isolated, - __print_symbolic(__entry->status, SCAN_STATUS)) + __print_symbolic(__entry->status, SCAN_STATUS), + __entry->order) ); TRACE_EVENT(mm_collapse_huge_page_isolate, TP_PROTO(struct folio *folio, int none_or_zero, - int referenced, int status), + int referenced, int status, unsigned int order), - TP_ARGS(folio, none_or_zero, referenced, status), + TP_ARGS(folio, none_or_zero, referenced, status, order), TP_STRUCT__entry( __field(unsigned long, pfn) __field(int, none_or_zero) __field(int, referenced) __field(int, status) + __field(unsigned int, order) ), TP_fast_assign( @@ -130,26 +134,30 @@ TRACE_EVENT(mm_collapse_huge_page_isolate, __entry->none_or_zero = none_or_zero; __entry->referenced = referenced; __entry->status = status; + __entry->order = order; ), - TP_printk("scan_pfn=0x%lx, none_or_zero=%d, referenced=%d, status=%s", + TP_printk("scan_pfn=0x%lx, none_or_zero=%d, referenced=%d, status=%s, order=%u", __entry->pfn, __entry->none_or_zero, __entry->referenced, - __print_symbolic(__entry->status, SCAN_STATUS)) + __print_symbolic(__entry->status, SCAN_STATUS), + __entry->order) ); TRACE_EVENT(mm_collapse_huge_page_swapin, - TP_PROTO(struct mm_struct *mm, int swapped_in, int referenced, int ret), + TP_PROTO(struct mm_struct *mm, int swapped_in, int referenced, int ret, + unsigned int order), - TP_ARGS(mm, swapped_in, referenced, ret), + TP_ARGS(mm, swapped_in, referenced, ret, order), TP_STRUCT__entry( __field(struct mm_struct *, mm) __field(int, swapped_in) __field(int, referenced) __field(int, ret) + __field(unsigned int, order) ), TP_fast_assign( @@ -157,13 +165,15 @@ TRACE_EVENT(mm_collapse_huge_page_swapin, __entry->swapped_in = swapped_in; __entry->referenced = referenced; __entry->ret = ret; + __entry->order = order; ), - TP_printk("mm=%p, swapped_in=%d, referenced=%d, ret=%d", + TP_printk("mm=%p, swapped_in=%d, referenced=%d, ret=%d, order=%u", __entry->mm, __entry->swapped_in, __entry->referenced, - __entry->ret) + __entry->ret, + __entry->order) ); TRACE_EVENT(mm_khugepaged_scan_file, diff --git a/include/trace/events/memory-failure.h b/include/trace/events/memory-failure.h index aa57cc8f896b..7a8ee5d1a44e 100644 --- a/include/trace/events/memory-failure.h +++ b/include/trace/events/memory-failure.h @@ -1,6 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ #undef TRACE_SYSTEM -#define TRACE_SYSTEM memory_failure +/* + * For historical versions, memory_failure_event is in ras subsystem, + * some user programs depend on it. + */ +#define TRACE_SYSTEM ras #define TRACE_INCLUDE_FILE memory-failure #if !defined(_TRACE_MEMORY_FAILURE_H) || defined(TRACE_HEADER_MULTI_READ) diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index cd01dd7b3640..15ee2ef201b5 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -23,7 +23,8 @@ EM( MR_CONTIG_RANGE, "contig_range") \ EM( MR_LONGTERM_PIN, "longterm_pin") \ EM( MR_DEMOTION, "demotion") \ - EMe(MR_DAMON, "damon") + EM( MR_DAMON, "damon") \ + EMe(MR_NEVER, "never_migrated") /* * First define the enums in the above macros to be exported to userspace @@ -51,7 +52,7 @@ TRACE_EVENT(mm_migrate_pages, TP_PROTO(unsigned long succeeded, unsigned long failed, unsigned long thp_succeeded, unsigned long thp_failed, unsigned long thp_split, unsigned long large_folio_split, - enum migrate_mode mode, int reason), + enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed, thp_split, large_folio_split, mode, reason), @@ -64,7 +65,7 @@ TRACE_EVENT(mm_migrate_pages, __field( unsigned long, thp_split) __field( unsigned long, large_folio_split) __field( enum migrate_mode, mode) - __field( int, reason) + __field( enum migrate_reason, reason) ), TP_fast_assign( @@ -91,13 +92,13 @@ TRACE_EVENT(mm_migrate_pages, TRACE_EVENT(mm_migrate_pages_start, - TP_PROTO(enum migrate_mode mode, int reason), + TP_PROTO(enum migrate_mode mode, enum migrate_reason reason), TP_ARGS(mode, reason), TP_STRUCT__entry( __field(enum migrate_mode, mode) - __field(int, reason) + __field(enum migrate_reason, reason) ), TP_fast_assign( diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 171524d3526d..36c3a90f0acc 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -77,6 +77,14 @@ TRACE_EVENT(mm_lru_activate, TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn) ); +DECLARE_TRACE(mm_lru_add_drain, + TP_PROTO(int cpu, unsigned int nr_folios), + TP_ARGS(cpu, nr_folios)); + +DECLARE_TRACE(mm_lru_add_drain_all, + TP_PROTO(bool force_all_cpus), + TP_ARGS(force_all_cpus)); + #endif /* _TRACE_PAGEMAP_H */ /* This part must be outside protection */ diff --git a/include/trace/events/qcom_geni_serial.h b/include/trace/events/qcom_geni_serial.h new file mode 100644 index 000000000000..417ec01f9fc8 --- /dev/null +++ b/include/trace/events/qcom_geni_serial.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM qcom_geni_serial + +#if !defined(_TRACE_QCOM_GENI_SERIAL_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_QCOM_GENI_SERIAL_H + +#include +#include + +TRACE_EVENT(geni_serial_set_termios, + TP_PROTO(struct device *dev, unsigned int baud, + unsigned int bits_per_char, u32 tx_trans_cfg, + u32 tx_parity_cfg, u32 rx_trans_cfg, + u32 rx_parity_cfg, u32 stop_bit_len), + TP_ARGS(dev, baud, bits_per_char, tx_trans_cfg, tx_parity_cfg, + rx_trans_cfg, rx_parity_cfg, stop_bit_len), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(unsigned int, baud) + __field(unsigned int, bits_per_char) + __field(u32, tx_trans_cfg) + __field(u32, tx_parity_cfg) + __field(u32, rx_trans_cfg) + __field(u32, rx_parity_cfg) + __field(u32, stop_bit_len) + ), + + TP_fast_assign(__assign_str(name); + __entry->baud = baud; + __entry->bits_per_char = bits_per_char; + __entry->tx_trans_cfg = tx_trans_cfg; + __entry->tx_parity_cfg = tx_parity_cfg; + __entry->rx_trans_cfg = rx_trans_cfg; + __entry->rx_parity_cfg = rx_parity_cfg; + __entry->stop_bit_len = stop_bit_len; + ), + + TP_printk("%s: baud=%u bpc=%u tx_trans=0x%08x tx_par=0x%08x rx_trans=0x%08x rx_par=0x%08x stop=%u", + __get_str(name), __entry->baud, __entry->bits_per_char, + __entry->tx_trans_cfg, __entry->tx_parity_cfg, + __entry->rx_trans_cfg, __entry->rx_parity_cfg, + __entry->stop_bit_len) +); + +TRACE_EVENT(geni_serial_clk_cfg, + TP_PROTO(struct device *dev, unsigned int desired_rate, + unsigned long clk_rate, unsigned int clk_div, + unsigned int clk_idx), + TP_ARGS(dev, desired_rate, clk_rate, clk_div, clk_idx), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(unsigned int, desired_rate) + __field(unsigned long, clk_rate) + __field(unsigned int, clk_div) + __field(unsigned int, clk_idx) + ), + + TP_fast_assign(__assign_str(name); + __entry->desired_rate = desired_rate; + __entry->clk_rate = clk_rate; + __entry->clk_div = clk_div; + __entry->clk_idx = clk_idx; + ), + + TP_printk("%s: desired_rate=%u clk_rate=%lu clk_div=%u clk_idx=%u", + __get_str(name), __entry->desired_rate, __entry->clk_rate, + __entry->clk_div, __entry->clk_idx) +); + +TRACE_EVENT(geni_serial_irq, + TP_PROTO(struct device *dev, u32 m_irq, u32 s_irq, + u32 dma_tx, u32 dma_rx), + TP_ARGS(dev, m_irq, s_irq, dma_tx, dma_rx), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(u32, m_irq) + __field(u32, s_irq) + __field(u32, dma_tx) + __field(u32, dma_rx) + ), + + TP_fast_assign(__assign_str(name); + __entry->m_irq = m_irq; + __entry->s_irq = s_irq; + __entry->dma_tx = dma_tx; + __entry->dma_rx = dma_rx; + ), + + TP_printk("%s: m_irq=0x%08x s_irq=0x%08x dma_tx=0x%08x dma_rx=0x%08x", + __get_str(name), __entry->m_irq, __entry->s_irq, + __entry->dma_tx, __entry->dma_rx) +); + +DECLARE_EVENT_CLASS(geni_serial_data, + TP_PROTO(struct device *dev, const u8 *buf, unsigned int len), + TP_ARGS(dev, buf, len), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(unsigned int, len) + __dynamic_array(u8, data, len) + ), + + TP_fast_assign(__assign_str(name); + __entry->len = len; + memcpy(__get_dynamic_array(data), buf, len); + ), + + TP_printk("%s: len=%u data=%s", + __get_str(name), __entry->len, + __print_hex(__get_dynamic_array(data), __entry->len)) +); + +DEFINE_EVENT(geni_serial_data, geni_serial_tx_data, + TP_PROTO(struct device *dev, const u8 *buf, unsigned int len), + TP_ARGS(dev, buf, len) +); + +DEFINE_EVENT(geni_serial_data, geni_serial_rx_data, + TP_PROTO(struct device *dev, const u8 *buf, unsigned int len), + TP_ARGS(dev, buf, len) +); + +TRACE_EVENT(geni_serial_set_mctrl, + TP_PROTO(struct device *dev, unsigned int mctrl, + u32 uart_manual_rfr), + TP_ARGS(dev, mctrl, uart_manual_rfr), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(unsigned int, mctrl) + __field(u32, uart_manual_rfr) + ), + + TP_fast_assign(__assign_str(name); + __entry->mctrl = mctrl; + __entry->uart_manual_rfr = uart_manual_rfr; + ), + + TP_printk("%s: mctrl=0x%04x uart_manual_rfr=0x%08x", + __get_str(name), __entry->mctrl, __entry->uart_manual_rfr) +); + +TRACE_EVENT(geni_serial_get_mctrl, + TP_PROTO(struct device *dev, unsigned int mctrl, u32 geni_ios), + TP_ARGS(dev, mctrl, geni_ios), + + TP_STRUCT__entry(__string(name, dev_name(dev)) + __field(unsigned int, mctrl) + __field(u32, geni_ios) + ), + + TP_fast_assign(__assign_str(name); + __entry->mctrl = mctrl; + __entry->geni_ios = geni_ios; + ), + + TP_printk("%s: mctrl=0x%04x geni_ios=0x%08x", + __get_str(name), __entry->mctrl, __entry->geni_ios) +); + +#endif /* _TRACE_QCOM_GENI_SERIAL_H */ + +/* This part must be outside protection */ +#include diff --git a/include/uapi/asm-generic/errno.h b/include/uapi/asm-generic/errno.h index bd78e69e0a43..c84ebf89c8b6 100644 --- a/include/uapi/asm-generic/errno.h +++ b/include/uapi/asm-generic/errno.h @@ -76,7 +76,7 @@ #define ENOPROTOOPT 92 /* Protocol not available */ #define EPROTONOSUPPORT 93 /* Protocol not supported */ #define ESOCKTNOSUPPORT 94 /* Socket type not supported */ -#define EOPNOTSUPP 95 /* Operation not supported on transport endpoint */ +#define EOPNOTSUPP 95 /* Operation not supported */ #define EPFNOSUPPORT 96 /* Protocol family not supported */ #define EAFNOSUPPORT 97 /* Address family not supported by protocol */ #define EADDRINUSE 98 /* Address already in use */ diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 9f3090db2f16..b32c72a662b6 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -58,6 +58,7 @@ extern "C" { #define DRM_AMDGPU_USERQ_SIGNAL 0x17 #define DRM_AMDGPU_USERQ_WAIT 0x18 #define DRM_AMDGPU_GEM_LIST_HANDLES 0x19 +#define DRM_AMDGPU_PROC_OPTIONS 0x1A #define DRM_IOCTL_AMDGPU_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create) #define DRM_IOCTL_AMDGPU_GEM_MMAP DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap) @@ -79,6 +80,7 @@ extern "C" { #define DRM_IOCTL_AMDGPU_USERQ_SIGNAL DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal) #define DRM_IOCTL_AMDGPU_USERQ_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait) #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles) +#define DRM_IOCTL_AMDGPU_PROC_OPTIONS DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_PROC_OPTIONS, struct drm_amdgpu_proc_options) /** * DOC: memory domains @@ -1673,6 +1675,25 @@ struct drm_amdgpu_info_uq_metadata { #define AMDGPU_FAMILY_GC_11_5_4 154 /* GC 11.5.4 */ #define AMDGPU_FAMILY_GC_12_0_0 152 /* GC 12.0.0 */ +/* + * Definition of user options + * + * option: AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY + * 0: Disable sigbus delay - SIGBUS will be raised immediately + * 0xFFFFFFFF: SIGBUS will not be raised + * other: Set the sigbus delay in milliseconds + */ +#define AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY 0 + +#define AMDGPU_PROC_OPTIONS_KFD_SIGBUS_DELAY_DISABLED 0xFFFFFFFFu + +struct drm_amdgpu_proc_options { + __u32 op; + struct { + __u32 value; + } kfd_sigbus_delay; +}; + #if defined(__cplusplus) } #endif diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 381a3e857d4e..bd435effdcee 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -155,6 +155,7 @@ extern "C" { /* Panel type property */ #define DRM_MODE_PANEL_TYPE_UNKNOWN 0 #define DRM_MODE_PANEL_TYPE_OLED 1 +#define DRM_MODE_PANEL_TYPE_LCD 2 /* * DRM_MODE_ROTATE_ diff --git a/include/uapi/drm/ethosu_accel.h b/include/uapi/drm/ethosu_accel.h index af78bb4686d7..5b97d59a7806 100644 --- a/include/uapi/drm/ethosu_accel.h +++ b/include/uapi/drm/ethosu_accel.h @@ -43,6 +43,11 @@ enum drm_ethosu_ioctl_id { /** @DRM_ETHOSU_SUBMIT: Submit a job and BOs to run. */ DRM_ETHOSU_SUBMIT, + + DRM_ETHOSU_PERFMON_CREATE, + DRM_ETHOSU_PERFMON_DESTROY, + DRM_ETHOSU_PERFMON_GET_VALUES, + DRM_ETHOSU_PERFMON_SET_GLOBAL, }; /** @@ -79,6 +84,7 @@ struct drm_ethosu_npu_info { __u32 config; __u32 sram_size; + __u32 pmu_counters; }; /** @@ -220,10 +226,54 @@ struct drm_ethosu_submit { /** Input: Number of jobs passed in. */ __u32 job_count; - /** Reserved, must be zero. */ + /** Input: Id returned by DRM_ETHOSU_PERFMON_CREATE */ + __u32 perfmon_id; +}; + +#define DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS 8 +#define DRM_ETHOSU_MAX_PERF_COUNTERS \ + (DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS + 1) + +struct drm_ethosu_perfmon_create { + __u32 id; + __u32 ncounters; + __u16 counters[DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS]; +}; + +struct drm_ethosu_perfmon_destroy { + __u32 id; __u32 pad; }; +/* + * Returns the values of the performance counters tracked by this + * perfmon (as an array of (ncounters + 1) u64 values). + * + * No implicit synchronization is performed, so the user has to + * guarantee that any jobs using this perfmon have already been + * completed. + */ +struct drm_ethosu_perfmon_get_values { + __u32 id; + __u32 pad; + __u64 values_ptr; +}; + +#define DRM_ETHOSU_PERFMON_CLEAR_GLOBAL 0x0001 + +/** + * struct drm_ethosu_perfmon_set_global - ioctl to define a global performance + * monitor + * + * The global performance monitor will be used for all jobs. If a global + * performance monitor is defined, jobs with a self-defined performance + * monitor won't be allowed. + */ +struct drm_ethosu_perfmon_set_global { + __u32 flags; + __u32 id; +}; + /** * DRM_IOCTL_ETHOSU() - Build a ethosu IOCTL number * @__access: Access type. Must be R, W or RW. @@ -252,6 +302,14 @@ enum { DRM_IOCTL_ETHOSU(WR, CMDSTREAM_BO_CREATE, cmdstream_bo_create), DRM_IOCTL_ETHOSU_SUBMIT = DRM_IOCTL_ETHOSU(WR, SUBMIT, submit), + DRM_IOCTL_ETHOSU_PERFMON_CREATE = + DRM_IOCTL_ETHOSU(WR, PERFMON_CREATE, perfmon_create), + DRM_IOCTL_ETHOSU_PERFMON_DESTROY = + DRM_IOCTL_ETHOSU(WR, PERFMON_DESTROY, perfmon_destroy), + DRM_IOCTL_ETHOSU_PERFMON_GET_VALUES = + DRM_IOCTL_ETHOSU(WR, PERFMON_GET_VALUES, perfmon_get_values), + DRM_IOCTL_ETHOSU_PERFMON_SET_GLOBAL = + DRM_IOCTL_ETHOSU(WR, PERFMON_SET_GLOBAL, perfmon_set_global), }; #if defined(__cplusplus) diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h index 0e455d91e77d..a2ff0f4ec691 100644 --- a/include/uapi/drm/panthor_drm.h +++ b/include/uapi/drm/panthor_drm.h @@ -253,6 +253,9 @@ enum drm_panthor_dev_query_type { * @DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO: Query allowed group priorities information. */ DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO, + + /** @DRM_PANTHOR_DEV_QUERY_MMU_INFO: Query MMU information. */ + DRM_PANTHOR_DEV_QUERY_MMU_INFO, }; /** @@ -487,6 +490,16 @@ struct drm_panthor_timestamp_info { __u64 cpu_timestamp_nsec; }; +/** + * struct drm_panthor_mmu_info - MMU information + * + * Structure grouping all queryable information relating to the MMU. + */ +struct drm_panthor_mmu_info { + /** @page_size_bitmap: Allowed page sizes */ + __u64 page_size_bitmap; +}; + /** * struct drm_panthor_group_priorities_info - Group priorities information * @@ -601,6 +614,18 @@ enum drm_panthor_vm_bind_op_flags { */ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED = 1 << 2, + /** + * @DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE: Sparsely map a virtual memory range + * + * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. + * + * When this flag is set, the whole vm_bind range is mapped over a dummy object in a cyclic + * fashion, and all GPU reads from addresses in the range return undefined values. This flag + * being set means drm_panthor_vm_bind_op::bo_offset and drm_panthor_vm_bind_op::bo_handle + * must both be set to 0. DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC must also be set. + */ + DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE = 1 << 3, + /** * @DRM_PANTHOR_VM_BIND_OP_TYPE_MASK: Mask used to determine the type of operation. */ @@ -664,7 +689,6 @@ struct drm_panthor_vm_bind_op { * This array shall not be empty for sync-only operations. */ struct drm_panthor_obj_array syncs; - }; /** diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 48e9f1fdb78d..509202a7b13e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -2537,21 +2537,21 @@ struct drm_xe_exec_queue_set_property { * Refer to Documentation/netlink/specs/drm_ras.yaml for complete interface specification. * * Node Registration - * ================= + * ----------------- * * The driver registers DRM RAS nodes for each error severity level. * enum drm_xe_ras_error_severity defines the node-id, while DRM_XE_RAS_ERROR_SEVERITY_NAMES maps * node-id to node-name. * * Error Classification - * ==================== + * -------------------- * * Each node contains a list of error counters. Each error is identified by a error-id and * an error-name. enum drm_xe_ras_error_component defines the error-id, while * DRM_XE_RAS_ERROR_COMPONENT_NAMES maps error-id to error-name. * * User Interface - * ============== + * -------------- * * To retrieve error values of a error counter, userspace applications should * follow the below steps: @@ -2589,6 +2589,12 @@ enum drm_xe_ras_error_component { DRM_XE_RAS_ERR_COMP_CORE_COMPUTE = 1, /** @DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: SoC Internal Error */ DRM_XE_RAS_ERR_COMP_SOC_INTERNAL, + /** @DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: Device Memory Error */ + DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY, + /** @DRM_XE_RAS_ERR_COMP_PCIE: PCIe Subsystem Error */ + DRM_XE_RAS_ERR_COMP_PCIE, + /** @DRM_XE_RAS_ERR_COMP_FABRIC: Fabric Subsystem Error */ + DRM_XE_RAS_ERR_COMP_FABRIC, /** @DRM_XE_RAS_ERR_COMP_MAX: Max Error */ DRM_XE_RAS_ERR_COMP_MAX /* non-ABI */ }; @@ -2606,7 +2612,10 @@ enum drm_xe_ras_error_component { */ #define DRM_XE_RAS_ERROR_COMPONENT_NAMES { \ [DRM_XE_RAS_ERR_COMP_CORE_COMPUTE] = "core-compute", \ - [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal" \ + [DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal", \ + [DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY] = "device-memory", \ + [DRM_XE_RAS_ERR_COMP_PCIE] = "pcie", \ + [DRM_XE_RAS_ERR_COMP_FABRIC] = "fabric", \ } #if defined(__cplusplus) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 89b36de5fdbb..2f1d24fef857 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5079,17 +5079,19 @@ union bpf_attr { * Description * Redirect the packet to another net device of index *ifindex*. * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. + * that the redirection happens to the *ifindex*' peer device. If + * *flags* is 0, the netns switch takes place from ingress to + * ingress without going through the CPU's backlog queue. If the + * **BPF_F_EGRESS** flag is provided then redirection happens in + * the egress direction of the peer device. * * *skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during * the netns switch. * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. + * If the *flags* argument is 0, the helper is currently only + * supported for tc BPF program types at the ingress hook and for + * veth and netkit target device types. The peer device must reside + * in a different network namespace. * Return * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. @@ -6336,9 +6338,10 @@ enum { /* Flags for bpf_redirect and bpf_redirect_map helpers */ enum { BPF_F_INGRESS = (1ULL << 0), /* used for skb path */ + BPF_F_EGRESS = (1ULL << 1), /* used for skb path */ BPF_F_BROADCAST = (1ULL << 3), /* used for XDP path */ BPF_F_EXCLUDE_INGRESS = (1ULL << 4), /* used for XDP path */ -#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) +#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) }; #define __bpf_md_ptr(type, name) \ @@ -6839,6 +6842,15 @@ struct bpf_link_info { __u32 flags; __u32 pid; } uprobe_multi; + struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; diff --git a/include/uapi/linux/efs_fs_sb.h b/include/uapi/linux/efs_fs_sb.h deleted file mode 100644 index 6bad29a10faa..000000000000 --- a/include/uapi/linux/efs_fs_sb.h +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * efs_fs_sb.h - * - * Copyright (c) 1999 Al Smith - * - * Portions derived from IRIX header files (c) 1988 Silicon Graphics - */ - -#ifndef __EFS_FS_SB_H__ -#define __EFS_FS_SB_H__ - -#include -#include - -/* EFS superblock magic numbers */ -#define EFS_MAGIC 0x072959 -#define EFS_NEWMAGIC 0x07295a - -#define IS_EFS_MAGIC(x) ((x == EFS_MAGIC) || (x == EFS_NEWMAGIC)) - -#define EFS_SUPER 1 -#define EFS_ROOTINODE 2 - -/* efs superblock on disk */ -struct efs_super { - __be32 fs_size; /* size of filesystem, in sectors */ - __be32 fs_firstcg; /* bb offset to first cg */ - __be32 fs_cgfsize; /* size of cylinder group in bb's */ - __be16 fs_cgisize; /* bb's of inodes per cylinder group */ - __be16 fs_sectors; /* sectors per track */ - __be16 fs_heads; /* heads per cylinder */ - __be16 fs_ncg; /* # of cylinder groups in filesystem */ - __be16 fs_dirty; /* fs needs to be fsck'd */ - __be32 fs_time; /* last super-block update */ - __be32 fs_magic; /* magic number */ - char fs_fname[6]; /* file system name */ - char fs_fpack[6]; /* file system pack name */ - __be32 fs_bmsize; /* size of bitmap in bytes */ - __be32 fs_tfree; /* total free data blocks */ - __be32 fs_tinode; /* total free inodes */ - __be32 fs_bmblock; /* bitmap location. */ - __be32 fs_replsb; /* Location of replicated superblock. */ - __be32 fs_lastialloc; /* last allocated inode */ - char fs_spare[20]; /* space for expansion - MUST BE ZERO */ - __be32 fs_checksum; /* checksum of volume portion of fs */ -}; - -/* efs superblock information in memory */ -struct efs_sb_info { - __u32 fs_magic; /* superblock magic number */ - __u32 fs_start; /* first block of filesystem */ - __u32 first_block; /* first data block in filesystem */ - __u32 total_blocks; /* total number of blocks in filesystem */ - __u32 group_size; /* # of blocks a group consists of */ - __u32 data_free; /* # of free data blocks */ - __u32 inode_free; /* # of free inodes */ - __u16 inode_blocks; /* # of blocks used for inodes in every grp */ - __u16 total_groups; /* # of groups */ -}; - -#endif /* __EFS_FS_SB_H__ */ - diff --git a/include/uapi/linux/fscrypt.h b/include/uapi/linux/fscrypt.h index 3aff99f2696a..84507280b3ea 100644 --- a/include/uapi/linux/fscrypt.h +++ b/include/uapi/linux/fscrypt.h @@ -30,7 +30,6 @@ #define FSCRYPT_MODE_SM4_CTS 8 #define FSCRYPT_MODE_ADIANTUM 9 #define FSCRYPT_MODE_AES_256_HCTR2 10 -/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */ /* * Legacy policy version; ad-hoc KDF and no key verification. diff --git a/include/uapi/linux/if_addr.h b/include/uapi/linux/if_addr.h index aa7958b4e41d..7fb630b7fe31 100644 --- a/include/uapi/linux/if_addr.h +++ b/include/uapi/linux/if_addr.h @@ -36,6 +36,7 @@ enum { IFA_RT_PRIORITY, /* u32, priority/metric for prefix route */ IFA_TARGET_NETNSID, IFA_PROTO, /* u8, address protocol */ + IFA_MC_USERS, /* u32, multicast group users */ __IFA_MAX, }; diff --git a/include/uapi/linux/iio/types.h b/include/uapi/linux/iio/types.h index 6d269b844271..c9295c707041 100644 --- a/include/uapi/linux/iio/types.h +++ b/include/uapi/linux/iio/types.h @@ -53,6 +53,7 @@ enum iio_chan_type { IIO_CHROMATICITY, IIO_ATTENTION, IIO_ALTCURRENT, + IIO_COVERAGE, }; enum iio_modifier { @@ -113,6 +114,7 @@ enum iio_modifier { IIO_MOD_ACTIVE, IIO_MOD_REACTIVE, IIO_MOD_APPARENT, + IIO_MOD_QUATERNION_AXIS, }; enum iio_event_type { diff --git a/include/uapi/linux/liveupdate.h b/include/uapi/linux/liveupdate.h index 30bc66ee9436..4043d4038712 100644 --- a/include/uapi/linux/liveupdate.h +++ b/include/uapi/linux/liveupdate.h @@ -59,6 +59,7 @@ enum { LIVEUPDATE_CMD_SESSION_PRESERVE_FD = LIVEUPDATE_CMD_SESSION_BASE, LIVEUPDATE_CMD_SESSION_RETRIEVE_FD = 0x41, LIVEUPDATE_CMD_SESSION_FINISH = 0x42, + LIVEUPDATE_CMD_SESSION_GET_NAME = 0x43, }; /** @@ -168,7 +169,9 @@ struct liveupdate_session_preserve_fd { * associated with the token and populates the @fd field with a new file * descriptor referencing the restored resource in the current (new) kernel. * This operation must be performed *before* signaling completion via - * %LIVEUPDATE_IOCTL_FINISH. + * %LIVEUPDATE_IOCTL_FINISH. If a retrieve of a token fails, subsequent + * attempts to retrieve the token fail with the same error code. Failed + * retrieves are not retried. * * Return: 0 on success, negative error code on failure (e.g., invalid token). */ @@ -213,4 +216,24 @@ struct liveupdate_session_finish { #define LIVEUPDATE_SESSION_FINISH \ _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_FINISH) +/** + * struct liveupdate_session_get_name - ioctl(LIVEUPDATE_SESSION_GET_NAME) + * @size: Input; sizeof(struct liveupdate_session_get_name) + * @reserved: Input; Must be zero. Reserved for future use. + * @name: Output; A null-terminated string with the full session name. + * + * Retrieves the full name of the session associated with this file descriptor. + * This is useful because the kernel may truncate the name shown in /proc. + * + * Return: 0 on success, negative error code on failure. + */ +struct liveupdate_session_get_name { + __u32 size; + __u32 reserved; + __u8 name[LIVEUPDATE_SESSION_NAME_LENGTH]; +}; + +#define LIVEUPDATE_SESSION_GET_NAME \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_GET_NAME) + #endif /* _UAPI_LIVEUPDATE_H */ diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 6c962d866e86..7f6fc9599693 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -16,7 +16,7 @@ */ /* Policies */ -enum { +enum mempolicy_mode { MPOL_DEFAULT, MPOL_PREFERRED, MPOL_BIND, diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h index 56b6b60a814f..ee51045ae1d6 100644 --- a/include/uapi/linux/netfilter/nf_conntrack_common.h +++ b/include/uapi/linux/netfilter/nf_conntrack_common.h @@ -160,6 +160,7 @@ enum ip_conntrack_expect_events { #define NF_CT_EXPECT_USERSPACE 0x4 #ifdef __KERNEL__ +#define NF_CT_EXPECT_DEAD 0x8 #define NF_CT_EXPECT_MASK (NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE | \ NF_CT_EXPECT_USERSPACE) #endif diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 9998f6c0a665..020387d76412 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -922,13 +922,15 @@ * and wasn't already in a 4-addr VLAN. The event will be sent similarly * to the %NL80211_CMD_UNEXPECTED_FRAME event, to the same listener. * - * @NL80211_CMD_PROBE_CLIENT: Probe an associated station on an AP interface - * by sending a null data frame to it and reporting when the frame is - * acknowledged. This is used to allow timing out inactive clients. Uses - * %NL80211_ATTR_IFINDEX and %NL80211_ATTR_MAC. The command returns a - * direct reply with an %NL80211_ATTR_COOKIE that is later used to match - * up the event with the request. The event includes the same data and - * has %NL80211_ATTR_ACK set if the frame was ACKed. + * @NL80211_CMD_PROBE_PEER: Probe a connected peer by sending a null data + * frame and reporting when the frame is acknowledged. + * In AP/GO mode, %NL80211_ATTR_MAC is required to identify the client. + * In STA/P2P-client mode, %NL80211_ATTR_MAC must be omitted (the AP is + * implied); the driver must advertise %NL80211_EXT_FEATURE_PROBE_AP. + * The command returns a direct reply with an %NL80211_ATTR_COOKIE that + * is later used to match up the event with the request. The event + * includes the same data and has %NL80211_ATTR_ACK set if the frame + * was ACKed. * * @NL80211_CMD_REGISTER_BEACONS: Register this socket to receive beacons from * other BSSes when any interfaces are in AP mode. This helps implement @@ -1558,7 +1560,7 @@ enum nl80211_commands { NL80211_CMD_UNEXPECTED_FRAME, - NL80211_CMD_PROBE_CLIENT, + NL80211_CMD_PROBE_PEER, NL80211_CMD_REGISTER_BEACONS, @@ -1729,6 +1731,7 @@ enum nl80211_commands { #define NL80211_CMD_GET_MESH_PARAMS NL80211_CMD_GET_MESH_CONFIG #define NL80211_CMD_SET_MESH_PARAMS NL80211_CMD_SET_MESH_CONFIG #define NL80211_MESH_SETUP_VENDOR_PATH_SEL_IE NL80211_MESH_SETUP_IE +#define NL80211_CMD_PROBE_CLIENT NL80211_CMD_PROBE_PEER /** * enum nl80211_attrs - nl80211 netlink attributes @@ -3165,6 +3168,23 @@ enum nl80211_commands { * @NL80211_ATTR_NPCA_PRIMARY_FREQ: NPCA primary channel (u32) * @NL80211_ATTR_NPCA_PUNCT_BITMAP: NPCA puncturing bitmap (u32) * + * @NL80211_ATTR_STA_DUMP_LINK_STATS: Request flag for %NL80211_CMD_GET_STATION + * (dump mode only). When set on an MLD station, the dump produces two + * %NL80211_CMD_NEW_STATION messages per station per dump call: + * + * 1. An aggregated-stats message whose top-level %NL80211_ATTR_STA_INFO + * contains MLO-combined statistics (same content as a dump without + * this flag). + * + * 2. For each active link, a per-link message containing + * %NL80211_ATTR_MLO_LINKS with a single link entry. Each entry holds + * %NL80211_ATTR_MLO_LINK_ID, the link-specific %NL80211_ATTR_MAC, + * and %NL80211_ATTR_STA_INFO with per-link statistics (see + * &enum nl80211_sta_info). + * + * The aggregated message always precedes the per-link messages for the + * same station within a dump sequence. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -3763,6 +3783,8 @@ enum nl80211_attrs { NL80211_ATTR_NPCA_PRIMARY_FREQ, NL80211_ATTR_NPCA_PUNCT_BITMAP, + NL80211_ATTR_STA_DUMP_LINK_STATS, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -4474,8 +4496,8 @@ enum nl80211_mpath_info { * capabilities IE * @NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE: HE PPE thresholds information as * defined in HE capabilities IE - * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities (__le16), - * given for all 6 GHz band channels + * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities, + * given for all 6 GHz band channels (binary, element content) * @NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS: vendor element capabilities that are * advertised on this band/for this iftype (binary) * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC: EHT MAC capabilities as in EHT @@ -7085,6 +7107,9 @@ enum nl80211_feature_flags { * LTF key seed via %NL80211_KEY_LTF_SEED. The seed is used to generate * secure LTF keys for secure LTF measurement sessions. * + * @NL80211_EXT_FEATURE_PROBE_AP: Driver supports probing the associated AP + * in STA mode using @NL80211_CMD_PROBE_PEER. + * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. */ @@ -7166,6 +7191,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_IEEE8021X_AUTH, NL80211_EXT_FEATURE_ROC_ADDR_FILTER, NL80211_EXT_FEATURE_SET_KEY_LTF_SEED, + NL80211_EXT_FEATURE_PROBE_AP, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/include/uapi/linux/synclink.h b/include/uapi/linux/synclink.h deleted file mode 100644 index 62f32d4e1021..000000000000 --- a/include/uapi/linux/synclink.h +++ /dev/null @@ -1,301 +0,0 @@ -/* SPDX-License-Identifier: GPL-1.0+ WITH Linux-syscall-note */ -/* - * SyncLink Multiprotocol Serial Adapter Driver - * - * $Id: synclink.h,v 3.14 2006/07/17 20:15:43 paulkf Exp $ - * - * Copyright (C) 1998-2000 by Microgate Corporation - * - * Redistribution of this file is permitted under - * the terms of the GNU Public License (GPL) - */ - -#ifndef _UAPI_SYNCLINK_H_ -#define _UAPI_SYNCLINK_H_ -#define SYNCLINK_H_VERSION 3.6 - -#include - -#define BIT0 0x0001 -#define BIT1 0x0002 -#define BIT2 0x0004 -#define BIT3 0x0008 -#define BIT4 0x0010 -#define BIT5 0x0020 -#define BIT6 0x0040 -#define BIT7 0x0080 -#define BIT8 0x0100 -#define BIT9 0x0200 -#define BIT10 0x0400 -#define BIT11 0x0800 -#define BIT12 0x1000 -#define BIT13 0x2000 -#define BIT14 0x4000 -#define BIT15 0x8000 -#define BIT16 0x00010000 -#define BIT17 0x00020000 -#define BIT18 0x00040000 -#define BIT19 0x00080000 -#define BIT20 0x00100000 -#define BIT21 0x00200000 -#define BIT22 0x00400000 -#define BIT23 0x00800000 -#define BIT24 0x01000000 -#define BIT25 0x02000000 -#define BIT26 0x04000000 -#define BIT27 0x08000000 -#define BIT28 0x10000000 -#define BIT29 0x20000000 -#define BIT30 0x40000000 -#define BIT31 0x80000000 - - -#define HDLC_MAX_FRAME_SIZE 65535 -#define MAX_ASYNC_TRANSMIT 4096 -#define MAX_ASYNC_BUFFER_SIZE 4096 - -#define ASYNC_PARITY_NONE 0 -#define ASYNC_PARITY_EVEN 1 -#define ASYNC_PARITY_ODD 2 -#define ASYNC_PARITY_SPACE 3 - -#define HDLC_FLAG_UNDERRUN_ABORT7 0x0000 -#define HDLC_FLAG_UNDERRUN_ABORT15 0x0001 -#define HDLC_FLAG_UNDERRUN_FLAG 0x0002 -#define HDLC_FLAG_UNDERRUN_CRC 0x0004 -#define HDLC_FLAG_SHARE_ZERO 0x0010 -#define HDLC_FLAG_AUTO_CTS 0x0020 -#define HDLC_FLAG_AUTO_DCD 0x0040 -#define HDLC_FLAG_AUTO_RTS 0x0080 -#define HDLC_FLAG_RXC_DPLL 0x0100 -#define HDLC_FLAG_RXC_BRG 0x0200 -#define HDLC_FLAG_RXC_TXCPIN 0x8000 -#define HDLC_FLAG_RXC_RXCPIN 0x0000 -#define HDLC_FLAG_TXC_DPLL 0x0400 -#define HDLC_FLAG_TXC_BRG 0x0800 -#define HDLC_FLAG_TXC_TXCPIN 0x0000 -#define HDLC_FLAG_TXC_RXCPIN 0x0008 -#define HDLC_FLAG_DPLL_DIV8 0x1000 -#define HDLC_FLAG_DPLL_DIV16 0x2000 -#define HDLC_FLAG_DPLL_DIV32 0x0000 -#define HDLC_FLAG_HDLC_LOOPMODE 0x4000 - -#define HDLC_CRC_NONE 0 -#define HDLC_CRC_16_CCITT 1 -#define HDLC_CRC_32_CCITT 2 -#define HDLC_CRC_MASK 0x00ff -#define HDLC_CRC_RETURN_EX 0x8000 - -#define RX_OK 0 -#define RX_CRC_ERROR 1 - -#define HDLC_TXIDLE_FLAGS 0 -#define HDLC_TXIDLE_ALT_ZEROS_ONES 1 -#define HDLC_TXIDLE_ZEROS 2 -#define HDLC_TXIDLE_ONES 3 -#define HDLC_TXIDLE_ALT_MARK_SPACE 4 -#define HDLC_TXIDLE_SPACE 5 -#define HDLC_TXIDLE_MARK 6 -#define HDLC_TXIDLE_CUSTOM_8 0x10000000 -#define HDLC_TXIDLE_CUSTOM_16 0x20000000 - -#define HDLC_ENCODING_NRZ 0 -#define HDLC_ENCODING_NRZB 1 -#define HDLC_ENCODING_NRZI_MARK 2 -#define HDLC_ENCODING_NRZI_SPACE 3 -#define HDLC_ENCODING_NRZI HDLC_ENCODING_NRZI_SPACE -#define HDLC_ENCODING_BIPHASE_MARK 4 -#define HDLC_ENCODING_BIPHASE_SPACE 5 -#define HDLC_ENCODING_BIPHASE_LEVEL 6 -#define HDLC_ENCODING_DIFF_BIPHASE_LEVEL 7 - -#define HDLC_PREAMBLE_LENGTH_8BITS 0 -#define HDLC_PREAMBLE_LENGTH_16BITS 1 -#define HDLC_PREAMBLE_LENGTH_32BITS 2 -#define HDLC_PREAMBLE_LENGTH_64BITS 3 - -#define HDLC_PREAMBLE_PATTERN_NONE 0 -#define HDLC_PREAMBLE_PATTERN_ZEROS 1 -#define HDLC_PREAMBLE_PATTERN_FLAGS 2 -#define HDLC_PREAMBLE_PATTERN_10 3 -#define HDLC_PREAMBLE_PATTERN_01 4 -#define HDLC_PREAMBLE_PATTERN_ONES 5 - -#define MGSL_MODE_ASYNC 1 -#define MGSL_MODE_HDLC 2 -#define MGSL_MODE_MONOSYNC 3 -#define MGSL_MODE_BISYNC 4 -#define MGSL_MODE_RAW 6 -#define MGSL_MODE_BASE_CLOCK 7 -#define MGSL_MODE_XSYNC 8 - -#define MGSL_BUS_TYPE_ISA 1 -#define MGSL_BUS_TYPE_EISA 2 -#define MGSL_BUS_TYPE_PCI 5 - -#define MGSL_INTERFACE_MASK 0xf -#define MGSL_INTERFACE_DISABLE 0 -#define MGSL_INTERFACE_RS232 1 -#define MGSL_INTERFACE_V35 2 -#define MGSL_INTERFACE_RS422 3 -#define MGSL_INTERFACE_RTS_EN 0x10 -#define MGSL_INTERFACE_LL 0x20 -#define MGSL_INTERFACE_RL 0x40 -#define MGSL_INTERFACE_MSB_FIRST 0x80 - -typedef struct _MGSL_PARAMS -{ - /* Common */ - - unsigned long mode; /* Asynchronous or HDLC */ - unsigned char loopback; /* internal loopback mode */ - - /* HDLC Only */ - - unsigned short flags; - unsigned char encoding; /* NRZ, NRZI, etc. */ - unsigned long clock_speed; /* external clock speed in bits per second */ - unsigned char addr_filter; /* receive HDLC address filter, 0xFF = disable */ - unsigned short crc_type; /* None, CRC16-CCITT, or CRC32-CCITT */ - unsigned char preamble_length; - unsigned char preamble; - - /* Async Only */ - - unsigned long data_rate; /* bits per second */ - unsigned char data_bits; /* 7 or 8 data bits */ - unsigned char stop_bits; /* 1 or 2 stop bits */ - unsigned char parity; /* none, even, or odd */ - -} MGSL_PARAMS, *PMGSL_PARAMS; - -#define MICROGATE_VENDOR_ID 0x13c0 -#define SYNCLINK_DEVICE_ID 0x0010 -#define MGSCC_DEVICE_ID 0x0020 -#define SYNCLINK_SCA_DEVICE_ID 0x0030 -#define SYNCLINK_GT_DEVICE_ID 0x0070 -#define SYNCLINK_GT4_DEVICE_ID 0x0080 -#define SYNCLINK_AC_DEVICE_ID 0x0090 -#define SYNCLINK_GT2_DEVICE_ID 0x00A0 -#define MGSL_MAX_SERIAL_NUMBER 30 - -/* -** device diagnostics status -*/ - -#define DiagStatus_OK 0 -#define DiagStatus_AddressFailure 1 -#define DiagStatus_AddressConflict 2 -#define DiagStatus_IrqFailure 3 -#define DiagStatus_IrqConflict 4 -#define DiagStatus_DmaFailure 5 -#define DiagStatus_DmaConflict 6 -#define DiagStatus_PciAdapterNotFound 7 -#define DiagStatus_CantAssignPciResources 8 -#define DiagStatus_CantAssignPciMemAddr 9 -#define DiagStatus_CantAssignPciIoAddr 10 -#define DiagStatus_CantAssignPciIrq 11 -#define DiagStatus_MemoryError 12 - -#define SerialSignal_DCD 0x01 /* Data Carrier Detect */ -#define SerialSignal_TXD 0x02 /* Transmit Data */ -#define SerialSignal_RI 0x04 /* Ring Indicator */ -#define SerialSignal_RXD 0x08 /* Receive Data */ -#define SerialSignal_CTS 0x10 /* Clear to Send */ -#define SerialSignal_RTS 0x20 /* Request to Send */ -#define SerialSignal_DSR 0x40 /* Data Set Ready */ -#define SerialSignal_DTR 0x80 /* Data Terminal Ready */ - - -/* - * Counters of the input lines (CTS, DSR, RI, CD) interrupts - */ -struct mgsl_icount { - __u32 cts, dsr, rng, dcd, tx, rx; - __u32 frame, parity, overrun, brk; - __u32 buf_overrun; - __u32 txok; - __u32 txunder; - __u32 txabort; - __u32 txtimeout; - __u32 rxshort; - __u32 rxlong; - __u32 rxabort; - __u32 rxover; - __u32 rxcrc; - __u32 rxok; - __u32 exithunt; - __u32 rxidle; -}; - -struct gpio_desc { - __u32 state; - __u32 smask; - __u32 dir; - __u32 dmask; -}; - -#define DEBUG_LEVEL_DATA 1 -#define DEBUG_LEVEL_ERROR 2 -#define DEBUG_LEVEL_INFO 3 -#define DEBUG_LEVEL_BH 4 -#define DEBUG_LEVEL_ISR 5 - -/* -** Event bit flags for use with MgslWaitEvent -*/ - -#define MgslEvent_DsrActive 0x0001 -#define MgslEvent_DsrInactive 0x0002 -#define MgslEvent_Dsr 0x0003 -#define MgslEvent_CtsActive 0x0004 -#define MgslEvent_CtsInactive 0x0008 -#define MgslEvent_Cts 0x000c -#define MgslEvent_DcdActive 0x0010 -#define MgslEvent_DcdInactive 0x0020 -#define MgslEvent_Dcd 0x0030 -#define MgslEvent_RiActive 0x0040 -#define MgslEvent_RiInactive 0x0080 -#define MgslEvent_Ri 0x00c0 -#define MgslEvent_ExitHuntMode 0x0100 -#define MgslEvent_IdleReceived 0x0200 - -/* Private IOCTL codes: - * - * MGSL_IOCSPARAMS set MGSL_PARAMS structure values - * MGSL_IOCGPARAMS get current MGSL_PARAMS structure values - * MGSL_IOCSTXIDLE set current transmit idle mode - * MGSL_IOCGTXIDLE get current transmit idle mode - * MGSL_IOCTXENABLE enable or disable transmitter - * MGSL_IOCRXENABLE enable or disable receiver - * MGSL_IOCTXABORT abort transmitting frame (HDLC) - * MGSL_IOCGSTATS return current statistics - * MGSL_IOCWAITEVENT wait for specified event to occur - * MGSL_LOOPTXDONE transmit in HDLC LoopMode done - * MGSL_IOCSIF set the serial interface type - * MGSL_IOCGIF get the serial interface type - */ -#define MGSL_MAGIC_IOC 'm' -#define MGSL_IOCSPARAMS _IOW(MGSL_MAGIC_IOC,0,struct _MGSL_PARAMS) -#define MGSL_IOCGPARAMS _IOR(MGSL_MAGIC_IOC,1,struct _MGSL_PARAMS) -#define MGSL_IOCSTXIDLE _IO(MGSL_MAGIC_IOC,2) -#define MGSL_IOCGTXIDLE _IO(MGSL_MAGIC_IOC,3) -#define MGSL_IOCTXENABLE _IO(MGSL_MAGIC_IOC,4) -#define MGSL_IOCRXENABLE _IO(MGSL_MAGIC_IOC,5) -#define MGSL_IOCTXABORT _IO(MGSL_MAGIC_IOC,6) -#define MGSL_IOCGSTATS _IO(MGSL_MAGIC_IOC,7) -#define MGSL_IOCWAITEVENT _IOWR(MGSL_MAGIC_IOC,8,int) -#define MGSL_IOCCLRMODCOUNT _IO(MGSL_MAGIC_IOC,15) -#define MGSL_IOCLOOPTXDONE _IO(MGSL_MAGIC_IOC,9) -#define MGSL_IOCSIF _IO(MGSL_MAGIC_IOC,10) -#define MGSL_IOCGIF _IO(MGSL_MAGIC_IOC,11) -#define MGSL_IOCSGPIO _IOW(MGSL_MAGIC_IOC,16,struct gpio_desc) -#define MGSL_IOCGGPIO _IOR(MGSL_MAGIC_IOC,17,struct gpio_desc) -#define MGSL_IOCWAITGPIO _IOWR(MGSL_MAGIC_IOC,18,struct gpio_desc) -#define MGSL_IOCSXSYNC _IO(MGSL_MAGIC_IOC, 19) -#define MGSL_IOCGXSYNC _IO(MGSL_MAGIC_IOC, 20) -#define MGSL_IOCSXCTRL _IO(MGSL_MAGIC_IOC, 21) -#define MGSL_IOCGXCTRL _IO(MGSL_MAGIC_IOC, 22) - - -#endif /* _UAPI_SYNCLINK_H_ */ diff --git a/include/uapi/linux/userio.h b/include/uapi/linux/userio.h index 74c9951d2cd0..550c7465af1f 100644 --- a/include/uapi/linux/userio.h +++ b/include/uapi/linux/userio.h @@ -2,7 +2,7 @@ /* * userio: virtual serio device support * Copyright (C) 2015 Red Hat - * Copyright (C) 2015 Lyude (Stephen Chandler Paul) + * Copyright (C) 2015 Lyude Paul * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published by the @@ -27,7 +27,10 @@ enum userio_cmd_type { USERIO_CMD_REGISTER = 0, USERIO_CMD_SET_PORT_TYPE = 1, - USERIO_CMD_SEND_INTERRUPT = 2 + USERIO_CMD_SEND_INTERRUPT = 2, + USERIO_CMD_SET_PORT_EXTRA = 3, + USERIO_CMD_SET_PORT_ID = 4, + USERIO_CMD_SET_PORT_PROTO = 5, }; /* diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h index 361eea511c21..8c616895c511 100644 --- a/include/uapi/linux/vduse.h +++ b/include/uapi/linux/vduse.h @@ -14,6 +14,12 @@ #define VDUSE_API_VERSION_1 1 +/* The VDUSE instance expects a request for vq ready */ +#define VDUSE_F_QUEUE_READY 0 + +/* The VDUSE instance expects a request for suspend */ +#define VDUSE_F_SUSPEND 1 + /* * Get the version of VDUSE API that kernel supported (VDUSE_API_VERSION). * This is used for future extension. @@ -33,6 +39,7 @@ * @vq_align: the allocation alignment of virtqueue's metadata * @ngroups: number of vq groups that VDUSE device declares * @nas: number of address spaces that VDUSE device declares + * @vduse_features: VDUSE features * @reserved: for future use, needs to be initialized to zero * @config_size: the size of the configuration space * @config: the buffer of the configuration space @@ -49,7 +56,8 @@ struct vduse_dev_config { __u32 vq_align; __u32 ngroups; /* if VDUSE_API_VERSION >= 1 */ __u32 nas; /* if VDUSE_API_VERSION >= 1 */ - __u32 reserved[11]; + __u64 vduse_features; /* if VDUSE_GET_FEATURES is not EINVAL */ + __u32 reserved[9]; __u32 config_size; __u8 config[]; }; @@ -63,6 +71,9 @@ struct vduse_dev_config { */ #define VDUSE_DESTROY_DEV _IOW(VDUSE_BASE, 0x03, char[VDUSE_NAME_MAX]) +/* Get the VDUSE supported features */ +#define VDUSE_GET_FEATURES _IOR(VDUSE_BASE, 0x04, __u64) + /* The ioctls for VDUSE device (/dev/vduse/$NAME) */ /** @@ -325,6 +336,8 @@ enum vduse_req_type { VDUSE_SET_STATUS, VDUSE_UPDATE_IOTLB, VDUSE_SET_VQ_GROUP_ASID, + VDUSE_SET_VQ_READY, + VDUSE_SUSPEND, }; /** @@ -372,6 +385,15 @@ struct vduse_iova_range_v2 { __u32 padding; }; +/** + * struct vduse_vq_ready - Virtqueue ready request message + * @num: Virtqueue number + */ +struct vduse_vq_ready { + __u32 num; + __u32 ready; +}; + /** * struct vduse_dev_request - control request * @type: request type @@ -382,6 +404,7 @@ struct vduse_iova_range_v2 { * @iova: IOVA range for updating * @iova_v2: IOVA range for updating if API_VERSION >= 1 * @vq_group_asid: ASID of a virtqueue group + * @vq_ready: Virtqueue ready request * @padding: padding * * Structure used by read(2) on /dev/vduse/$NAME. @@ -399,6 +422,10 @@ struct vduse_dev_request { */ struct vduse_iova_range_v2 iova_v2; struct vduse_vq_group_asid vq_group_asid; + + /* Only if VDUSE_F_QUEUE_READY is negotiated */ + struct vduse_vq_ready vq_ready; + __u32 padding[32]; }; }; diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h index aac9782ddc09..3af946ecbac3 100644 --- a/include/uapi/rdma/rdma_netlink.h +++ b/include/uapi/rdma/rdma_netlink.h @@ -604,6 +604,11 @@ enum rdma_nldev_attr { RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES, /* u32 */ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY, /* u64 */ + /* + * Resource summary entry maximum value. + */ + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, /* u64 */ + /* * Always the end */ diff --git a/include/uapi/sound/sof/tokens.h b/include/uapi/sound/sof/tokens.h index f4a7baadb44d..cc694a397987 100644 --- a/include/uapi/sound/sof/tokens.h +++ b/include/uapi/sound/sof/tokens.h @@ -223,6 +223,7 @@ #define SOF_TKN_AMD_ACPI2S_RATE 1700 #define SOF_TKN_AMD_ACPI2S_CH 1701 #define SOF_TKN_AMD_ACPI2S_TDM_MODE 1702 +#define SOF_TKN_AMD_ACPI2S_FORMAT 1703 /* MICFIL PDM */ #define SOF_TKN_IMX_MICFIL_RATE 2000 diff --git a/include/ufs/ufs.h b/include/ufs/ufs.h index 602aa34c9822..0d48e137d66d 100644 --- a/include/ufs/ufs.h +++ b/include/ufs/ufs.h @@ -191,6 +191,8 @@ enum attr_idn { QUERY_ATTR_IDN_WB_BUF_RESIZE_HINT = 0x3C, QUERY_ATTR_IDN_WB_BUF_RESIZE_EN = 0x3D, QUERY_ATTR_IDN_WB_BUF_RESIZE_STATUS = 0x3E, + QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS = 0x47, + QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT = 0x48, }; /* Descriptor idn for Query requests */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index cfbc75d8df83..248d0a5bef40 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -358,6 +358,7 @@ struct ufshcd_tx_eqtr_record { * @eqtr_record: Pointer to TX EQTR record * @is_valid: True if parameter contains valid TX Equalization settings * @is_applied: True if settings have been applied to UniPro of both sides + * @is_trained: True if parameters obtained from TX EQTR procedure */ struct ufshcd_tx_eq_params { struct ufshcd_tx_eq_settings host[UFS_MAX_LANES]; @@ -365,6 +366,7 @@ struct ufshcd_tx_eq_params { struct ufshcd_tx_eqtr_record *eqtr_record; bool is_valid; bool is_applied; + bool is_trained; }; /** @@ -804,6 +806,13 @@ enum ufshcd_quirks { * delay after enabling VCC to ensure it's stable. */ UFSHCD_QUIRK_VCC_ON_DELAY = 1 << 27, + + /* + * This quirk indicates that Host supports TX Equalization Training + * (EQTR) using Adapt L0L1L2L3 length which is larger than what is + * allowed by M-PHY spec ver 6.0. + */ + UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3 = 1 << 28, }; enum ufshcd_caps { @@ -1020,8 +1029,6 @@ enum ufshcd_mcq_opr { * @caps: bitmask with information about UFS controller capabilities * @devfreq: frequency scaling information owned by the devfreq core * @clk_scaling: frequency scaling information owned by the UFS driver - * @system_suspending: system suspend has been started and system resume has - * not yet finished. * @is_sys_suspended: UFS device has been suspended because of system suspend * @urgent_bkops_lvl: keeps track of urgent bkops level for device * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for @@ -1197,7 +1204,6 @@ struct ufs_hba { struct devfreq *devfreq; struct ufs_clk_scaling clk_scaling; - bool system_suspending; bool is_sys_suspended; enum bkops_status urgent_bkops_lvl; diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 5977723fb3b5..09897f76ae07 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -2,7 +2,7 @@ #ifndef __VDSO_DATAPAGE_H #define __VDSO_DATAPAGE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include @@ -176,7 +176,7 @@ enum vdso_pages { VDSO_NR_PAGES }; -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ #ifdef CONFIG_VDSO_GETRANDOM #define __vdso_u_rng_data PROVIDE(vdso_u_rng_data = vdso_u_data + 2 * PAGE_SIZE); @@ -197,6 +197,6 @@ enum vdso_pages { __vdso_u_arch_data \ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_DATAPAGE_H */ diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index a3bf4f1c0d37..65151b681c4f 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -2,7 +2,7 @@ #ifndef __VDSO_HELPERS_H #define __VDSO_HELPERS_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include #include @@ -111,6 +111,6 @@ static __always_inline void vdso_write_end(struct vdso_time_data *vd) vdso_write_seq_end(&vc[CS_RAW]); } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_HELPERS_H */ diff --git a/include/vdso/processor.h b/include/vdso/processor.h index fbe8265ea3c4..cc781912a696 100644 --- a/include/vdso/processor.h +++ b/include/vdso/processor.h @@ -5,10 +5,10 @@ #ifndef __VDSO_PROCESSOR_H #define __VDSO_PROCESSOR_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __VDSO_PROCESSOR_H */ diff --git a/include/vdso/vsyscall.h b/include/vdso/vsyscall.h index b0fdc9c6bf43..c5c2a2c07857 100644 --- a/include/vdso/vsyscall.h +++ b/include/vdso/vsyscall.h @@ -2,13 +2,13 @@ #ifndef __VDSO_VSYSCALL_H #define __VDSO_VSYSCALL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include unsigned long vdso_update_begin(void); void vdso_update_end(unsigned long flags); -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __VDSO_VSYSCALL_H */ diff --git a/include/xen/interface/xen-mca.h b/include/xen/interface/xen-mca.h index 1c9afbe8cc26..8f5815f1d3ab 100644 --- a/include/xen/interface/xen-mca.h +++ b/include/xen/interface/xen-mca.h @@ -50,7 +50,7 @@ /* OUT: There was no machine check data to fetch. */ #define XEN_MC_NODATA 0x2 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* vIRQ injected to Dom0 */ #define VIRQ_MCA VIRQ_ARCH_0 @@ -388,5 +388,5 @@ struct xen_mce_log { #define MCE_GET_LOG_LEN _IOR('M', 2, int) #define MCE_GETCLEAR_FLAGS _IOR('M', 3, int) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __XEN_PUBLIC_ARCH_X86_MCA_H__ */ diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h index 0ca23eca2a9c..40c9793e9880 100644 --- a/include/xen/interface/xen.h +++ b/include/xen/interface/xen.h @@ -337,7 +337,7 @@ #define MMUEXT_MARK_SUPER 19 #define MMUEXT_UNMARK_SUPER 20 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct mmuext_op { unsigned int cmd; union { @@ -415,7 +415,7 @@ DEFINE_GUEST_HANDLE_STRUCT(mmuext_op); #define MAX_VMASST_TYPE 5 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef uint16_t domid_t; @@ -760,11 +760,11 @@ struct tmem_op { DEFINE_GUEST_HANDLE(u64); -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ /* In assembly code we cannot use C numeric constant suffixes. */ #define mk_unsigned_long(x) x -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __XEN_PUBLIC_XEN_H__ */ diff --git a/init/Kconfig b/init/Kconfig index 4403e53876e1..fa42fb264c9c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -190,6 +190,9 @@ config RUSTC_HAS_FILE_WITH_NUL config RUSTC_HAS_FILE_AS_C_STR def_bool RUSTC_VERSION >= 109100 +config RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS + def_bool RUSTC_VERSION >= 109800 + config PAHOLE_VERSION int default "$(PAHOLE_VERSION)" @@ -1061,14 +1064,14 @@ config PAGE_COUNTER bool config CGROUP_FAVOR_DYNMODS - bool "Favor dynamic modification latency reduction by default" - help - This option enables the "favordynmods" mount option by default - which reduces the latencies of dynamic cgroup modifications such - as task migrations and controller on/offs at the cost of making - hot path operations such as forks and exits more expensive. + bool "Favor dynamic modification latency reduction by default" + help + This option enables the "favordynmods" mount option by default + which reduces the latencies of dynamic cgroup modifications such + as task migrations and controller on/offs at the cost of making + hot path operations such as forks and exits more expensive. - Say N if unsure. + Say N if unsure. config MEMCG bool "Memory controller" @@ -1150,7 +1153,7 @@ config GROUP_SCHED_WEIGHT def_bool n config GROUP_SCHED_BANDWIDTH - def_bool n + def_bool n config FAIR_GROUP_SCHED bool "Group scheduling for SCHED_OTHER" @@ -1566,6 +1569,42 @@ config BOOT_CONFIG_EMBED_FILE This bootconfig will be used if there is no initrd or no other bootconfig in the initrd. +config ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG + bool + help + Silent symbol; no C code reads it directly. Architectures + select it once their setup_arch() calls + xbc_prepend_embedded_cmdline() before parse_early_param(). + Its only role is to gate the user-visible + CMDLINE_FROM_BOOTCONFIG option per-arch, the same + ARCH_SUPPORTS_* idiom used by ARCH_SUPPORTS_CFI, etc. + +config CMDLINE_FROM_BOOTCONFIG + bool "Render embedded bootconfig as kernel cmdline at build time" + depends on BOOT_CONFIG_EMBED_FILE != "" + depends on ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG + depends on CMDLINE = "" + default n + help + Render the "kernel" subtree of the embedded bootconfig file into a + flat cmdline string at kernel build time and prepend it to + boot_command_line during early architecture setup. This makes + early_param() handlers (e.g. mem=, earlycon=, loglevel=) see the + values supplied via the embedded bootconfig. + + The runtime bootconfig parser is unaffected, so tree-structured + consumers such as ftrace boot-time tracing keep working. + + Note: when an initrd also carries a bootconfig, its "kernel" + subtree is still parsed at runtime, but the embedded "kernel" + keys remain in boot_command_line for parse_early_param() and + end up later than the initrd keys in saved_command_line, so + parse_args() last-wins favors the embedded values. If you need + initrd to override embedded kernel.* keys, leave this option + off. + + If unsure, say N. + config CMDLINE_LOG_WRAP_IDEAL_LEN int "Length to try to wrap the cmdline when logged at boot" default 1021 @@ -1656,10 +1695,10 @@ config LD_ORPHAN_WARN depends on $(ld-option,--orphan-handling=error) config LD_ORPHAN_WARN_LEVEL - string - depends on LD_ORPHAN_WARN - default "error" if WERROR - default "warn" + string + depends on LD_ORPHAN_WARN + default "error" if WERROR + default "warn" config SYSCTL bool diff --git a/init/init_task.c b/init/init_task.c index b67ef6040a65..ba5c2523f7e0 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -162,6 +162,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { RCU_POINTER_INITIALIZER(cred, &init_cred), .comm = INIT_TASK_COMM, .thread = INIT_THREAD, + .real_fs = &init_fs, .fs = &init_fs, .files = &init_files, #ifdef CONFIG_IO_URING diff --git a/init/initramfs.c b/init/initramfs.c index 20a18fcda48e..4e27b97a8844 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -716,7 +717,7 @@ static void __init populate_initrd_image(char *err) } #endif /* CONFIG_BLK_DEV_RAM */ -static void __init do_populate_rootfs(void *unused, async_cookie_t cookie) +static void __init unpack_initramfs(async_cookie_t cookie) { /* Load the built in initramfs */ char *err = unpack_to_rootfs(__initramfs_start, __initramfs_size); @@ -724,7 +725,7 @@ static void __init do_populate_rootfs(void *unused, async_cookie_t cookie) panic_show_mem("%s", err); /* Failed to decompress INTERNAL initramfs */ if (!initrd_start || IS_ENABLED(CONFIG_INITRAMFS_FORCE)) - goto done; + return; if (IS_ENABLED(CONFIG_BLK_DEV_RAM)) printk(KERN_INFO "Trying to unpack rootfs image as initramfs...\n"); @@ -739,9 +740,14 @@ static void __init do_populate_rootfs(void *unused, async_cookie_t cookie) printk(KERN_EMERG "Initramfs unpacking failed: %s\n", err); #endif } +} -done: - security_initramfs_populated(); +static void __init do_populate_rootfs(void *unused, async_cookie_t cookie) +{ + scoped_with_init_fs() { + unpack_initramfs(cookie); + security_initramfs_populated(); + } /* * If the initrd region is overlapped with crashkernel reserved region, diff --git a/init/initramfs_test.c b/init/initramfs_test.c index bc55306d226d..1fc990a66563 100644 --- a/init/initramfs_test.c +++ b/init/initramfs_test.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -116,38 +117,41 @@ static void __init initramfs_test_extract(struct kunit *test) GFP_KERNEL); len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); - ktime_get_real_ts64(&ts_before); - err = unpack_to_rootfs(cpio_srcbuf, len); - ktime_get_real_ts64(&ts_after); - if (err) { - KUNIT_FAIL(test, "unpack failed %s", err); - goto out; + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + ktime_get_real_ts64(&ts_before); + err = unpack_to_rootfs(cpio_srcbuf, len); + ktime_get_real_ts64(&ts_after); + if (err) { + KUNIT_FAIL(test, "unpack failed %s", err); + goto out; + } + + KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st, 0), 0); + KUNIT_EXPECT_TRUE(test, S_ISREG(st.mode)); + KUNIT_EXPECT_TRUE(test, uid_eq(st.uid, KUIDT_INIT(c[0].uid))); + KUNIT_EXPECT_TRUE(test, gid_eq(st.gid, KGIDT_INIT(c[0].gid))); + KUNIT_EXPECT_EQ(test, st.nlink, 1); + if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) { + KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[0].mtime); + } else { + KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec); + KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec); + } + KUNIT_EXPECT_EQ(test, st.blocks, c[0].filesize); + + KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st, 0), 0); + KUNIT_EXPECT_TRUE(test, S_ISDIR(st.mode)); + if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) { + KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[1].mtime); + } else { + KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec); + KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec); + } + + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + KUNIT_EXPECT_EQ(test, init_rmdir(c[1].fname), 0); } - - KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st, 0), 0); - KUNIT_EXPECT_TRUE(test, S_ISREG(st.mode)); - KUNIT_EXPECT_TRUE(test, uid_eq(st.uid, KUIDT_INIT(c[0].uid))); - KUNIT_EXPECT_TRUE(test, gid_eq(st.gid, KGIDT_INIT(c[0].gid))); - KUNIT_EXPECT_EQ(test, st.nlink, 1); - if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) { - KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[0].mtime); - } else { - KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec); - KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec); - } - KUNIT_EXPECT_EQ(test, st.blocks, c[0].filesize); - - KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st, 0), 0); - KUNIT_EXPECT_TRUE(test, S_ISDIR(st.mode)); - if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) { - KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[1].mtime); - } else { - KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec); - KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec); - } - - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); - KUNIT_EXPECT_EQ(test, init_rmdir(c[1].fname), 0); out: kfree(cpio_srcbuf); } @@ -197,8 +201,11 @@ static void __init initramfs_test_fname_overrun(struct kunit *test) suffix_off--; } - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NOT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NOT_NULL(test, err); + } kfree(cpio_srcbuf); } @@ -233,22 +240,25 @@ static void __init initramfs_test_data(struct kunit *test) len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NULL(test, err); - file = filp_open(c[0].fname, O_RDONLY, 0); - if (IS_ERR(file)) { - KUNIT_FAIL(test, "open failed"); - goto out; - } + file = filp_open(c[0].fname, O_RDONLY, 0); + if (IS_ERR(file)) { + KUNIT_FAIL(test, "open failed"); + goto out; + } - /* read back file contents into @cpio_srcbuf and confirm match */ - len = kernel_read(file, cpio_srcbuf, c[0].filesize, NULL); - KUNIT_EXPECT_EQ(test, len, c[0].filesize); - KUNIT_EXPECT_MEMEQ(test, cpio_srcbuf, c[0].data, len); + /* read back file contents into @cpio_srcbuf and confirm match */ + len = kernel_read(file, cpio_srcbuf, c[0].filesize, NULL); + KUNIT_EXPECT_EQ(test, len, c[0].filesize); + KUNIT_EXPECT_MEMEQ(test, cpio_srcbuf, c[0].data, len); - fput(file); - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + fput(file); + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + } out: kfree(cpio_srcbuf); } @@ -288,25 +298,28 @@ static void __init initramfs_test_csum(struct kunit *test) len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NULL(test, err); - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); - KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0); + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0); - /* mess up the csum and confirm that unpack fails */ - c[0].csum--; - len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); + /* mess up the csum and confirm that unpack fails */ + c[0].csum--; + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NOT_NULL(test, err); + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NOT_NULL(test, err); - /* - * file (with content) is still retained in case of bad-csum abort. - * Perhaps we should change this. - */ - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); - KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), -ENOENT); + /* + * file (with content) is still retained in case of bad-csum abort. + * Perhaps we should change this. + */ + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), -ENOENT); + } kfree(cpio_srcbuf); } @@ -344,17 +357,20 @@ static void __init initramfs_test_hardlink(struct kunit *test) len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NULL(test, err); - KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st0, 0), 0); - KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st1, 0), 0); - KUNIT_EXPECT_EQ(test, st0.ino, st1.ino); - KUNIT_EXPECT_EQ(test, st0.nlink, 2); - KUNIT_EXPECT_EQ(test, st1.nlink, 2); + KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st0, 0), 0); + KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st1, 0), 0); + KUNIT_EXPECT_EQ(test, st0.ino, st1.ino); + KUNIT_EXPECT_EQ(test, st0.nlink, 2); + KUNIT_EXPECT_EQ(test, st1.nlink, 2); - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); - KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0); + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0); + } kfree(cpio_srcbuf); } @@ -387,12 +403,15 @@ static void __init initramfs_test_many(struct kunit *test) } len = p - cpio_srcbuf; - err = unpack_to_rootfs(cpio_srcbuf, len); - KUNIT_EXPECT_NULL(test, err); - - for (i = 0; i < INITRAMFS_TEST_MANY_LIMIT; i++) { - sprintf(thispath, "initramfs_test_many-%d", i); - KUNIT_EXPECT_EQ(test, init_unlink(thispath), 0); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(cpio_srcbuf, len); + KUNIT_EXPECT_NULL(test, err); + + for (i = 0; i < INITRAMFS_TEST_MANY_LIMIT; i++) { + sprintf(thispath, "initramfs_test_many-%d", i); + KUNIT_EXPECT_EQ(test, init_unlink(thispath), 0); + } } kfree(cpio_srcbuf); @@ -439,22 +458,25 @@ static void __init initramfs_test_fname_pad(struct kunit *test) memcpy(tbufs->padded_fname, "padded_fname", sizeof("padded_fname")); len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_srcbuf); - err = unpack_to_rootfs(tbufs->cpio_srcbuf, len); - KUNIT_EXPECT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(tbufs->cpio_srcbuf, len); + KUNIT_EXPECT_NULL(test, err); - file = filp_open(c[0].fname, O_RDONLY, 0); - if (IS_ERR(file)) { - KUNIT_FAIL(test, "open failed"); - goto out; - } + file = filp_open(c[0].fname, O_RDONLY, 0); + if (IS_ERR(file)) { + KUNIT_FAIL(test, "open failed"); + goto out; + } - /* read back file contents into @cpio_srcbuf and confirm match */ - len = kernel_read(file, tbufs->cpio_srcbuf, c[0].filesize, NULL); - KUNIT_EXPECT_EQ(test, len, c[0].filesize); - KUNIT_EXPECT_MEMEQ(test, tbufs->cpio_srcbuf, c[0].data, len); + /* read back file contents into @cpio_srcbuf and confirm match */ + len = kernel_read(file, tbufs->cpio_srcbuf, c[0].filesize, NULL); + KUNIT_EXPECT_EQ(test, len, c[0].filesize); + KUNIT_EXPECT_MEMEQ(test, tbufs->cpio_srcbuf, c[0].data, len); - fput(file); - KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + fput(file); + KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0); + } out: kfree(tbufs); } @@ -495,13 +517,16 @@ static void __init initramfs_test_fname_path_max(struct kunit *test) memcpy(tbufs->fname_ok, "fname_ok", sizeof("fname_ok") - 1); len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_src); - /* unpack skips over fname_oversize instead of returning an error */ - err = unpack_to_rootfs(tbufs->cpio_src, len); - KUNIT_EXPECT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + /* unpack skips over fname_oversize instead of returning an error */ + err = unpack_to_rootfs(tbufs->cpio_src, len); + KUNIT_EXPECT_NULL(test, err); - KUNIT_EXPECT_EQ(test, init_stat("fname_oversize", &st0, 0), -ENOENT); - KUNIT_EXPECT_EQ(test, init_stat("fname_ok", &st1, 0), 0); - KUNIT_EXPECT_EQ(test, init_rmdir("fname_ok"), 0); + KUNIT_EXPECT_EQ(test, init_stat("fname_oversize", &st0, 0), -ENOENT); + KUNIT_EXPECT_EQ(test, init_stat("fname_ok", &st1, 0), 0); + KUNIT_EXPECT_EQ(test, init_rmdir("fname_ok"), 0); + } kfree(tbufs); } @@ -539,8 +564,11 @@ static void __init initramfs_test_hdr_hex(struct kunit *test) /* inject_ox=true to add "0x" cpio field prefixes */ len = fill_cpio(c, ARRAY_SIZE(c), true, tbufs->cpio_src); - err = unpack_to_rootfs(tbufs->cpio_src, len); - KUNIT_EXPECT_NOT_NULL(test, err); + /* Tests run in a nullfs kthread; borrow the init fs for path resolution. */ + scoped_with_init_fs() { + err = unpack_to_rootfs(tbufs->cpio_src, len); + KUNIT_EXPECT_NOT_NULL(test, err); + } kfree(tbufs); } diff --git a/init/main.c b/init/main.c index e363232b428b..2613d3f9b3ce 100644 --- a/init/main.c +++ b/init/main.c @@ -103,6 +103,7 @@ #include #include #include +#include #include #include #include @@ -356,45 +357,33 @@ static char * __init xbc_make_cmdline(const char *key) return new_cmdline; } -static int __init bootconfig_params(char *param, char *val, - const char *unused, void *arg) -{ - if (strcmp(param, "bootconfig") == 0) { - bootconfig_found = true; - } - return 0; -} - static int __init warn_bootconfig(char *str) { - /* The 'bootconfig' has been handled by bootconfig_params(). */ + /* The 'bootconfig' option is handled by setup_boot_config(). */ return 0; } static void __init setup_boot_config(void) { - static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; const char *msg, *data; - int pos, ret; + int pos, ret, offs; size_t size; - char *err; + bool from_embedded = false; /* Cut out the bootconfig data even if we have no bootconfig option */ data = get_boot_config_from_initrd(&size); /* If there is no bootconfig in initrd, try embedded one. */ - if (!data) + if (!data) { data = xbc_get_embedded_bootconfig(&size); + from_embedded = true; + } - strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); - err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL, - bootconfig_params); - - if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE))) + bootconfig_found = bootconfig_cmdline_requested(boot_command_line, &offs); + if (!(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE))) return; - /* parse_args() stops at the next param of '--' and returns an address */ - if (err) - initargs_offs = err - tmp_cmdline; + /* Offset of the init arguments after a "--", located by the helper. */ + initargs_offs = offs; if (!data) { /* If user intended to use bootconfig, show an error level message */ @@ -421,8 +410,24 @@ static void __init setup_boot_config(void) } else { xbc_get_info(&ret, NULL); pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret); - /* keys starting with "kernel." are passed via cmdline */ - extra_command_line = xbc_make_cmdline("kernel"); + /* + * keys starting with "kernel." are passed via cmdline. When + * this bootconfig came from the embedded source and + * setup_arch() already prepended the rendered "kernel" subtree + * to boot_command_line, rendering again here would duplicate + * the keys in saved_command_line and make accumulating handlers + * (console=, earlycon=, ...) re-register the same value. Skip + * only when the prepend really happened. + * + * On arches that do not select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG, + * CONFIG_CMDLINE_FROM_BOOTCONFIG is unselectable and + * xbc_embedded_cmdline_applied() collapses to a stub returning + * false, so this path still runs and the embedded "kernel" + * keys reach the cmdline via the runtime parser exactly as + * before this series. + */ + if (!from_embedded || !xbc_embedded_cmdline_applied()) + extra_command_line = xbc_make_cmdline("kernel"); /* Also, "init." keys are init arguments */ extra_init_args = xbc_make_cmdline("init"); } @@ -670,6 +675,11 @@ static __initdata DECLARE_COMPLETION(kthreadd_done); static noinline void __ref __noreturn rest_init(void) { + struct kernel_clone_args init_args = { + .flags = (CLONE_VM | CLONE_UNTRACED), + .fn = kernel_init, + .fn_arg = NULL, + }; struct task_struct *tsk; int pid; @@ -679,7 +689,7 @@ static noinline void __ref __noreturn rest_init(void) * the init task will end up wanting to create kthreads, which, if * we schedule it before we create kthreadd, will OOPS. */ - pid = user_mode_thread(kernel_init, NULL, CLONE_FS); + pid = kernel_clone(&init_args); /* * Pin init on the boot CPU. Task migration is not properly working * until sched_init_smp() has been run. It will set the allowed @@ -1540,6 +1550,8 @@ static int __ref kernel_init(void *unused) { int ret; + init_userspace_fs(); + /* * Wait until kthreadd is all set-up. */ diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 2e14880eef92..be8d75731d24 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "io-wq.h" #include "slist.h" @@ -643,13 +644,17 @@ static void io_worker_handle_work(struct io_wq_acct *acct, unsigned int hash = __io_wq_is_hashed(work_flags) ? __io_get_work_hash(work_flags) : -1U; + struct io_kiocb *req; next_hashed = wq_next_work(work); if (do_kill && (work_flags & IO_WQ_WORK_UNBOUND)) atomic_or(IO_WQ_WORK_CANCEL, &work->flags); + req = container_of(work, struct io_kiocb, work); + kcov_remote_start_common(req->ctx->kcov_handle); io_wq_submit_work(work); + kcov_remote_stop(); io_assign_current_work(worker, NULL); linked = io_wq_free_work(work); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..1279e27c2c6d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -59,6 +59,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -293,6 +294,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd); io_napi_init(ctx); mutex_init(&ctx->mmap_lock); + ctx->kcov_handle = kcov_common_handle(); return ctx; diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 4f9b439319c4..23e8a85111bc 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -53,7 +53,7 @@ struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages) nr_pages = end - start; if (WARN_ON_ONCE(!nr_pages)) return ERR_PTR(-EINVAL); - if (WARN_ON_ONCE(nr_pages > INT_MAX)) + if (nr_pages > INT_MAX / sizeof(struct page *)) return ERR_PTR(-EOVERFLOW); pages = kvmalloc_objs(struct page *, nr_pages, GFP_KERNEL_ACCOUNT); @@ -337,7 +337,7 @@ unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr, ptr = io_uring_validate_mmap_request(filp, pgoff); if (IS_ERR(ptr)) - return -ENOMEM; + return PTR_ERR(ptr); /* * Some architectures have strong cache aliasing requirements. diff --git a/io_uring/mpscq.h b/io_uring/mpscq.h index c801384c6a0a..f910526766fd 100644 --- a/io_uring/mpscq.h +++ b/io_uring/mpscq.h @@ -122,4 +122,13 @@ static inline struct llist_node *mpscq_pop(struct mpscq *q, return NULL; } +/* + * Returns true if the most recent mpscq_pop() that returned a node also + * emptied the queue. Consumer must be serialized. + */ +static inline bool mpscq_pop_emptied(struct mpscq *q, struct llist_node *head) +{ + return head == &q->stub; +} + #endif /* IOU_MPSCQ_H */ diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 3ff9098573db..3067c9343991 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -93,19 +93,38 @@ static void io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, io_req_task_work_add_remote(req, IOU_F_TWQ_LAZY_WAKE); } +static int io_msg_ring_cqe_flags(struct io_ring_ctx *target_ctx, + const struct io_msg *msg, u32 *flags) +{ + *flags = 0; + + if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS)) + return 0; + + *flags = msg->cqe_flags; + if ((*flags & IORING_CQE_F_32) && + !(target_ctx->flags & (IORING_SETUP_CQE32 | + IORING_SETUP_CQE_MIXED))) + return -EINVAL; + + return 0; +} + static int io_msg_data_remote(struct io_ring_ctx *target_ctx, struct io_msg *msg) { struct io_kiocb *target; - u32 flags = 0; + u32 flags; + int ret; - target = kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO) ; + ret = io_msg_ring_cqe_flags(target_ctx, msg, &flags); + if (ret) + return ret; + + target = kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); if (unlikely(!target)) return -ENOMEM; - if (msg->flags & IORING_MSG_RING_FLAGS_PASS) - flags = msg->cqe_flags; - io_msg_remote_post(target_ctx, target, msg->len, flags, msg->user_data); return 0; } @@ -130,8 +149,9 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, if (io_msg_need_remote(target_ctx)) return io_msg_data_remote(target_ctx, msg); - if (msg->flags & IORING_MSG_RING_FLAGS_PASS) - flags = msg->cqe_flags; + ret = io_msg_ring_cqe_flags(target_ctx, msg, &flags); + if (ret) + return ret; ret = -EOVERFLOW; if (target_ctx->flags & IORING_SETUP_IOPOLL) { diff --git a/io_uring/nop.c b/io_uring/nop.c index 91ae0b2e7e55..60ab19604b36 100644 --- a/io_uring/nop.c +++ b/io_uring/nop.c @@ -40,6 +40,8 @@ int io_nop_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) nop->fd = READ_ONCE(sqe->fd); else nop->fd = -1; + if (nop->flags & IORING_NOP_FIXED_FILE) + req->flags |= REQ_F_FIXED_FILE; if (nop->flags & IORING_NOP_FIXED_BUFFER) req->buf_index = READ_ONCE(sqe->buf_index); if (nop->flags & IORING_NOP_CQE32) { @@ -59,12 +61,10 @@ int io_nop(struct io_kiocb *req, unsigned int issue_flags) int ret = nop->result; if (nop->flags & IORING_NOP_FILE) { - if (nop->flags & IORING_NOP_FIXED_FILE) { + if (req->flags & REQ_F_FIXED_FILE) req->file = io_file_get_fixed(req, nop->fd, issue_flags); - req->flags |= REQ_F_FIXED_FILE; - } else { + else req->file = io_file_get_normal(req, nop->fd); - } if (!req->file) { ret = -EBADF; goto done; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 88a45c7d897f..4e58eb1344ea 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -520,6 +520,7 @@ const struct io_issue_def io_issue_defs[] = { #endif }, [IORING_OP_RECV_ZC] = { + .audit_skip = 1, .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, diff --git a/io_uring/register.c b/io_uring/register.c index dce5e2f9cf77..02bc103bcc9d 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -503,6 +503,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) unsigned i, tail, old_head; struct io_uring_params *p = &config.p; struct io_rings_layout *rl = &config.layout; + u32 *o_sq_array, *n_sq_array = NULL; int ret; memset(&config, 0, sizeof(config)); @@ -589,6 +590,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) ctx->rings = NULL; o.sq_sqes = ctx->sq_sqes; ctx->sq_sqes = NULL; + o_sq_array = ctx->sq_array; + if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) + n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset); /* * Now copy SQ and CQ entries, if any. If either of the destination @@ -599,20 +603,27 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) if (tail - old_head > p->sq_entries) goto overflow; for (i = old_head; i < tail; i++) { - unsigned index, dst_mask, src_mask; + unsigned int dst, src; size_t sq_size; - index = i; + dst = i & (p->sq_entries - 1); + src = i & (ctx->sq_entries - 1); + if (n_sq_array) { + src = READ_ONCE(o_sq_array[src]); + if (unlikely(src >= ctx->sq_entries)) { + WRITE_ONCE(n_sq_array[dst], UINT_MAX); + continue; + } + WRITE_ONCE(n_sq_array[dst], dst); + } + sq_size = sizeof(struct io_uring_sqe); - src_mask = ctx->sq_entries - 1; - dst_mask = p->sq_entries - 1; if (ctx->flags & IORING_SETUP_SQE128) { - index <<= 1; + dst <<= 1; + src <<= 1; sq_size <<= 1; - src_mask = (ctx->sq_entries << 1) - 1; - dst_mask = (p->sq_entries << 1) - 1; } - memcpy(&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size); + memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size); } WRITE_ONCE(n.rings->sq.head, old_head); WRITE_ONCE(n.rings->sq.tail, tail); @@ -655,8 +666,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow)); /* all done, store old pointers and assign new ones */ - if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset); + if (n_sq_array) + ctx->sq_array = n_sq_array; ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries; diff --git a/io_uring/rw.c b/io_uring/rw.c index 0c4834645279..63b6519e498c 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -601,15 +601,15 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) { struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); struct io_kiocb *req = cmd_to_io_kiocb(rw); + int final_res = io_fixup_rw_res(req, res); if (kiocb->ki_flags & IOCB_WRITE) io_req_end_write(req); - if (unlikely(res != req->cqe.res)) { - if (res == -EAGAIN && io_rw_should_reissue(req)) - req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; - else - req->cqe.res = res; - } + + if (res == -EAGAIN && io_rw_should_reissue(req)) + req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; + else if (unlikely(final_res != req->cqe.res)) + req->cqe.res = final_res; /* order with io_iopoll_complete() checking ->iopoll_completed */ smp_store_release(&req->iopoll_completed, 1); diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 2460bd605266..ad42e8eb1002 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -332,10 +333,14 @@ static int io_sq_thread(void *data) cap_entries = !list_is_singular(&sqd->ctx_list); list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { - int ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); + int ret; + + kcov_remote_start_common(ctx->kcov_handle); + ret = __io_sq_thread(ctx, sqd, cap_entries, &ist); if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) sqt_spin = true; + kcov_remote_stop(); } if (io_sq_tw(IORING_TW_CAP_ENTRIES_VALUE)) sqt_spin = true; diff --git a/io_uring/tw.c b/io_uring/tw.c index e74372233f40..a4c872870d81 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -34,10 +34,6 @@ void io_tctx_fallback_work(struct work_struct *work) fallback_work); unsigned int count = 0; - /* see tctx_task_work() - a set bit must always have a run coming */ - clear_bit(0, &tctx->tw_pending); - smp_mb__after_atomic(); - /* * Run the entries directly. We're in PF_KTHRED context, hence * io_should_terminate_tw() is true and they will be marked as @@ -55,7 +51,7 @@ static void io_fallback_tw(struct io_uring_task *tctx) * the queued work) stay around until the drain has run. */ get_task_struct(tctx->task); - if (!queue_work(system_unbound_wq, &tctx->fallback_work)) + if (!queue_work(system_dfl_wq, &tctx->fallback_work)) put_task_struct(tctx->task); } @@ -101,6 +97,13 @@ void tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, io_poll_task_func, io_req_rw_complete, (struct io_tw_req){req}, ts); (*count)++; + /* + * Break if most recent pop emptied the queue. This helps + * bound task_work run, and also protects the regular + * task_work addition. + */ + if (mpscq_pop_emptied(&tctx->task_list, tctx->task_head)) + break; if (unlikely(need_resched())) { ctx_flush_and_put(ctx, ts); ctx = NULL; @@ -127,8 +130,6 @@ void tctx_task_work(struct callback_head *cb) unsigned int count = 0; tctx = container_of(cb, struct io_uring_task, task_work); - clear_bit(0, &tctx->tw_pending); - smp_mb__after_atomic(); tctx_task_work_run(tctx, UINT_MAX, &count); } @@ -206,7 +207,7 @@ void io_req_normal_work_add(struct io_kiocb *req) struct io_uring_task *tctx = req->tctx; struct io_ring_ctx *ctx = req->ctx; - /* task_work already pending, we're done */ + /* tw run already pending, nothing else to do */ if (!mpscq_push(&tctx->task_list, &req->io_task_work.node)) return; @@ -223,10 +224,6 @@ void io_req_normal_work_add(struct io_kiocb *req) return; } - /* task_work must only be added once */ - if (test_and_set_bit(0, &tctx->tw_pending)) - return; - if (likely(!task_work_add(tctx->task, &tctx->task_work, ctx->notify_method))) return; diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 7b25dcd9d05f..c14c22cff49e 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -90,7 +90,7 @@ static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd, } /* - * Mark this command as concelable, then io_uring_try_cancel_uring_cmd() + * Mark this command as cancelable, then io_uring_try_cancel_uring_cmd() * will try to cancel this issued command by sending ->uring_cmd() with * issue_flags of IO_URING_F_CANCEL. * @@ -168,7 +168,7 @@ void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, } io_req_uring_cleanup(req, issue_flags); if (req->flags & REQ_F_IOPOLL) { - /* order with io_iopoll_req_issued() checking ->iopoll_complete */ + /* order with io_do_iopoll() checking ->iopoll_completed */ smp_store_release(&req->iopoll_completed, 1); } else if (issue_flags & IO_URING_F_COMPLETE_DEFER) { if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED)) diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 9b087ebeb643..d038d944257f 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -28,7 +28,7 @@ static int proc_ipc_dointvec_minmax_orphans(const struct ctl_table *table, int w if (err < 0) return err; - if (ns->shm_rmid_forced) + if (write && ns->shm_rmid_forced) shm_destroy_orphaned(ns); return err; } diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 4798b375972b..d1dd36a651b0 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -608,7 +608,7 @@ static int mqueue_create_attr(struct dentry *dentry, umode_t mode, void *arg) } static int mqueue_create(struct mnt_idmap *idmap, struct inode *dir, - struct dentry *dentry, umode_t mode, bool excl) + struct dentry *dentry, umode_t mode) { return mqueue_create_attr(dentry, mode, NULL); } @@ -790,7 +790,6 @@ static void __do_notify(struct mqueue_inode_info *info) struct kernel_siginfo sig_i; struct task_struct *task; - /* do_mq_notify() accepts sigev_signo == 0, why?? */ if (!info->notify.sigev_signo) break; @@ -1281,9 +1280,9 @@ static int do_mq_notify(mqd_t mqdes, const struct sigevent *notification) notification->sigev_notify != SIGEV_THREAD)) return -EINVAL; if (notification->sigev_notify == SIGEV_SIGNAL && - !valid_signal(notification->sigev_signo)) { + (!notification->sigev_signo || + !valid_signal(notification->sigev_signo))) return -EINVAL; - } if (notification->sigev_notify == SIGEV_THREAD) { long timeo; diff --git a/kernel/audit.c b/kernel/audit.c index dcc657d35776..feaa4e2271d7 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -950,7 +950,7 @@ static int kauditd_thread(void *dummy) * do the multicast send and rotate records from the * main queue to the retry/hold queues */ wait_event_freezable(kauditd_wait, - (skb_queue_len(&audit_queue) ? 1 : 0)); + (skb_queue_len_lockless(&audit_queue) ? 1 : 0)); } return 0; @@ -1283,7 +1283,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh, s.rate_limit = audit_rate_limit; s.backlog_limit = audit_backlog_limit; s.lost = atomic_read(&audit_lost); - s.backlog = skb_queue_len(&audit_queue); + s.backlog = skb_queue_len_lockless(&audit_queue); s.feature_bitmap = AUDIT_FEATURE_BITMAP_ALL; s.backlog_wait_time = audit_backlog_wait_time; s.backlog_wait_time_actual = atomic_read(&audit_backlog_wait_time_actual); @@ -1627,7 +1627,7 @@ static void audit_receive(struct sk_buff *skb) /* can't block with the ctrl lock, so penalize the sender now */ if (audit_backlog_limit && - (skb_queue_len(&audit_queue) > audit_backlog_limit)) { + (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) { DECLARE_WAITQUEUE(wait, current); /* wake kauditd to try and flush the queue */ @@ -1933,7 +1933,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, long stime = audit_backlog_wait_time; while (audit_backlog_limit && - (skb_queue_len(&audit_queue) > audit_backlog_limit)) { + (skb_queue_len_lockless(&audit_queue) > audit_backlog_limit)) { /* wake kauditd to try and flush the queue */ wake_up_interruptible(&kauditd_wait); @@ -1953,7 +1953,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, } else { if (audit_rate_check() && printk_ratelimit()) pr_warn("audit_backlog=%d > audit_backlog_limit=%d\n", - skb_queue_len(&audit_queue), + skb_queue_len_lockless(&audit_queue), audit_backlog_limit); audit_log_lost("backlog limit exceeded"); return NULL; diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c index 0da8d923e39d..f9e81060c1f4 100644 --- a/kernel/bpf/bpf_inode_storage.c +++ b/kernel/bpf/bpf_inode_storage.c @@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) { + /* + * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM + * was not initialized by the LSM framework at boot. Without proper + * initialization, the BPF inode security blob offset remains unprepared, + * causing bpf_inode() to calculate an invalid memory offset and corrupt + * inode->i_security. + */ + if (!bpf_lsm_initialized) + return ERR_PTR(-EOPNOTSUPP); return bpf_local_storage_map_alloc(attr, &inode_cache); } diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 564071a92d7d..3983b4ce73c8 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -51,6 +51,9 @@ BTF_ID(func, bpf_lsm_key_getsecurity) #ifdef CONFIG_AUDIT BTF_ID(func, bpf_lsm_audit_rule_match) #endif +#ifdef CONFIG_SECURITY_NETWORK_XFRM +BTF_ID(func, bpf_lsm_xfrm_decode_session) +#endif BTF_ID(func, bpf_lsm_ismaclabel) BTF_ID(func, bpf_lsm_file_alloc_security) BTF_SET_END(bpf_lsm_disabled_hooks) @@ -292,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 15ae7c43f594..64572f85edc8 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -3472,12 +3473,69 @@ static int btf_find_struct(const struct btf *btf, const struct btf_type *t, return BTF_FIELD_FOUND; } +struct btf_type_tag_match { + const char *name; + u32 flag; +}; + +struct btf_type_tag_walk_ctx { + const struct btf_type *t; /* Input/Output */ + u32 id; /* Output */ + u32 res; /* Output */ +}; + +static int btf_type_tag_walk(const struct btf *btf, + struct btf_type_tag_walk_ctx *ctx, + const struct btf_type_tag_match *matches, + u32 match_cnt) +{ + const struct btf_type *t = ctx->t; + u32 res = 0; + const char *tag; + u32 id, i; + + do { + id = t->type; + t = btf_type_by_id(btf, id); + + if (!btf_type_is_modifier(t)) + break; + + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + + tag = __btf_name_by_offset(btf, t->name_off); + for (i = 0; i < match_cnt; i++) { + if (strcmp(tag, matches[i].name)) + continue; + res |= matches[i].flag; + break; + } + } while (true); + + /* We only support a single tag. */ + if (hweight32(res) > 1) + return -EINVAL; + + ctx->t = t; + ctx->id = id; + ctx->res = res; + + return 0; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { - enum btf_field_type type; - const char *tag_value; - bool is_type_tag; + static const struct btf_type_tag_match kptr_type_tags[] = { + { "kptr_untrusted", BPF_KPTR_UNREF }, + { "kptr", BPF_KPTR_REF }, + { "percpu_kptr", BPF_KPTR_PERCPU }, + { "uptr", BPF_UPTR }, + }; + struct btf_type_tag_walk_ctx ctx; + enum btf_field_type type = 0; + int err; u32 res_id; /* Permit modifiers on the pointer itself */ @@ -3486,30 +3544,20 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, /* For PTR, sz is always == 8 */ if (!btf_type_is_ptr(t)) return BTF_FIELD_IGNORE; - t = btf_type_by_id(btf, t->type); - is_type_tag = btf_type_is_type_tag(t) && !btf_type_kflag(t); - if (!is_type_tag) - return BTF_FIELD_IGNORE; - /* Reject extra tags */ - if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) - return -EINVAL; - tag_value = __btf_name_by_offset(btf, t->name_off); - if (!strcmp("kptr_untrusted", tag_value)) - type = BPF_KPTR_UNREF; - else if (!strcmp("kptr", tag_value)) - type = BPF_KPTR_REF; - else if (!strcmp("percpu_kptr", tag_value)) - type = BPF_KPTR_PERCPU; - else if (!strcmp("uptr", tag_value)) - type = BPF_UPTR; - else - return -EINVAL; + + ctx.t = t; + err = btf_type_tag_walk(btf, &ctx, kptr_type_tags, + ARRAY_SIZE(kptr_type_tags)); + if (err) + return err; + + t = ctx.t; + res_id = ctx.id; + type = ctx.res; if (!(type & field_mask)) return BTF_FIELD_IGNORE; - /* Get the base type */ - t = btf_type_skip_modifiers(btf, t->type, &res_id); /* Only pointer to struct is allowed */ if (!__btf_type_is_struct(t)) return -EINVAL; @@ -5859,11 +5907,10 @@ struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) return bsearch(&btf_id, tab->types, tab->cnt, sizeof(tab->types[0]), btf_id_cmp_func); } -static int btf_check_type_tags(struct btf_verifier_env *env, - struct btf *btf, int start_id) +static int btf_check_modifier_chain_length(struct btf_verifier_env *env, + struct btf *btf, int start_id) { int i, n, good_id = start_id - 1; - bool in_tags; n = btf_nr_types(btf); for (i = start_id; i < n; i++) { @@ -5879,20 +5926,12 @@ static int btf_check_type_tags(struct btf_verifier_env *env, cond_resched(); - in_tags = btf_type_is_type_tag(t); while (btf_type_is_modifier(t)) { if (!chain_limit--) { btf_verifier_log(env, "Max chain length or cycle detected"); return -ELOOP; } - if (btf_type_is_type_tag(t)) { - if (!in_tags) { - btf_verifier_log(env, "Type tags don't precede modifiers"); - return -EINVAL; - } - } else if (in_tags) { - in_tags = false; - } + if (cur_id <= good_id) break; /* Move to next type */ @@ -5970,7 +6009,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err) goto errout; - err = btf_check_type_tags(env, btf, 1); + err = btf_check_modifier_chain_length(env, btf, 1); if (err) goto errout; @@ -6378,7 +6417,7 @@ static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name if (err) goto errout; - err = btf_check_type_tags(env, btf, 1); + err = btf_check_modifier_chain_length(env, btf, 1); if (err) goto errout; @@ -6504,7 +6543,7 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, if (err) goto errout; - err = btf_check_type_tags(env, btf, btf_nr_types(base_btf)); + err = btf_check_modifier_chain_length(env, btf, btf_nr_types(base_btf)); if (err) goto errout; @@ -6810,14 +6849,18 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { + static const struct btf_type_tag_match ctx_type_tags[] = { + { "user", MEM_USER }, + { "percpu", MEM_PERCPU }, + }; const struct btf_type *t = prog->aux->attach_func_proto; struct bpf_prog *tgt_prog = prog->aux->dst_prog; struct btf *btf = bpf_prog_get_target_btf(prog); const char *tname = prog->aux->attach_func_name; struct bpf_verifier_log *log = info->log; + struct btf_type_tag_walk_ctx ctx; const struct btf_param *args; bool ptr_err_raw_tp = false; - const char *tag_value; u32 nr_args, arg; int i, ret; @@ -7020,22 +7063,18 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } info->btf = btf; - info->btf_id = t->type; - t = btf_type_by_id(btf, t->type); - - if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) { - tag_value = __btf_name_by_offset(btf, t->name_off); - if (strcmp(tag_value, "user") == 0) - info->reg_type |= MEM_USER; - if (strcmp(tag_value, "percpu") == 0) - info->reg_type |= MEM_PERCPU; + ctx.t = t; + ret = btf_type_tag_walk(btf, &ctx, ctx_type_tags, + ARRAY_SIZE(ctx_type_tags)); + if (ret) { + bpf_log(log, "func '%s' arg%d type %s has multiple type tags\n", + tname, arg, btf_type_str(t)); + return false; } + info->reg_type |= ctx.res; + info->btf_id = ctx.id; + t = ctx.t; - /* skip modifiers */ - while (btf_type_is_modifier(t)) { - info->btf_id = t->type; - t = btf_type_by_id(btf, t->type); - } if (!btf_type_is_struct(t)) { bpf_log(log, "func '%s' arg%d type %s is not a struct\n", @@ -7074,7 +7113,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; const struct btf_member *member; - const char *tname, *mname, *tag_value; + const char *tname, *mname; u32 vlen, elem_id, mid; again: @@ -7270,8 +7309,15 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, } if (btf_type_is_ptr(mtype)) { - const struct btf_type *stype, *t; + static const struct btf_type_tag_match walk_type_tags[] = { + { "user", MEM_USER }, + { "percpu", MEM_PERCPU }, + { "rcu", MEM_RCU }, + }; enum bpf_type_flag tmp_flag = 0; + struct btf_type_tag_walk_ctx ctx = { .t = mtype }; + const struct btf_type *stype; + int err; u32 id; if (msize != size || off != moff) { @@ -7281,22 +7327,17 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, return -EACCES; } - /* check type tag */ - t = btf_type_by_id(btf, mtype->type); - if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) { - tag_value = __btf_name_by_offset(btf, t->name_off); - /* check __user tag */ - if (strcmp(tag_value, "user") == 0) - tmp_flag = MEM_USER; - /* check __percpu tag */ - if (strcmp(tag_value, "percpu") == 0) - tmp_flag = MEM_PERCPU; - /* check __rcu tag */ - if (strcmp(tag_value, "rcu") == 0) - tmp_flag = MEM_RCU; + err = btf_type_tag_walk(btf, &ctx, walk_type_tags, + ARRAY_SIZE(walk_type_tags)); + if (err) { + bpf_log(log, "type '%s' has multiple type tags\n", + btf_type_str(mtype)); + return err; } + tmp_flag = ctx.res; + id = ctx.id; + stype = ctx.t; - stype = btf_type_skip_modifiers(btf, mtype->type, &id); if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -7867,7 +7908,12 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id, u32 *tags) { + static const struct btf_type_tag_match func_type_tags[] = { + { "arena", ARG_TAG_ARENA }, + }; + struct btf_type_tag_walk_ctx ctx; const struct btf_type *t; + int err; /* Find the first pointer type in the chain. */ t = btf_type_skip_modifiers(btf, type_id, NULL); @@ -7879,24 +7925,15 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, if (!t || !btf_type_is_ptr(t)) return 0; - /* We got a pointer, get all associated type tags. */ - for (t = btf_type_by_id(btf, t->type); t && btf_type_is_modifier(t); - t = btf_type_by_id(btf, t->type)) { - - /* Skip non-type tag modifiers. */ - if (!btf_type_is_type_tag(t)) - continue; - - const char *tag = __btf_name_by_offset(btf, t->name_off); - - if (strcmp(tag, "arena") == 0) { - *tags |= ARG_TAG_ARENA; - } else { - bpf_log(&env->log, "function signature member has unsupported type tag '%s'\n", - tag); - return -EOPNOTSUPP; - } + ctx.t = t; + err = btf_type_tag_walk(btf, &ctx, func_type_tags, + ARRAY_SIZE(func_type_tags)); + if (err) { + bpf_log(&env->log, + "function signature member has multiple type tags\n"); + return err; } + *tags |= ctx.res; return 0; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 83ce66296ac1..4355ccb78a9c 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -939,19 +939,65 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, return ret; } +static int effective_prog_pos(struct cgroup *cgrp, + enum cgroup_bpf_attach_type atype, + struct bpf_prog_list *target_pl) +{ + int cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart, pos = -1; + struct bpf_prog_list *pl; + struct cgroup *p = cgrp; + + /* count effective programs to find where the preorder region ends */ + do { + if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt); + p = cgroup_parent(p); + } while (p); + + /* replay compute_effective_progs() placement and record target's slot */ + cnt = 0; + p = cgrp; + fstart = preorder_cnt; + bstart = preorder_cnt - 1; + do { + if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) + continue; + + init_bstart = bstart; + hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { + if (!prog_list_prog(pl)) + continue; + + if (pl->flags & BPF_F_PREORDER) { + if (pl == target_pl) + pos = bstart; + bstart--; + } else { + if (pl == target_pl) + pos = fstart; + fstart++; + } + cnt++; + } + + /* reverse pre-ordering progs at this cgroup level */ + if (pos >= bstart + 1 && pos <= init_bstart) + pos = bstart + 1 + init_bstart - pos; + } while ((p = cgroup_parent(p))); + + return pos; +} + /* Swap updated BPF program for given link in effective program arrays across * all descendant cgroups. This function is guaranteed to succeed. */ static void replace_effective_prog(struct cgroup *cgrp, enum cgroup_bpf_attach_type atype, - struct bpf_cgroup_link *link) + struct bpf_prog_list *pl) { struct bpf_prog_array_item *item; struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; css_for_each_descendant_pre(css, &cgrp->self) { @@ -960,27 +1006,15 @@ static void replace_effective_prog(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; + pos = effective_prog_pos(desc, atype, pl); + if (WARN_ON_ONCE(pos < 0)) + continue; - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->link == link) - goto found; - pos++; - } - } -found: - BUG_ON(!cg); progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, link->link.prog); + WRITE_ONCE(item->prog, pl->link->link.prog); } } @@ -1024,7 +1058,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); - replace_effective_prog(cgrp, atype, link); + replace_effective_prog(cgrp, atype, pl); bpf_prog_put(old_prog); return 0; } @@ -1091,19 +1125,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs, * recomputing the array in place. * * @cgrp: The cgroup which descendants to travers - * @prog: A program to detach or NULL - * @link: A link to detach or NULL + * @pl: The prog_list entry being detached * @atype: Type of detach operation */ -static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, - struct bpf_cgroup_link *link, +static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, enum cgroup_bpf_attach_type atype) { struct cgroup_subsys_state *css; struct bpf_prog_array *progs; - struct bpf_prog_list *pl; - struct hlist_head *head; - struct cgroup *cg; int pos; /* recompute effective prog array in place */ @@ -1113,24 +1142,11 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; - /* find position of link or prog in effective progs array */ - for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) { - if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI)) - continue; - - head = &cg->bpf.progs[atype]; - hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) - continue; - if (pl->prog == prog && pl->link == link) - goto found; - pos++; - } - } - + pos = effective_prog_pos(desc, atype, pl); /* no link or prog match, skip the cgroup of this layer */ - continue; -found: + if (pos < 0) + continue; + progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); @@ -1196,7 +1212,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, /* if update effective array failed replace the prog with a dummy prog*/ pl->prog = old_prog; pl->link = link; - purge_effective_progs(cgrp, old_prog, link, atype); + purge_effective_progs(cgrp, pl, atype); } /* now can actually delete it from this cgroup list */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 649cce41e13f..6e19a030da6f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -875,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, struct bpf_prog_pack { struct list_head list; void *ptr; + bool arch_flush_needed; unsigned long bitmap[]; }; @@ -883,6 +885,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -918,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; set_vm_flush_reset_perms(pack->ptr); err = set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); @@ -932,15 +945,23 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); - struct bpf_prog_pack *pack; - unsigned long pos; + struct bpf_prog_pack *pack, *fallback_pack = NULL; + unsigned long pos, fallback_pos = 0; void *ptr = NULL; mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -960,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) list_for_each_entry(pack, &pack_list, list) { pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, nbits, 0); - if (pos < BPF_PROG_CHUNK_COUNT) + if (pos >= BPF_PROG_CHUNK_COUNT) + continue; + /* Flush not enabled, use any pack */ + if (!static_branch_unlikely(&bpf_pred_flush_enabled)) goto found_free_area; + /* + * cBPF reuse of a dirty pack triggers a flush, so prefer a + * clean pack for cBPF. eBPF never flushes, so steer it to a + * dirty pack and keep clean packs free for cBPF. + */ + if (was_classic ^ pack->arch_flush_needed) + goto found_free_area; + if (!fallback_pack) { + fallback_pack = pack; + fallback_pos = pos; + } + } + + /* No preferred pack found */ + if (fallback_pack) { + pack = fallback_pack; + pos = fallback_pos; + goto found_free_area; } pack = alloc_new_pack(bpf_fill_ill_insns); @@ -971,6 +1013,16 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && + pack->arch_flush_needed && + was_classic) { + struct bpf_prog_pack *p; + + static_call_cond(bpf_arch_pred_flush)(); + list_for_each_entry(p, &pack_list, list) + p->arch_flush_needed = false; + } bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1008,6 +1060,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); bitmap_clear(pack->bitmap, pos, nbits); + + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pack->arch_flush_needed = true; if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, BPF_PROG_CHUNK_COUNT, 0) == 0) { list_del(&pack->list); @@ -1130,7 +1185,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1143,7 +1199,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index f8a3c7eb451e..0391b3bc0073 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c..ea2d60dc1fee 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab6..12a8a4eb757f 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2338,7 +2338,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) func_id_name(insn->imm), insn->imm); return -EFAULT; } - insn->imm = fn->func - __bpf_call_base; + insn->imm = BPF_CALL_IMM(fn->func); next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 8e196c9b7c50..c18f1e16edee 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3700,7 +3700,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, return ret + 1; } -/* Keep unsinged long in prototype so that kfunc is usable when emitted to +/* Keep unsigned long in prototype so that kfunc is usable when emitted to * vmlinux.h in BPF programs directly, but note that while in BPF prog, the * unsigned long always points to 8-byte region on stack, the kernel may only * read and write the 4-bytes on 32-bit. diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 77ba03216c09..41fe87d7302f 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -175,6 +175,95 @@ static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id, memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX); } +/* + * A cached VMA lookup result. The range [vm_start, vm_end) is always set. + * vm_pgoff, file, build_id are set only when the build ID was resolved. + * Zero vm_end marks the slot empty. build_id aliases the id_offs[] entry. + */ +struct stack_map_cached_vma { + unsigned long vm_start; + unsigned long vm_end; + unsigned long vm_pgoff; + struct file *file; /* pinned in the sleepable path; NULL otherwise */ + const unsigned char *build_id; +}; + +/* + * Per stack_map_get_build_id_offset() call cache of the last VMA with a build ID + * resolved and the last VMA with no usable build ID. Adjacent stack frames tend + * to land in the same VMA or the same backing file, so caching the last result + * of each kind lets us skip unnecessary VMA lookups and build ID parse calls. + * Keeping the two slots independent means a build-ID-less VMA doesn't evict the + * last resolved build ID. + */ +struct stack_map_build_id_cache { + struct stack_map_cached_vma resolved; + struct stack_map_cached_vma unresolved; +}; + +/* + * Fill @id from a cached range covering @ip. On a hit this writes @id (resolved + * range -> build ID + offset, unresolved range -> raw ip) and returns 0; on a + * miss it leaves @id untouched and returns -ENOENT. + */ +static int stack_map_build_id_set_from_cache(struct stack_map_build_id_cache *cache, + struct bpf_stack_build_id *id, u64 ip) +{ + unsigned long vm_start, vm_end, vm_pgoff; + u64 offset; + + vm_start = cache->resolved.vm_start; + vm_end = cache->resolved.vm_end; + if (vm_end && ip >= vm_start && ip < vm_end) { + vm_pgoff = cache->resolved.vm_pgoff; + offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip); + stack_map_build_id_set_valid(id, offset, cache->resolved.build_id); + return 0; + } + + vm_start = cache->unresolved.vm_start; + vm_end = cache->unresolved.vm_end; + if (vm_end && ip >= vm_start && ip < vm_end) { + stack_map_build_id_set_ip(id); + return 0; + } + + return -ENOENT; +} + +/* + * Record @vma's build ID as the last resolved one. @file is the pinned backing + * file in the sleepable path (released when evicted), or NULL otherwise. + */ +static void stack_map_build_id_cache_set_resolved(struct stack_map_build_id_cache *cache, + struct file *file, + const unsigned char *build_id, + unsigned long vm_start, + unsigned long vm_end, + unsigned long vm_pgoff) +{ + if (cache->resolved.file) + fput(cache->resolved.file); + cache->resolved = (struct stack_map_cached_vma){ + .vm_start = vm_start, + .vm_end = vm_end, + .vm_pgoff = vm_pgoff, + .file = file, + .build_id = build_id, + }; +} + +/* Record [vm_start, vm_end) as a range with no usable build ID. */ +static void stack_map_build_id_cache_set_unresolved(struct stack_map_build_id_cache *cache, + unsigned long vm_start, + unsigned long vm_end) +{ + cache->unresolved = (struct stack_map_cached_vma){ + .vm_start = vm_start, + .vm_end = vm_end, + }; +} + struct stack_map_vma_lock { struct vm_area_struct *vma; struct mm_struct *mm; @@ -244,15 +333,9 @@ static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *id_offs, u32 trace_nr) { - struct mm_struct *mm = current->mm; - struct stack_map_vma_lock lock = { .mm = mm }; - struct { - struct file *file; - const unsigned char *build_id; - unsigned long vm_start; - unsigned long vm_end; - unsigned long vm_pgoff; - } cache = {}; + struct stack_map_vma_lock lock = { .mm = current->mm }; + struct stack_map_build_id_cache cache = {}; + struct stack_map_cached_vma *res = &cache.resolved; unsigned long vm_pgoff, vm_start, vm_end; struct vm_area_struct *vma; struct file *file; @@ -262,44 +345,39 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i for (u32 i = 0; i < trace_nr; i++) { ip = READ_ONCE(id_offs[i].ip); - /* - * Range cache fast path: if ip falls within the previously - * resolved VMA range, reuse the cache build_id without - * re-acquiring the VMA lock. - */ - if (cache.build_id && ip >= cache.vm_start && ip < cache.vm_end) { - offset = stack_map_build_id_offset(cache.vm_pgoff, cache.vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); + if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip)) continue; - } vma = stack_map_lock_vma(&lock, ip); if (!vma) { stack_map_build_id_set_ip(&id_offs[i]); continue; } + + vm_pgoff = vma->vm_pgoff; + vm_start = vma->vm_start; + vm_end = vma->vm_end; + if (vma_is_anonymous(vma) || !vma->vm_file) { - stack_map_build_id_set_ip(&id_offs[i]); stack_map_unlock_vma(&lock); + stack_map_build_id_set_ip(&id_offs[i]); + stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end); continue; } file = vma->vm_file; - vm_pgoff = vma->vm_pgoff; - vm_start = vma->vm_start; - vm_end = vma->vm_end; offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip); /* - * Same backing file as previous (e.g. different VMAs - * of the same ELF binary). Reuse the cache build_id. + * Same backing file as the last resolved VMA (another mapping + * of the same ELF binary): reuse its build_id without re-parsing. */ - if (file == cache.file) { + if (file == res->file) { stack_map_unlock_vma(&lock); - stack_map_build_id_set_valid(&id_offs[i], offset, cache.build_id); - cache.vm_start = vm_start; - cache.vm_end = vm_end; - cache.vm_pgoff = vm_pgoff; + stack_map_build_id_set_valid(&id_offs[i], offset, res->build_id); + res->vm_start = vm_start; + res->vm_end = vm_end; + res->vm_pgoff = vm_pgoff; continue; } @@ -310,21 +388,17 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i if (build_id_parse_file(file, id_offs[i].build_id, NULL)) { stack_map_build_id_set_ip(&id_offs[i]); fput(file); + stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end); continue; } stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); - if (cache.file) - fput(cache.file); - cache.file = file; - cache.build_id = id_offs[i].build_id; - cache.vm_start = vm_start; - cache.vm_end = vm_end; - cache.vm_pgoff = vm_pgoff; + stack_map_build_id_cache_set_resolved(&cache, file, id_offs[i].build_id, + vm_start, vm_end, vm_pgoff); } - if (cache.file) - fput(cache.file); + if (res->file) + fput(res->file); } /* @@ -343,8 +417,8 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, struct mmap_unlock_irq_work *work = NULL; bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); bool has_user_ctx = user && current && current->mm; - struct vm_area_struct *vma, *prev_vma = NULL; - const unsigned char *prev_build_id = NULL; + struct stack_map_build_id_cache cache = {}; + struct vm_area_struct *vma; int i; if (may_fault && has_user_ctx) { @@ -365,27 +439,30 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, for (i = 0; i < trace_nr; i++) { u64 ip = READ_ONCE(id_offs[i].ip); - u64 offset; - if (prev_build_id && range_in_vma(prev_vma, ip, ip)) { - vma = prev_vma; - offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, prev_build_id); + if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip)) continue; - } + vma = find_vma(current->mm, ip); if (!vma || vma_is_anonymous(vma) || fetch_build_id(vma, id_offs[i].build_id, may_fault)) { - /* per entry fall back to ips */ + /* per entry fall back to ips; cache build-ID-less range */ stack_map_build_id_set_ip(&id_offs[i]); - prev_vma = vma; - prev_build_id = NULL; + if (vma) + stack_map_build_id_cache_set_unresolved(&cache, + vma->vm_start, vma->vm_end); continue; } - offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); - stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); - prev_vma = vma; - prev_build_id = id_offs[i].build_id; + /* + * mmap_lock is held for the whole loop, so the cached VMA + * fields stay valid; no file pinning is needed here. + */ + stack_map_build_id_set_valid(&id_offs[i], + stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip), + id_offs[i].build_id); + stack_map_build_id_cache_set_resolved(&cache, NULL, id_offs[i].build_id, + vma->vm_start, vma->vm_end, + vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); } diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 32f346ce3ffc..ea2153cf28d0 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -436,12 +436,10 @@ static void __clean_func_state(struct bpf_verifier_env *env, continue; /* - * Only destroy spilled_ptr when hi half is dead. - * If hi half is still live with STACK_SPILL, the - * spilled_ptr metadata is needed for correct state - * comparison in stacksafe(). - * is_spilled_reg() is using slot_type[7], but - * is_spilled_scalar_after() check either slot_type[0] or [4] + * Only scalar spills can be degraded to raw stack bytes + * when their high half is dead. Pointer spills need the + * saved spilled_ptr metadata so partial fills keep + * rejecting as non-scalar register fills. */ if (!hi_live) { struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; @@ -449,6 +447,9 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (lo_live && stype == STACK_SPILL) { u8 val = STACK_MISC; + if (spill->type != SCALAR_VALUE) + continue; + /* * 8 byte spill of scalar 0 where half slot is dead * should become STACK_ZERO in lo 4 bytes. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index b44106c8ea75..6db306d23b47 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -6308,7 +6308,7 @@ static int prog_stream_read(union bpf_attr *attr) return ret; } -#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.prog_fd +#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.flags static int prog_assoc_struct_ops(union bpf_attr *attr) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2abc79dbf281..d46f7db20d8f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3479,7 +3479,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_spillable_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[i]; + u8 type = state->stack[spi].slot_type[(slot - i) % + BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { sanitize = true; @@ -3701,14 +3702,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, * SCALAR. This function does not deal with register filling; the caller must * ensure that all spilled registers in the stack range have been marked as * read. + * + * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered + * zero bytes. In that case, mark the contributing stack slots precise so + * pruning cannot reuse a zero-spill state for a later non-zero spill state. + * + * Returns an error if precision backtracking fails. */ -static void mark_reg_stack_read(struct bpf_verifier_env *env, - /* func where src register points to */ - struct bpf_func_state *ptr_state, - int min_off, int max_off, int dst_regno) +static int mark_reg_stack_read(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *ptr_state, + int min_off, int max_off, int dst_regno) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; + u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; int zeros = 0; @@ -3718,19 +3726,33 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env, spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); stype = ptr_state->stack[spi].slot_type; - if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) - break; - zeros++; + if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { + zeros++; + continue; + } + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && + bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { + zero_spill_mask |= 1ull << spi; + zeros++; + continue; + } + break; } if (zeros == max_off - min_off) { /* Any access_size read into register is zero extended, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); + if (zero_spill_mask) { + bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + return mark_chain_precision_batch(env, env->cur_state); + } } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); } + + return 0; } /* Read the stack at 'off' and put the results into the register indicated by @@ -3752,6 +3774,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; @@ -3834,7 +3857,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, __mark_reg_const_zero(env, &state->regs[dst_regno]); insn_flags = 0; /* not restoring original register state */ } else { - mark_reg_unknown(env, state->regs, dst_regno); + err = mark_reg_stack_read(env, reg_state, off, off + size, + dst_regno); + if (err) + return err; insn_flags = 0; /* not restoring original register state */ } } @@ -3879,8 +3905,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } return -EACCES; } - if (dst_regno >= 0) - mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (dst_regno >= 0) { + err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (err) + return err; + } insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) @@ -3934,7 +3963,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; - mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); + err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size, + dst_regno); + if (err) + return err; check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -5786,6 +5818,10 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EFAULT; } ret = env->ops->btf_struct_access(&env->log, reg, off, size); + if (ret < 0) + verbose(env, + "%s cannot write into ptr_%s at off=%d size=%d\n", + reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for * program allocated objects (which always have id > 0), @@ -6196,6 +6232,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b */ if (info.reg_type == SCALAR_VALUE) { if (info.is_retval && get_func_retval_range(env->prog, &range)) { + mark_reg_unknown(env, regs, value_regno); err = __mark_reg_s32_range(env, regs, value_regno, range.minval, range.maxval); if (err) @@ -7990,9 +8027,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) +static int __check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type, + bool btf_id_fixed_off_ok) { u32 type = reg->type; @@ -8049,12 +8087,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero. This was already checked above. So pass - * fixed_off_ok as true to allow fixed offset for all other - * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we - * still need to do checks instead of returning. + * can be non-zero unless the caller requires otherwise. + * var_off always must be 0 for PTR_TO_BTF_ID, hence we still + * need to do checks instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, true); + return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8070,6 +8107,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, } } +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) +{ + return __check_func_arg_reg_off(env, reg, argno, arg_type, true); +} + static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -11941,6 +11985,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ enum bpf_arg_type arg_type = ARG_DONTCARE; argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); + bool btf_id_fixed_off_ok = true; u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; @@ -12114,7 +12159,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_MEM: case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: @@ -12128,6 +12172,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_CTX: arg_type = ARG_PTR_TO_CTX; break; + case KF_ARG_PTR_TO_REFCOUNTED_KPTR: + arg_type = ARG_PTR_TO_BTF_ID; + btf_id_fixed_off_ok = false; + break; default: verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); return -EFAULT; @@ -12135,7 +12183,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (regno == meta->release_regno) arg_type |= OBJ_RELEASE; - ret = check_func_arg_reg_off(env, reg, argno, arg_type); + ret = __check_func_arg_reg_off(env, reg, argno, arg_type, + btf_id_fixed_off_ok); if (ret < 0) return ret; @@ -19988,13 +20037,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (!is_priv) mutex_unlock(&bpf_verifier_lock); bpf_clear_insn_aux_data(env, 0, env->prog->len); - vfree(env->insn_aux_data); err_free_env: bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + vfree(env->insn_aux_data); kvfree(env); return ret; } diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 591e3aa487fc..083a3ac988e8 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1089,12 +1089,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) * @cs: the cpuset the need to recompute the new effective_cpus mask * @parent: the parent cpuset * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. * The result is valid only if the given cpuset isn't a partition root. */ static void compute_effective_cpumask(struct cpumask *new_cpus, struct cpuset *cs, struct cpuset *parent) { - cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + bool has_cpus; + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -2143,15 +2166,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, goto update_parent_effective; } - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs unless - * it is a partition root that has explicitly distributed - * out all its CPUs. - */ - if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) - cpumask_copy(tmp->new_cpus, parent->effective_cpus); - /* * Skip the whole subtree if * 1) the cpumask remains the same, @@ -2647,13 +2661,17 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) cpuset_change_task_nodemask(task, &newmems); + /* Rebind and migrate mm only for thread group leader */ + if (!thread_group_leader(task)) + continue; + mm = get_task_mm(task); if (!mm) continue; migrate = is_memory_migrate(cs); - mpol_rebind_mm(mm, &cs->mems_allowed); + mpol_rebind_mm(mm, &cs->effective_mems); if (migrate) cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); else @@ -2692,14 +2710,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); - bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (is_in_v2_mode() && !has_mems) - *new_mems = parent->effective_mems; + compute_effective_nodemask(new_mems, cp, parent); /* Skip the whole subtree if the nodemask remains the same. */ if (nodes_equal(*new_mems, cp->effective_mems)) { @@ -3773,7 +3784,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); - nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); + compute_effective_nodemask(&new_mems, cs, parent); if (!tmp || !cs->partition_root_state) goto update_tasks; @@ -4124,6 +4135,28 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) return changed; } +/* + * Returns the number of CPUs available for this cgroup. + * + * This only really works for cgroup-v2 where all the controllers are mounted + * in the same hierarchy. If not cgroup-v2 or no cpuset controller is + * configured it reverts to num_online_cpus(). + */ +int cpuset_num_cpus(struct cgroup *cgrp) +{ + int nr = num_online_cpus(); + struct cpuset *cs; + + if (is_in_v2_mode()) { + guard(rcu)(); + cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); + if (cs) + nr = cpumask_weight(cs->effective_cpus); + } + + return nr; +} + void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); @@ -4157,7 +4190,7 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk) * * Are any of the nodes in the nodemask allowed in current->mems_allowed? */ -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return nodes_intersects(*nodemask, current->mems_allowed); } diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 4753a67d0f0f..6430c7ce1e03 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -182,6 +182,11 @@ static u64 get_resource_current(struct dmem_cgroup_pool_state *pool) return pool ? page_counter_read(&pool->cnt) : 0; } +static u64 get_resource_peak(struct dmem_cgroup_pool_state *pool) +{ + return pool ? READ_ONCE(pool->cnt.watermark) : 0; +} + static void reset_all_resource_limits(struct dmem_cgroup_pool_state *rpool) { set_resource_min(rpool, 0); @@ -808,6 +813,11 @@ static int dmemcg_limit_show(struct seq_file *sf, void *v, return 0; } +static int dmem_cgroup_region_peak_show(struct seq_file *sf, void *v) +{ + return dmemcg_limit_show(sf, v, get_resource_peak); +} + static int dmem_cgroup_region_current_show(struct seq_file *sf, void *v) { return dmemcg_limit_show(sf, v, get_resource_current); @@ -856,6 +866,11 @@ static struct cftype files[] = { .name = "current", .seq_show = dmem_cgroup_region_current_show, }, + { + .name = "peak", + .seq_show = dmem_cgroup_region_peak_show, + .flags = CFTYPE_NOT_ON_ROOT, + }, { .name = "min", .write = dmem_cgroup_region_min_write, diff --git a/kernel/cpu.c b/kernel/cpu.c index f975bb34915b..b3c8553d7bd6 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -174,7 +174,7 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, struct cpuhp_step *step = cpuhp_get_step(state); int (*cbm)(unsigned int cpu, struct hlist_node *node); int (*cb)(unsigned int cpu); - int ret, cnt; + int ret, cnt, rollback_ret; if (st->fail == state) { st->fail = CPUHP_INVALID; @@ -238,12 +238,12 @@ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); - ret = cbm(cpu, node); - trace_cpuhp_exit(cpu, st->state, state, ret); + rollback_ret = cbm(cpu, node); + trace_cpuhp_exit(cpu, st->state, state, rollback_ret); /* * Rollback must not fail, */ - WARN_ON_ONCE(ret); + WARN_ON_ONCE(rollback_ret); } return ret; } @@ -2854,21 +2854,17 @@ static const struct attribute_group cpuhp_cpu_attr_group = { .name = "hotplug", }; -static ssize_t states_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t cur, res = 0; + ssize_t res = 0; int i; mutex_lock(&cpuhp_state_mutex); for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { struct cpuhp_step *sp = cpuhp_get_step(i); - if (sp->name) { - cur = sprintf(buf, "%3d: %s\n", i, sp->name); - buf += cur; - res += cur; - } + if (sp->name) + res += sysfs_emit_at(buf, res, "%3d: %s\n", i, sp->name); } mutex_unlock(&cpuhp_state_mutex); return res; diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 4f21fc3b108b..2b36aa9fade0 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -168,9 +168,6 @@ static inline resource_size_t crash_resource_size(const struct resource *res) return !res->end ? 0 : resource_size(res); } - - - int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz) { @@ -272,6 +269,92 @@ int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, return 0; } +static struct crash_mem *alloc_cmem(unsigned int nr_ranges) +{ + struct crash_mem *cmem; + + cmem = kvzalloc_flex(*cmem, ranges, nr_ranges); + if (!cmem) + return NULL; + + cmem->max_nr_ranges = nr_ranges; + return cmem; +} + +unsigned int __weak arch_get_system_nr_ranges(void) { return 0; } +int __weak arch_crash_populate_cmem(struct crash_mem *cmem) { return -1; } +int __weak arch_crash_exclude_ranges(struct crash_mem *cmem) { return 0; } + +int __weak arch_crash_exclude_mem_range(struct crash_mem **mem, + unsigned long long mstart, + unsigned long long mend) +{ + return crash_exclude_mem_range(*mem, mstart, mend); +} + +int crash_exclude_core_ranges(struct crash_mem **cmem) +{ + int ret, i; + + /* Exclude crashkernel region */ + ret = arch_crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); + if (ret) + return ret; + + if (crashk_low_res.end) { + ret = arch_crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); + if (ret) + return ret; + } + + for (i = 0; i < crashk_cma_cnt; ++i) { + ret = arch_crash_exclude_mem_range(cmem, crashk_cma_ranges[i].start, + crashk_cma_ranges[i].end); + if (ret) + return ret; + } + + return 0; +} + +int crash_prepare_headers(int need_kernel_map, void **addr, unsigned long *sz, + unsigned long *nr_mem_ranges) +{ + unsigned int max_nr_ranges; + struct crash_mem *cmem; + int ret; + + max_nr_ranges = arch_get_system_nr_ranges(); + if (!max_nr_ranges) + return -ENOMEM; + + cmem = alloc_cmem(max_nr_ranges); + if (!cmem) + return -ENOMEM; + + ret = arch_crash_populate_cmem(cmem); + if (ret) + goto out; + + ret = crash_exclude_core_ranges(&cmem); + if (ret) + goto out; + + ret = arch_crash_exclude_ranges(cmem); + if (ret) + goto out; + + /* Return the computed number of memory ranges, for hotplug usage */ + if (nr_mem_ranges) + *nr_mem_ranges = cmem->nr_ranges; + + ret = crash_prepare_elf64_headers(cmem, need_kernel_map, addr, sz); + +out: + kvfree(cmem); + return ret; +} + /** * crash_exclude_mem_range - exclude a mem range for existing ranges * @mem: mem->range contains an array of ranges sorted in ascending order diff --git a/kernel/events/core.c b/kernel/events/core.c index b1e1c5f0c7ba..5f6390369475 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4729,7 +4729,7 @@ static void perf_remove_from_owner(struct perf_event *event); static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke); + unsigned long detach_flags); /* * Removes all events from the current task that have been marked @@ -4756,7 +4756,7 @@ static void perf_event_remove_on_exec(struct perf_event_context *ctx) modified = true; - perf_event_exit_event(event, ctx, ctx->task, false); + perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP); } raw_spin_lock_irqsave(&ctx->lock, flags); @@ -5304,6 +5304,7 @@ static void free_event_rcu(struct rcu_head *head) if (event->ns) put_pid_ns(event->ns); perf_event_free_filter(event); + kfree(event->addr_filter_ranges); kmem_cache_free(perf_event_cache, event); } @@ -5751,8 +5752,6 @@ static void __free_event(struct perf_event *event) if (event->attach_state & PERF_ATTACH_CALLCHAIN) put_callchain_buffers(); - kfree(event->addr_filter_ranges); - if (event->attach_state & PERF_ATTACH_EXCLUSIVE) exclusive_event_destroy(event); @@ -7792,10 +7791,20 @@ unsigned long perf_misc_flags(struct perf_event *event, unsigned long perf_instruction_pointer(struct perf_event *event, struct pt_regs *regs) { - if (should_sample_guest(event)) - return perf_guest_get_ip(); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (should_sample_guest(event)) { + return event->attr.exclude_kernel && + !(perf_guest_state() & PERF_GUEST_USER) ? + 0 : perf_guest_get_ip(); + } - return perf_arch_instruction_pointer(regs); + return event->attr.exclude_kernel && !user_mode(regs) ? + 0 : perf_arch_instruction_pointer(regs); } static void @@ -7829,10 +7838,22 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, } static void perf_sample_regs_intr(struct perf_regs *regs_intr, - struct pt_regs *regs) + struct pt_regs *regs, + bool exclude_kernel) { - regs_intr->regs = regs; - regs_intr->abi = perf_reg_abi(current); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (exclude_kernel && !user_mode(regs)) { + regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE; + regs_intr->regs = NULL; + } else { + regs_intr->regs = regs; + regs_intr->abi = perf_reg_abi(current); + } } @@ -8723,7 +8744,8 @@ void perf_prepare_sample(struct perf_sample_data *data, /* regs dump ABI info */ int size = sizeof(u64); - perf_sample_regs_intr(&data->regs_intr, regs); + perf_sample_regs_intr(&data->regs_intr, regs, + event->attr.exclude_kernel); if (data->regs_intr.regs) { u64 mask = event->attr.sample_regs_intr; @@ -12938,7 +12960,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, /* * De-schedule the event and mark it REVOKED. */ - perf_event_exit_event(event, ctx, ctx->task, true); + perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE); /* * All _free_event() bits that rely on event->pmu: @@ -13910,7 +13932,9 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel) { + if (!attr.exclude_kernel || + ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && + !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); if (err) return err; @@ -14526,12 +14550,13 @@ static void perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx, struct task_struct *task, - bool revoke) + unsigned long detach_flags) { struct perf_event *parent_event = event->parent; - unsigned long detach_flags = DETACH_EXIT; unsigned int attach_state; + detach_flags |= DETACH_EXIT; + if (parent_event) { /* * Do not destroy the 'original' grouping; because of the @@ -14554,8 +14579,8 @@ perf_event_exit_event(struct perf_event *event, sync_child_event(event, task); } - if (revoke) - detach_flags |= DETACH_GROUP | DETACH_REVOKE; + if (detach_flags & DETACH_REVOKE) + detach_flags |= DETACH_GROUP; perf_remove_from_context(event, detach_flags); /* @@ -14643,7 +14668,7 @@ static void perf_event_exit_task_context(struct task_struct *task, bool exit) perf_event_task(task, ctx, 0); list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) - perf_event_exit_event(child_event, ctx, exit ? task : NULL, false); + perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0); mutex_unlock(&ctx->mutex); @@ -14769,6 +14794,24 @@ int perf_allow_kernel(void) } EXPORT_SYMBOL_GPL(perf_allow_kernel); +int perf_allow_cpu(void) +{ + if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) + return -EACCES; + + return security_perf_event_open(PERF_SECURITY_CPU); +} +EXPORT_SYMBOL_GPL(perf_allow_cpu); + +int perf_allow_tracepoint(void) +{ + if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) + return -EPERM; + + return security_perf_event_open(PERF_SECURITY_TRACEPOINT); +} +EXPORT_SYMBOL_GPL(perf_allow_tracepoint); + /* * Inherit an event from parent task to child task. * diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 4084e926e284..b5c516168f84 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1806,14 +1806,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1817,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; diff --git a/kernel/exit.c b/kernel/exit.c index 1056422bc101..7ac52196d819 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1111,7 +1111,7 @@ void __noreturn make_task_dead(int signr) SYSCALL_DEFINE1(exit, int, error_code) { - do_exit((error_code&0xff)<<8); + do_exit((error_code & 0xff) << 8); } /* diff --git a/kernel/fork.c b/kernel/fork.c index 892a95214c54..175c73bbe2bf 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1009,6 +1009,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->mm_cid.active = 0; INIT_HLIST_NODE(&tsk->mm_cid.node); #endif + +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(tsk->bpf_storage, NULL); + tsk->bpf_ctx = NULL; +#endif return tsk; free_stack: @@ -1064,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } @@ -1613,9 +1617,27 @@ static int copy_exec_state(u64 clone_flags, struct task_struct *tsk) return task_exec_state_copy(tsk); } -static int copy_fs(u64 clone_flags, struct task_struct *tsk) +static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh) { - struct fs_struct *fs = current->fs; + struct fs_struct *fs; + + /* + * Usermodehelper may copy userspace_init_fs filesystem state but + * they don't get to create mount namespaces, share the + * filesystem state, or be started from a non-initial mount + * namespace. + */ + if (umh) { + if (clone_flags & (CLONE_NEWNS | CLONE_FS)) + return -EINVAL; + if (current->nsproxy->mnt_ns != &init_mnt_ns) + return -EINVAL; + fs = userspace_init_fs; + } else { + fs = current->fs; + VFS_WARN_ON_ONCE(current->fs != current->real_fs); + } + if (clone_flags & CLONE_FS) { /* tsk->fs is already what we want */ read_seqlock_excl(&fs->seq); @@ -1628,7 +1650,7 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk) read_sequnlock_excl(&fs->seq); return 0; } - tsk->fs = copy_fs_struct(fs); + tsk->real_fs = tsk->fs = copy_fs_struct(fs); if (!tsk->fs) return -ENOMEM; return 0; @@ -2247,10 +2269,6 @@ __latent_entropy struct task_struct *copy_process( p->sequential_io = 0; p->sequential_io_avg = 0; #endif -#ifdef CONFIG_BPF_SYSCALL - RCU_INIT_POINTER(p->bpf_storage, NULL); - p->bpf_ctx = NULL; -#endif unwind_task_init(p); @@ -2276,7 +2294,7 @@ __latent_entropy struct task_struct *copy_process( retval = copy_files(clone_flags, p, args->no_files); if (retval) goto bad_fork_cleanup_semundo; - retval = copy_fs(clone_flags, p); + retval = copy_fs(clone_flags, p, args->umh); if (retval) goto bad_fork_cleanup_files; retval = copy_sighand(clone_flags, p); @@ -2338,6 +2356,7 @@ __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_BLOCK p->plug = NULL; + p->flags &= ~PF_BLOCK_TS; #endif futex_init_task(p); @@ -2817,6 +2836,7 @@ pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags) .exit_signal = (flags & CSIGNAL), .fn = fn, .fn_arg = arg, + .umh = 1, }; return kernel_clone(&args); @@ -3214,7 +3234,7 @@ static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp */ int ksys_unshare(unsigned long unshare_flags) { - struct fs_struct *fs, *new_fs = NULL; + struct fs_struct *new_fs = NULL; struct files_struct *new_fd = NULL; struct cred *new_cred = NULL; struct nsproxy *new_nsproxy = NULL; @@ -3245,6 +3265,10 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; + /* No unsharing with overriden fs state */ + VFS_WARN_ON_ONCE(unshare_flags & (CLONE_NEWNS | CLONE_FS) && + current->fs != current->real_fs); + err = check_unshare_flags(unshare_flags); if (err) goto bad_unshare_out; @@ -3292,23 +3316,13 @@ int ksys_unshare(unsigned long unshare_flags) new_nsproxy = NULL; } - task_lock(current); + if (new_fs) + new_fs = switch_fs_struct(new_fs); - if (new_fs) { - fs = current->fs; - read_seqlock_excl(&fs->seq); - current->fs = new_fs; - if (--fs->users) - new_fs = NULL; - else - new_fs = fs; - read_sequnlock_excl(&fs->seq); - } - - if (new_fd) + if (new_fd) { + guard(task_lock)(current); swap(current->files, new_fd); - - task_unlock(current); + } if (new_cred) { /* Install the new user namespace */ diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 7384672916fb..79823ad13683 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -645,12 +645,6 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, continue; } - /* Self-deadlock: non-top waiter already owns the PI futex. */ - if (rt_mutex_owner(&pi_state->pi_mutex) == this->task) { - ret = -EDEADLK; - break; - } - ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, this->rt_waiter, this->task); diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index f15c9f1223bb..57c819da30c2 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1658,7 +1658,7 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, for (i = 0; i < nr_irqs; i++) { ret = irq_domain_trim_hierarchy(virq + i); if (ret) - goto out_free_irq_data; + goto out_free_irqs; } for (i = 0; i < nr_irqs; i++) @@ -1666,6 +1666,8 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base, return virq; +out_free_irqs: + irq_domain_free_irqs_hierarchy(domain, virq, nr_irqs); out_free_irq_data: irq_domain_free_irq_data(virq, nr_irqs); out_free_desc: @@ -1963,7 +1965,7 @@ EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent); static void __irq_domain_deactivate_irq(struct irq_data *irq_data) { - if (irq_data && irq_data->domain) { + if (irq_data->domain) { struct irq_domain *domain = irq_data->domain; if (domain->ops->deactivate) @@ -1977,7 +1979,7 @@ static int __irq_domain_activate_irq(struct irq_data *irqd, bool reserve) { int ret = 0; - if (irqd && irqd->domain) { + if (irqd->domain) { struct irq_domain *domain = irqd->domain; if (irqd->parent_data) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 7eb07e3bdb4c..2fbff2618a1e 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -171,6 +171,7 @@ int irq_can_set_affinity(unsigned int irq) { return __irq_can_set_affinity(irq_to_desc(irq)); } +EXPORT_SYMBOL_GPL(irq_can_set_affinity); /** * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space @@ -2703,11 +2704,7 @@ static int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state return -ENODEV; if (chip->irq_get_irqchip_state) break; -#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY - data = data->parent_data; -#else - data = NULL; -#endif + data = irqd_get_parent_data(data); } while (data); if (data) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 3cafa40e6ce3..903be7289c53 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -591,7 +591,7 @@ void msi_device_destroy_sysfs(struct device *dev) msi_for_each_desc(desc, dev, MSI_DESC_ALL) msi_sysfs_remove_desc(dev, desc); } -#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK || CONFIG_PCI_XEN */ +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS || CONFIG_PCI_XEN */ #else /* CONFIG_SYSFS */ static inline int msi_sysfs_create_group(struct device *dev) { return 0; } static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } diff --git a/kernel/kcmp.c b/kernel/kcmp.c index 7c1a65bd5f8d..76476aeee067 100644 --- a/kernel/kcmp.c +++ b/kernel/kcmp.c @@ -186,7 +186,7 @@ SYSCALL_DEFINE5(kcmp, pid_t, pid1, pid_t, pid2, int, type, ret = kcmp_ptr(task1->files, task2->files, KCMP_FILES); break; case KCMP_FS: - ret = kcmp_ptr(task1->fs, task2->fs, KCMP_FS); + ret = kcmp_ptr(task1->real_fs, task2->real_fs, KCMP_FS); break; case KCMP_SIGHAND: ret = kcmp_ptr(task1->sighand, task2->sighand, KCMP_SIGHAND); diff --git a/kernel/kcov.c b/kernel/kcov.c index 0b369e88c7c9..1df373fb562b 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -368,6 +368,7 @@ static void kcov_start(struct task_struct *t, struct kcov *kcov, WRITE_ONCE(t->kcov_mode, mode); } +/* operates on coverage-generator-owned fields */ static void kcov_stop(struct task_struct *t) { WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); @@ -377,16 +378,17 @@ static void kcov_stop(struct task_struct *t) t->kcov_area = NULL; } +/* operates on coverage-generator-owned fields */ static void kcov_task_reset(struct task_struct *t) { kcov_stop(t); t->kcov_sequence = 0; - t->kcov_handle = 0; } void kcov_task_init(struct task_struct *t) { kcov_task_reset(t); + t->kcov_remote = NULL; t->kcov_handle = current->kcov_handle; } @@ -423,11 +425,14 @@ static void kcov_remote_reset(struct kcov *kcov) static void kcov_disable(struct task_struct *t, struct kcov *kcov) __must_hold(&kcov->lock) { - kcov_task_reset(t); - if (kcov->remote) + if (kcov->remote) { + t->kcov_handle = 0; + t->kcov_remote = NULL; kcov_remote_reset(kcov); - else + } else { + kcov_task_reset(t); kcov_reset(kcov); + } } static void kcov_get(struct kcov *kcov) @@ -453,41 +458,47 @@ void kcov_task_exit(struct task_struct *t) unsigned long flags; kcov = t->kcov; - if (kcov == NULL) - return; - - spin_lock_irqsave(&kcov->lock, flags); - kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); - /* - * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, - * which comes down to: - * WARN_ON(!kcov->remote && kcov->t != t); - * - * For KCOV_REMOTE_ENABLE devices, the exiting task is either: - * - * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). - * In this case we should print a warning right away, since a task - * shouldn't be exiting when it's in a kcov coverage collection - * section. Here t points to the task that is collecting remote - * coverage, and t->kcov->t points to the thread that created the - * kcov device. Which means that to detect this case we need to - * check that t != t->kcov->t, and this gives us the following: - * WARN_ON(kcov->remote && kcov->t != t); - * - * 2. The task that created kcov exiting without calling KCOV_DISABLE, - * and then again we make sure that t->kcov->t == t: - * WARN_ON(kcov->remote && kcov->t != t); - * - * By combining all three checks into one we get: - */ - if (WARN_ON(kcov->t != t)) { + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This could be a remote task between kcov_remote_start() and + * kcov_remote_stop(). + * In this case we should print a warning right away, since a + * task shouldn't be exiting when it's in a kcov coverage + * collection section. + * + * Otherwise, this should be a task that created a local + * kcov instance and hasn't called KCOV_DISABLE. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); spin_unlock_irqrestore(&kcov->lock, flags); - return; + kcov_put(kcov); + } + kcov = t->kcov_remote; + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This is a KCOV_REMOTE_ENABLE device, and the task is the + * user task which has requested remote coverage collection. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(!kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); + spin_unlock_irqrestore(&kcov->lock, flags); + kcov_put(kcov); } - /* Just to not leave dangling references behind. */ - kcov_disable(t, kcov); - spin_unlock_irqrestore(&kcov->lock, flags); - kcov_put(kcov); } static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) @@ -629,9 +640,9 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, case KCOV_DISABLE: /* Disable coverage for the current task. */ unused = arg; - if (unused != 0 || current->kcov != kcov) - return -EINVAL; t = current; + if (unused != 0 || (kcov != t->kcov && kcov != t->kcov_remote)) + return -EINVAL; if (WARN_ON(kcov->t != t)) return -EINVAL; kcov_disable(t, kcov); @@ -641,7 +652,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, if (kcov->mode != KCOV_MODE_INIT || !kcov->area) return -EINVAL; t = current; - if (kcov->t != NULL || t->kcov != NULL) + if (kcov->t != NULL || t->kcov_remote != NULL) return -EBUSY; remote_arg = (struct kcov_remote_arg *)arg; mode = kcov_get_mode(remote_arg->trace_mode); @@ -651,8 +662,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, LONG_MAX / sizeof(unsigned long)) return -EINVAL; kcov->mode = mode; - t->kcov = kcov; - t->kcov_mode = KCOV_MODE_REMOTE; + t->kcov_remote = kcov; kcov->t = t; kcov->remote = true; kcov->remote_size = remote_arg->area_size; @@ -1083,11 +1093,11 @@ void kcov_remote_stop(void) EXPORT_SYMBOL(kcov_remote_stop); /* See the comment before kcov_remote_start() for usage details. */ -u64 kcov_common_handle(void) +struct kcov_common_handle_id kcov_common_handle(void) { if (!in_task()) - return 0; - return current->kcov_handle; + return (struct kcov_common_handle_id){ .val = 0 }; + return (struct kcov_common_handle_id){ .val = current->kcov_handle }; } EXPORT_SYMBOL(kcov_common_handle); @@ -1109,10 +1119,10 @@ static void __init selftest(void) * potentially traced functions in this region. */ start = jiffies; - current->kcov_mode = KCOV_MODE_TRACE_PC; + WRITE_ONCE(current->kcov_mode, KCOV_MODE_TRACE_PC); while ((jiffies - start) * MSEC_PER_SEC / HZ < 300) ; - current->kcov_mode = 0; + WRITE_ONCE(current->kcov_mode, 0); pr_err("done running self test\n"); } #endif diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index a43d2da0fe3e..dc770b9a6d05 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1146,9 +1146,11 @@ int kernel_kexec(void) goto Unlock; } - error = liveupdate_reboot(); - if (error) - goto Unlock; + if (!kexec_image->preserve_context) { + error = liveupdate_reboot(); + if (error) + goto Unlock; + } #ifdef CONFIG_KEXEC_JUMP if (kexec_image->preserve_context) { diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 2bfbb2d144e6..59fb9d71e9d8 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "kexec_internal.h" #ifdef CONFIG_KEXEC_SIG @@ -798,6 +799,16 @@ int kexec_add_buffer(struct kexec_buf *kbuf) return 0; } +static bool kexec_only_cma_segments(struct kimage *image) +{ + for (int i = 0; i < image->nr_segments; i++) { + if (!image->segment_cma[i]) + return false; + } + + return true; +} + /* Calculate and store the digest of segments */ static int kexec_calculate_store_digests(struct kimage *image) { @@ -822,6 +833,21 @@ static int kexec_calculate_store_digests(struct kimage *image) sha256_init(&sctx); + /* + * If KHO is enabled, the destinations are located in KHO scratch. + * KHO scratch can only contain early boot allocations and movable + * allocations. That means there is no risk of memory corruption by + * uncancelled DMA. + * + * If all segments were loaded into contiguous memory, there will be no + * relocations at all, so also no risk of corruption. + */ + if (image->type != KEXEC_TYPE_CRASH && + (kho_is_enabled() || kexec_only_cma_segments(image))) { + pr_debug("disabling checksum verification in purgatory\n"); + goto skip_checksum; + } + for (j = i = 0; i < image->nr_segments; i++) { struct kexec_segment *ksegment; @@ -867,6 +893,7 @@ static int kexec_calculate_store_digests(struct kimage *image) j++; } +skip_checksum: sha256_final(&sctx, digest); ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions", diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig index 1a8513f16ef7..c13af38ba23a 100644 --- a/kernel/liveupdate/Kconfig +++ b/kernel/liveupdate/Kconfig @@ -1,12 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Live Update and Kexec HandOver" - depends on !DEFERRED_STRUCT_PAGE_INIT config KEXEC_HANDOVER bool "kexec handover" depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE - depends on !DEFERRED_STRUCT_PAGE_INIT select MEMBLOCK_KHO_SCRATCH select KEXEC_FILE select LIBFDT diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile index d2f779cbe279..eec9d3ae07eb 100644 --- a/kernel/liveupdate/Makefile +++ b/kernel/liveupdate/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 luo-y := \ + kho_block.o \ luo_core.o \ luo_file.o \ luo_flb.o \ diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 1b592d86dc48..4834a809985a 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -459,6 +459,31 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages) } EXPORT_SYMBOL_GPL(kho_restore_pages); +/* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions + * may not be initialized yet at the time KHO deserializes preserved memory. + * KHO uses the struct page to store metadata and a later initialization would + * overwrite it. + * Ensure all the struct pages in the preservation are + * initialized. kho_preserved_memory_reserve() marks the reservation as noinit + * to make sure they don't get re-initialized later. + */ +static struct page *__init kho_get_preserved_page(phys_addr_t phys, + unsigned int order) +{ + unsigned long pfn = PHYS_PFN(phys); + int nid; + + if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) + return pfn_to_page(pfn); + + nid = early_pfn_to_nid(pfn); + for (unsigned long i = 0; i < (1UL << order); i++) + init_deferred_page(pfn + i, nid); + + return pfn_to_page(pfn); +} + static int __init kho_preserved_memory_reserve(phys_addr_t phys, unsigned int order) { @@ -467,7 +492,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, u64 sz; sz = 1 << (order + PAGE_SHIFT); - page = phys_to_page(phys); + page = kho_get_preserved_page(phys, order); /* Reserve the memory preserved in KHO in memblock */ memblock_reserve(phys, sz); @@ -593,20 +618,30 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { - phys_addr_t size; + /* + * If fixed sizes are not provided via command line, calculate them + * now. + */ + if (scratch_scale) { + phys_addr_t size; - if (!scratch_scale) - return; + size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); + size = size * scratch_scale / 100; + scratch_size_lowmem = size; - size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); - size = size * scratch_scale / 100; - scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); + size = size * scratch_scale / 100 - scratch_size_lowmem; + scratch_size_global = size; + } - size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, - NUMA_NO_NODE); - size = size * scratch_scale / 100 - scratch_size_lowmem; - scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + /* + * Scratch areas are released as MIGRATE_CMA. Round them up to the right + * size. + */ + scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) @@ -1584,35 +1619,10 @@ static __init int kho_init(void) } fs_initcall(kho_init); -static void __init kho_release_scratch(void) -{ - phys_addr_t start, end; - u64 i; - - memmap_init_kho_scratch_pages(); - - /* - * Mark scratch mem as CMA before we return it. That way we - * ensure that no kernel allocations happen on it. That means - * we can reuse it as scratch memory again later. - */ - __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, - MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) { - ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start)); - ulong end_pfn = pageblock_align(PFN_UP(end)); - ulong pfn; - - for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) - init_pageblock_migratetype(pfn_to_page(pfn), - MIGRATE_CMA, false); - } -} - void __init kho_memory_init(void) { if (kho_in.scratch_phys) { kho_scratch = phys_to_virt(kho_in.scratch_phys); - kho_release_scratch(); if (kho_mem_retrieve(kho_get_fdt())) kho_in.fdt_phys = 0; diff --git a/kernel/liveupdate/kho_block.c b/kernel/liveupdate/kho_block.c new file mode 100644 index 000000000000..0d2a342ef422 --- /dev/null +++ b/kernel/liveupdate/kho_block.c @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +/** + * DOC: KHO Serialization Blocks + * + * KHO provides a mechanism to preserve stateful data across a kexec handover + * by serializing it into memory blocks, and provides the common + * infrastructure for managing these blocks. + * + * Each block consists of a header (struct kho_block_header_ser) followed by an + * array of serialized entries. Multiple blocks are linked together via a + * physical pointer in the header, forming a linked list that can be easily + * traversed in both the current and the next kernel. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +/* + * Safeguard limit for the number of serialization blocks. This is used to + * prevent infinite loops and excessive memory allocation in case of memory + * corruption in the preserved state. + * + * With a 4KB page size, 10k blocks is about 40MB. For 32-byte entries + * (e.g. 4 u64s), each block holds up to 127 entries (accounting for the + * 16-byte header), allowing the block set to hold up to 1.27M entries. + */ +#define KHO_MAX_BLOCKS 10000 + +/** + * kho_block_set_init - Initialize a block set. + * @bs: The block set to initialize. + * @entry_size: The size of each entry in the blocks. + */ +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size) +{ + *bs = (struct kho_block_set)KHO_BLOCK_SET_INIT(*bs, entry_size); + WARN_ON_ONCE(!bs->count_per_block); +} + +/* Serialized entries start immediately after the block header */ +static void *kho_block_entries(struct kho_block *block) +{ + return (void *)(block->ser + 1); +} + +/* Get the address of the serialized entry at the specified index */ +static void *kho_block_entry(struct kho_block_set_it *it, u64 index) +{ + return kho_block_entries(it->block) + (index * it->bs->entry_size); +} + +/* Free serialized data */ +static void kho_block_free_ser(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + if (bs->incoming) + kho_restore_free(ser); + else + kho_unpreserve_free(ser); +} + +static struct kho_block_header_ser *kho_block_alloc_ser(struct kho_block_set *bs) +{ + WARN_ON_ONCE(bs->incoming); + return kho_alloc_preserve(KHO_BLOCK_SIZE); +} + +static int kho_block_add(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + struct kho_block *block, *last; + + if (bs->nblocks >= KHO_MAX_BLOCKS) + return -ENOSPC; + + block = kzalloc_obj(*block); + if (!block) + return -ENOMEM; + + block->ser = ser; + last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + list_add_tail(&block->list, &bs->blocks); + bs->nblocks++; + + if (last) + last->ser->next = virt_to_phys(ser); + else + bs->head_pa = virt_to_phys(ser); + + return 0; +} + +static int kho_block_set_grow_one(struct kho_block_set *bs) +{ + struct kho_block_header_ser *ser; + int err; + + ser = kho_block_alloc_ser(bs); + if (IS_ERR(ser)) + return PTR_ERR(ser); + + err = kho_block_add(bs, ser); + if (err) { + kho_block_free_ser(bs, ser); + return err; + } + + return 0; +} + +static void kho_block_set_shrink_one(struct kho_block_set *bs) +{ + struct kho_block *last, *new_last; + + if (list_empty(&bs->blocks)) + return; + + last = list_last_entry(&bs->blocks, struct kho_block, list); + list_del(&last->list); + bs->nblocks--; + kho_block_free_ser(bs, last->ser); + kfree(last); + + new_last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + if (new_last) + new_last->ser->next = 0; + else + bs->head_pa = 0; +} + +/** + * kho_block_set_grow - Expand the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Dynamically preallocates and links preserved memory blocks if the target + * entry count exceeds the current total capacity of the set, ensuring they + * are available during serialization/deserialization. + * + * Context: Caller must hold a lock protecting the block set. + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_grow(struct kho_block_set *bs, u64 count) +{ + long orig_nblocks = bs->nblocks; + int err; + + if (WARN_ON_ONCE(bs->incoming)) + return -EINVAL; + + while (count > bs->nblocks * bs->count_per_block) { + err = kho_block_set_grow_one(bs); + if (err) + goto err_shrink; + } + + return 0; + +err_shrink: + while (bs->nblocks > orig_nblocks) + kho_block_set_shrink_one(bs); + return err; +} + +/** + * kho_block_set_shrink - Shrink the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Releases and unallocates redundant preserved memory blocks. Checks if the + * last block in the set can be removed because the remaining entry count is + * fully accommodated by the preceding blocks. + * + * Note: It is the caller's responsibility to ensure that entries are removed + * in the reverse order of their insertion. Because shrinking destroys the last + * block in the set, removing entries in any other order would corrupt active + * data. + * + * Context: Caller must hold a lock protecting the block set. + */ +void kho_block_set_shrink(struct kho_block_set *bs, u64 count) +{ + while (bs->nblocks > 0 && count <= (bs->nblocks - 1) * bs->count_per_block) + kho_block_set_shrink_one(bs); +} + +/* + * kho_block_set_is_cyclic - Check for cycles in a linked list of blocks. + * Uses Floyd's cycle-finding algorithm to ensure sanity of the incoming list. + * + * Return: true if a cycle or corruption is detected, false otherwise. + */ +static bool kho_block_set_is_cyclic(struct kho_block_set *bs) +{ + struct kho_block_header_ser *fast; + struct kho_block_header_ser *slow; + int count = 0; + + fast = phys_to_virt(bs->head_pa); + slow = fast; + + while (fast) { + if (count++ >= KHO_MAX_BLOCKS) { + pr_err("Block set is corrupted\n"); + return true; + } + + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + slow = phys_to_virt(slow->next); + + if (slow == fast) { + pr_err("Block set is corrupted\n"); + return true; + } + } + + return false; +} + +/** + * kho_block_set_restore - Restore a block set from a physical address. + * @bs: The block set to restore. + * @head_pa: Physical address of the first block header. + * + * Restores a serialized block set from a given physical address. The caller is + * responsible for ensuring that the block set @bs has been allocated and + * initialized prior to calling this function. + * + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa) +{ + struct kho_block_header_ser *ser; + u64 next_pa = head_pa; + int err; + + /* Restored block sets use size from the previous kernel */ + bs->incoming = true; + if (!head_pa) + return 0; + + bs->head_pa = head_pa; + if (kho_block_set_is_cyclic(bs)) { + bs->head_pa = 0; + return -EINVAL; + } + + while (next_pa) { + ser = phys_to_virt(next_pa); + if (!ser->count || ser->count > bs->count_per_block) { + pr_warn("Block contains invalid entry count: %llu\n", + ser->count); + err = -EINVAL; + goto err_destroy; + } + err = kho_block_add(bs, ser); + if (err) + goto err_destroy; + next_pa = ser->next; + } + + return 0; + +err_destroy: + kho_block_set_destroy(bs); + + /* Free the remaining un-restored blocks in the physical chain */ + while (next_pa) { + struct kho_block_header_ser *next_ser = phys_to_virt(next_pa); + + next_pa = next_ser->next; + kho_block_free_ser(bs, next_ser); + } + return err; +} + +/** + * kho_block_set_destroy - Destroy all blocks in a block set. + * @bs: The block set. + */ +void kho_block_set_destroy(struct kho_block_set *bs) +{ + struct kho_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &bs->blocks, list) { + list_del(&block->list); + kho_block_free_ser(bs, block->ser); + kfree(block); + } + bs->nblocks = 0; + bs->head_pa = 0; +} + +/** + * kho_block_set_clear - Clear all serialized data in a block set. + * @bs: The block set to clear. + */ +void kho_block_set_clear(struct kho_block_set *bs) +{ + struct kho_block *block; + + list_for_each_entry(block, &bs->blocks, list) { + block->ser->count = 0; + memset(block->ser + 1, 0, KHO_BLOCK_SIZE - sizeof(*block->ser)); + } +} + +/** + * kho_block_set_it_init - Initialize a block set iterator. + * @it: The iterator to initialize. + * @bs: The block set to iterate over. + */ +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs) +{ + it->bs = bs; + it->block = list_first_entry_or_null(&bs->blocks, struct kho_block, list); + it->i = 0; +} + +/** + * kho_block_set_it_reserve_entry - Reserve and return the next available slot for writing. + * @it: The block iterator. + * + * Reserves a slot in the current block during state serialization to add a new + * entry, advancing the internal index. If the current block is full, it + * automatically moves to the next block in the set. + * + * Return: A pointer to the reserved entry slot, or NULL if the block set's + * capacity is fully exhausted. + */ +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it) +{ + void *entry; + + if (!it->block) + return NULL; + + if (it->i == it->bs->count_per_block) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + entry = kho_block_entry(it, it->i++); + it->block->ser->count = it->i; + return entry; +} + +/** + * kho_block_set_it_read_entry - Read the next serialized entry from the block set. + * @it: The block iterator. + * + * Iterates through previously written entries during state deserialization, + * respecting the actual count stored in each block's header. + * + * Return: A pointer to the next serialized entry, or NULL if all serialized + * entries have been read. + */ +void *kho_block_set_it_read_entry(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == it->block->ser->count) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + return kho_block_entry(it, it->i++); +} + +/** + * kho_block_set_it_prev - Return the previous entry slot in the block set. + * @it: The block iterator. + * + * If the current index is at the start of a block, it automatically moves to + * the end of the previous block. + * + * Return: A pointer to the previous entry slot, or NULL if at the very + * beginning of the block set. + */ +void *kho_block_set_it_prev(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == 0) { + if (list_is_first(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_prev_entry(it->block, list); + it->i = it->bs->count_per_block; + } + + return kho_block_entry(it, --it->i); +} diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 803f51c84275..1b2bda22902d 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -36,6 +36,10 @@ * * LUO uses Kexec Handover to transfer memory state from the current kernel to * the next kernel. For more details see Documentation/core-api/kho/index.rst. + * + * .. note:: + * To enable LUO, boot the kernel with the ``liveupdate=on`` command line + * parameter. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -50,22 +54,19 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include "kexec_handover_internal.h" #include "luo_internal.h" static struct { bool enabled; - void *fdt_out; - void *fdt_in; + struct luo_ser *luo_ser_out; u64 liveupdate_num; } luo_global; @@ -82,9 +83,10 @@ early_param("liveupdate", early_liveupdate_param); static int __init luo_early_startup(void) { - phys_addr_t fdt_phys; - int err, ln_size; - const void *ptr; + phys_addr_t luo_ser_phys; + struct luo_ser *luo_ser; + size_t len; + int err; if (!kho_is_enabled()) { if (liveupdate_enabled()) @@ -93,48 +95,43 @@ static int __init luo_early_startup(void) return 0; } - /* Retrieve LUO subtree, and verify its format. */ - err = kho_retrieve_subtree(LUO_FDT_KHO_ENTRY_NAME, &fdt_phys, NULL); + /* Retrieve LUO state from KHO. */ + err = kho_retrieve_subtree(LUO_KHO_ENTRY_NAME, &luo_ser_phys, &len); if (err) { if (err != -ENOENT) { - pr_err("failed to retrieve FDT '%s' from KHO: %pe\n", - LUO_FDT_KHO_ENTRY_NAME, ERR_PTR(err)); + pr_err("failed to retrieve LUO state '%s' from KHO: %pe\n", + LUO_KHO_ENTRY_NAME, ERR_PTR(err)); return err; } return 0; } - luo_global.fdt_in = phys_to_virt(fdt_phys); - err = fdt_node_check_compatible(luo_global.fdt_in, 0, - LUO_FDT_COMPATIBLE); - if (err) { - pr_err("FDT '%s' is incompatible with '%s' [%d]\n", - LUO_FDT_KHO_ENTRY_NAME, LUO_FDT_COMPATIBLE, err); - + if (len < sizeof(*luo_ser)) { + pr_err("LUO state is too small (%zu < %zu)\n", len, sizeof(*luo_ser)); return -EINVAL; } - ln_size = 0; - ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_LIVEUPDATE_NUM, - &ln_size); - if (!ptr || ln_size != sizeof(luo_global.liveupdate_num)) { - pr_err("Unable to get live update number '%s' [%d]\n", - LUO_FDT_LIVEUPDATE_NUM, ln_size); - + luo_ser = phys_to_virt(luo_ser_phys); + if (strncmp(luo_ser->compatible, LUO_ABI_COMPATIBLE, LUO_ABI_COMPAT_LEN)) { + pr_err("LUO state is incompatible with '%s'\n", LUO_ABI_COMPATIBLE); return -EINVAL; } - luo_global.liveupdate_num = get_unaligned((u64 *)ptr); + luo_global.liveupdate_num = luo_ser->liveupdate_num; pr_info("Retrieved live update data, liveupdate number: %lld\n", luo_global.liveupdate_num); - err = luo_session_setup_incoming(luo_global.fdt_in); + err = luo_session_setup_incoming(luo_ser->sessions_pa); if (err) - return err; + goto out_free_ser; + + luo_flb_setup_incoming(luo_ser->flbs_pa); - err = luo_flb_setup_incoming(luo_global.fdt_in); + err = 0; +out_free_ser: + kho_restore_free(luo_ser); return err; } @@ -153,42 +150,38 @@ static int __init liveupdate_early_init(void) } early_initcall(liveupdate_early_init); -/* Called during boot to create outgoing LUO fdt tree */ -static int __init luo_fdt_setup(void) +/* Called during boot to create outgoing LUO state */ +static int __init luo_state_setup(void) { - const u64 ln = luo_global.liveupdate_num + 1; - void *fdt_out; + struct luo_ser *luo_ser; int err; - fdt_out = kho_alloc_preserve(LUO_FDT_SIZE); - if (IS_ERR(fdt_out)) { - pr_err("failed to allocate/preserve FDT memory\n"); - return PTR_ERR(fdt_out); + luo_ser = kho_alloc_preserve(sizeof(*luo_ser)); + if (IS_ERR(luo_ser)) { + pr_err("failed to allocate/preserve LUO state memory\n"); + return PTR_ERR(luo_ser); } - err = fdt_create(fdt_out, LUO_FDT_SIZE); - err |= fdt_finish_reservemap(fdt_out); - err |= fdt_begin_node(fdt_out, ""); - err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln)); - err |= luo_session_setup_outgoing(fdt_out); - err |= luo_flb_setup_outgoing(fdt_out); - err |= fdt_end_node(fdt_out); - err |= fdt_finish(fdt_out); + strscpy(luo_ser->compatible, LUO_ABI_COMPATIBLE, sizeof(luo_ser->compatible)); + luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; + + luo_session_setup_outgoing(&luo_ser->sessions_pa); + + err = luo_flb_setup_outgoing(&luo_ser->flbs_pa); if (err) - goto exit_free; + goto exit_free_luo_ser; - err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out, - fdt_totalsize(fdt_out)); + err = kho_add_subtree(LUO_KHO_ENTRY_NAME, luo_ser, sizeof(*luo_ser)); if (err) - goto exit_free; - luo_global.fdt_out = fdt_out; + goto exit_free_luo_ser; + + luo_global.luo_ser_out = luo_ser; return 0; -exit_free: - kho_unpreserve_free(fdt_out); - pr_err("failed to prepare LUO FDT: %d\n", err); +exit_free_luo_ser: + kho_unpreserve_free(luo_ser); + pr_err("failed to prepare LUO state: %d\n", err); return err; } @@ -204,7 +197,7 @@ static int __init luo_late_startup(void) if (!liveupdate_enabled()) return 0; - err = luo_fdt_setup(); + err = luo_state_setup(); if (err) luo_global.enabled = false; diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index a0a419085e28..c39f96961a85 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -118,11 +118,6 @@ static LIST_HEAD(luo_file_handler_list); /* Keep track of files being preserved by LUO */ static DEFINE_XARRAY(luo_preserved_files); -/* 2 4K pages, give space for 128 files per file_set */ -#define LUO_FILE_PGCNT 2ul -#define LUO_FILE_MAX \ - ((LUO_FILE_PGCNT << PAGE_SHIFT) / sizeof(struct luo_file_ser)) - /** * struct luo_file - Represents a single preserved file instance. * @fh: Pointer to the &struct liveupdate_file_handler that manages @@ -174,39 +169,6 @@ struct luo_file { u64 token; }; -static int luo_alloc_files_mem(struct luo_file_set *file_set) -{ - size_t size; - void *mem; - - if (file_set->files) - return 0; - - WARN_ON_ONCE(file_set->count); - - size = LUO_FILE_PGCNT << PAGE_SHIFT; - mem = kho_alloc_preserve(size); - if (IS_ERR(mem)) - return PTR_ERR(mem); - - file_set->files = mem; - - return 0; -} - -static void luo_free_files_mem(struct luo_file_set *file_set) -{ - /* If file_set has files, no need to free preservation memory */ - if (file_set->count) - return; - - if (!file_set->files) - return; - - kho_unpreserve_free(file_set->files); - file_set->files = NULL; -} - static unsigned long luo_get_id(struct liveupdate_file_handler *fh, struct file *file) { @@ -276,16 +238,15 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) if (luo_token_is_used(file_set, token)) return -EEXIST; - if (file_set->count == LUO_FILE_MAX) - return -ENOSPC; + err = kho_block_set_grow(&file_set->block_set, file_set->count + 1); + if (err) + return err; file = fget(fd); - if (!file) - return -EBADF; - - err = luo_alloc_files_mem(file_set); - if (err) - goto err_fput; + if (!file) { + err = -EBADF; + goto err_shrink; + } err = -ENOENT; down_read(&luo_register_rwlock); @@ -300,7 +261,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) /* err is still -ENOENT if no handler was found */ if (err) - goto err_free_files_mem; + goto err_fput; err = xa_insert(&luo_preserved_files, luo_get_id(fh, file), file, GFP_KERNEL); @@ -343,10 +304,10 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) xa_erase(&luo_preserved_files, luo_get_id(fh, file)); err_module_put: module_put(fh->ops->owner); -err_free_files_mem: - luo_free_files_mem(file_set); err_fput: fput(file); +err_shrink: + kho_block_set_shrink(&file_set->block_set, file_set->count); return err; } @@ -385,19 +346,21 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set) args.private_data = luo_file->private_data; luo_file->fh->ops->unpreserve(&args); luo_flb_file_unpreserve(luo_file->fh); - module_put(luo_file->fh->ops->owner); xa_erase(&luo_preserved_files, luo_get_id(luo_file->fh, luo_file->file)); + module_put(luo_file->fh->ops->owner); + list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); fput(luo_file->file); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - luo_free_files_mem(file_set); + kho_block_set_destroy(&file_set->block_set); } static int luo_file_freeze_one(struct luo_file_set *file_set, @@ -453,7 +416,7 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, luo_file_unfreeze_one(file_set, luo_file); } - memset(file_set->files, 0, LUO_FILE_PGCNT << PAGE_SHIFT); + kho_block_set_clear(&file_set->block_set); } /** @@ -492,19 +455,24 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, int luo_file_freeze(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { - struct luo_file_ser *file_ser = file_set->files; struct luo_file *luo_file; + struct kho_block_set_it it; int err; - int i; if (!file_set->count) return 0; - if (WARN_ON(!file_ser)) - return -EINVAL; + kho_block_set_it_init(&it, &file_set->block_set); - i = 0; list_for_each_entry(luo_file, &file_set->files_list, list) { + struct luo_file_ser *file_ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!file_ser)) { + err = -ENOSPC; + goto err_unfreeze; + } + err = luo_file_freeze_one(file_set, luo_file); if (err < 0) { pr_warn("Freeze failed for token[%#0llx] handler[%s] err[%pe]\n", @@ -513,16 +481,14 @@ int luo_file_freeze(struct luo_file_set *file_set, goto err_unfreeze; } - strscpy(file_ser[i].compatible, luo_file->fh->compatible, - sizeof(file_ser[i].compatible)); - file_ser[i].data = luo_file->serialized_data; - file_ser[i].token = luo_file->token; - i++; + strscpy(file_ser->compatible, luo_file->fh->compatible, + sizeof(file_ser->compatible)); + file_ser->data = luo_file->serialized_data; + file_ser->token = luo_file->token; } file_set_ser->count = file_set->count; - if (file_set->files) - file_set_ser->files = virt_to_phys(file_set->files); + file_set_ser->files = kho_block_set_head_pa(&file_set->block_set); return 0; @@ -677,7 +643,6 @@ static void luo_file_finish_one(struct luo_file_set *file_set, luo_file->fh->ops->finish(&args); luo_flb_file_finish(luo_file->fh); - module_put(luo_file->fh->ops->owner); } /** @@ -738,17 +703,56 @@ int luo_file_finish(struct luo_file_set *file_set) luo_get_id(luo_file->fh, luo_file->file)); fput(luo_file->file); } + module_put(luo_file->fh->ops->owner); list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - if (file_set->files) { - kho_restore_free(file_set->files); - file_set->files = NULL; + kho_block_set_destroy(&file_set->block_set); + + return 0; +} + +static int luo_file_deserialize_one(struct luo_file_set *file_set, + struct luo_file_ser *ser) +{ + struct liveupdate_file_handler *fh; + bool handler_found = false; + struct luo_file *luo_file; + + down_read(&luo_register_rwlock); + list_private_for_each_entry(fh, &luo_file_handler_list, list) { + if (!strcmp(fh->compatible, ser->compatible)) { + if (try_module_get(fh->ops->owner)) + handler_found = true; + break; + } + } + up_read(&luo_register_rwlock); + + if (!handler_found) { + pr_warn("No registered handler for compatible '%.*s'\n", + (int)sizeof(ser->compatible), + ser->compatible); + return -ENOENT; } + luo_file = kzalloc_obj(*luo_file); + if (!luo_file) { + module_put(fh->ops->owner); + return -ENOMEM; + } + + luo_file->fh = fh; + luo_file->file = NULL; + luo_file->serialized_data = ser->data; + luo_file->token = ser->token; + mutex_init(&luo_file->mutex); + list_add_tail(&luo_file->list, &file_set->files_list); + return 0; } @@ -781,15 +785,18 @@ int luo_file_deserialize(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { struct luo_file_ser *file_ser; - u64 i; + struct kho_block_set_it it; + int err; if (!file_set_ser->files) { WARN_ON(file_set_ser->count); return 0; } - file_set->count = file_set_ser->count; - file_set->files = phys_to_virt(file_set_ser->files); + file_set->count = 0; + err = kho_block_set_restore(&file_set->block_set, file_set_ser->files); + if (err) + return err; /* * Note on error handling: @@ -806,55 +813,50 @@ int luo_file_deserialize(struct luo_file_set *file_set, * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - file_ser = file_set->files; - for (i = 0; i < file_set->count; i++) { - struct liveupdate_file_handler *fh; - bool handler_found = false; - struct luo_file *luo_file; + kho_block_set_it_init(&it, &file_set->block_set); + while ((file_ser = kho_block_set_it_read_entry(&it))) { + err = luo_file_deserialize_one(file_set, file_ser); + if (err) + goto err_destroy_blocks; + file_set->count++; + } - down_read(&luo_register_rwlock); - list_private_for_each_entry(fh, &luo_file_handler_list, list) { - if (!strcmp(fh->compatible, file_ser[i].compatible)) { - if (try_module_get(fh->ops->owner)) - handler_found = true; - break; - } - } - up_read(&luo_register_rwlock); + if (file_set->count != file_set_ser->count) { + pr_warn("File count mismatch: expected %llu, found %llu\n", + file_set_ser->count, file_set->count); + err = -EINVAL; + goto err_destroy_blocks; + } - if (!handler_found) { - pr_warn("No registered handler for compatible '%.*s'\n", - (int)sizeof(file_ser[i].compatible), - file_ser[i].compatible); - return -ENOENT; - } + return 0; - luo_file = kzalloc_obj(*luo_file); - if (!luo_file) { - module_put(fh->ops->owner); - return -ENOMEM; - } +err_destroy_blocks: + while (!list_empty(&file_set->files_list)) { + struct luo_file *luo_file; - luo_file->fh = fh; - luo_file->file = NULL; - luo_file->serialized_data = file_ser[i].data; - luo_file->token = file_ser[i].token; - mutex_init(&luo_file->mutex); - list_add_tail(&luo_file->list, &file_set->files_list); + luo_file = list_first_entry(&file_set->files_list, + struct luo_file, list); + list_del(&luo_file->list); + module_put(luo_file->fh->ops->owner); + mutex_destroy(&luo_file->mutex); + kfree(luo_file); } - - return 0; + file_set->count = 0; + kho_block_set_destroy(&file_set->block_set); + return err; } void luo_file_set_init(struct luo_file_set *file_set) { INIT_LIST_HEAD(&file_set->files_list); + kho_block_set_init(&file_set->block_set, sizeof(struct luo_file_ser)); } void luo_file_set_destroy(struct luo_file_set *file_set) { WARN_ON(file_set->count); WARN_ON(!list_empty(&file_set->files_list)); + WARN_ON(!kho_block_set_is_empty(&file_set->block_set)); } /** diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 00f5494812c4..5c27134ce7ba 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -44,13 +44,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include "luo_internal.h" #define LUO_FLB_PGCNT 1ul @@ -111,7 +109,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - if (!private->outgoing.count) { + if (!refcount_read(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; int err; @@ -126,8 +124,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) } private->outgoing.data = args.data; private->outgoing.obj = args.obj; + refcount_set(&private->outgoing.count, 1); + } else { + refcount_inc(&private->outgoing.count); } - private->outgoing.count++; } return 0; @@ -138,8 +138,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - private->outgoing.count--; - if (!private->outgoing.count) { + if (refcount_dec_and_test(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; args.flb = flb; @@ -164,7 +163,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) bool found = false; int err; - guard(mutex)(&private->incoming.lock); + lockdep_assert_held(&private->incoming.lock); if (private->incoming.finished) return -ENODATA; @@ -178,7 +177,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) for (int i = 0; i < fh->header_ser->count; i++) { if (!strcmp(fh->ser[i].name, flb->compatible)) { private->incoming.data = fh->ser[i].data; - private->incoming.count = fh->ser[i].count; + refcount_set(&private->incoming.count, fh->ser[i].count); found = true; break; } @@ -205,16 +204,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_finish_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); - u64 count; - - scoped_guard(mutex, &private->incoming.lock) - count = --private->incoming.count; + struct liveupdate_flb_op_args args = {0}; - if (!count) { - struct liveupdate_flb_op_args args = {0}; + scoped_guard(mutex, &private->incoming.lock) { + if (!refcount_dec_and_test(&private->incoming.count)) + return; if (!private->incoming.retrieved) { int err = luo_flb_retrieve_one(flb); @@ -223,16 +220,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb) return; } - scoped_guard(mutex, &private->incoming.lock) { - args.flb = flb; - args.obj = private->incoming.obj; - flb->ops->finish(&args); + args.flb = flb; + args.obj = private->incoming.obj; + flb->ops->finish(&args); - private->incoming.data = 0; - private->incoming.obj = NULL; - private->incoming.finished = true; - module_put(flb->ops->owner); - } + private->incoming.data = 0; + private->incoming.obj = NULL; + private->incoming.finished = true; + module_put(flb->ops->owner); } } @@ -315,7 +310,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_finish_one(iter->flb); + liveupdate_flb_put_incoming(iter->flb); } static void luo_flb_unregister_one(struct liveupdate_file_handler *fh, @@ -512,6 +507,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) if (!liveupdate_enabled()) return -EOPNOTSUPP; + guard(mutex)(&private->incoming.lock); + if (!private->incoming.obj) { int err = luo_flb_retrieve_one(flb); @@ -519,7 +516,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) return err; } - guard(mutex)(&private->incoming.lock); + refcount_inc(&private->incoming.count); *objp = private->incoming.obj; return 0; @@ -552,27 +549,15 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return 0; } -int __init luo_flb_setup_outgoing(void *fdt_out) +int __init luo_flb_setup_outgoing(u64 *flbs_pa) { struct luo_flb_header_ser *header_ser; - u64 header_ser_pa; - int err; header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT); if (IS_ERR(header_ser)) return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); - - err = fdt_begin_node(fdt_out, LUO_FDT_FLB_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_FLB_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_FLB_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); - - if (err) - goto err_unpreserve; + *flbs_pa = virt_to_phys(header_ser); header_ser->pgcnt = LUO_FLB_PGCNT; luo_flb_global.outgoing.header_ser = header_ser; @@ -580,53 +565,19 @@ int __init luo_flb_setup_outgoing(void *fdt_out) luo_flb_global.outgoing.active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - - return err; } -int __init luo_flb_setup_incoming(void *fdt_in) +void __init luo_flb_setup_incoming(u64 flbs_pa) { struct luo_flb_header_ser *header_ser; - int err, header_size, offset; - const void *ptr; - u64 header_ser_pa; - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_FLB_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get FLB node [%s]\n", LUO_FDT_FLB_NODE_NAME); - - return -ENOENT; - } - - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_FLB_COMPATIBLE); - if (err) { - pr_err("FLB node is incompatible with '%s' [%d]\n", - LUO_FDT_FLB_COMPATIBLE, err); - - return -EINVAL; - } - - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_FLB_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get FLB header property '%s' [%d]\n", - LUO_FDT_FLB_HEADER, header_size); - - return -EINVAL; - } - - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); + if (!flbs_pa) + return; + header_ser = phys_to_virt(flbs_pa); luo_flb_global.incoming.header_ser = header_ser; luo_flb_global.incoming.ser = (void *)(header_ser + 1); luo_flb_global.incoming.active = true; - - return 0; } /** @@ -652,12 +603,13 @@ void luo_flb_serialize(void) guard(rwsem_read)(&luo_register_rwlock); list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) { struct luo_flb_private *private = luo_flb_get_private(gflb); + long count = refcount_read(&private->outgoing.count); - if (private->outgoing.count > 0) { + if (count > 0) { strscpy(fh->ser[i].name, gflb->compatible, sizeof(fh->ser[i].name)); fh->ser[i].data = private->outgoing.data; - fh->ser[i].count = private->outgoing.count; + fh->ser[i].count = count; i++; } } diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index 875844d7a41d..64879ffe7378 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -10,6 +10,7 @@ #include #include +#include struct luo_ucmd { void __user *ubuffer; @@ -44,22 +45,20 @@ static inline int luo_ucmd_respond(struct luo_ucmd *ucmd, * struct luo_file_set - A set of files that belong to the same sessions. * @files_list: An ordered list of files associated with this session, it is * ordered by preservation time. - * @files: The physically contiguous memory block that holds the serialized - * state of files. + * @block_set: The set of serialization blocks. * @count: A counter tracking the number of files currently stored in the * @files_list for this session. */ struct luo_file_set { struct list_head files_list; - struct luo_file_ser *files; - long count; + struct kho_block_set block_set; + u64 count; }; /** * struct luo_session - Represents an active or incoming Live Update session. * @name: A unique name for this session, used for identification and * retrieval. - * @ser: Pointer to the serialized data for this session. * @list: A list_head member used to link this session into a global list * of either outgoing (to be preserved) or incoming (restored from * previous kernel) sessions. @@ -70,7 +69,6 @@ struct luo_file_set { */ struct luo_session { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; - struct luo_session_ser *ser; struct list_head list; bool retrieved; struct luo_file_set file_set; @@ -81,8 +79,8 @@ extern struct rw_semaphore luo_register_rwlock; int luo_session_create(const char *name, struct file **filep); int luo_session_retrieve(const char *name, struct file **filep); -int __init luo_session_setup_outgoing(void *fdt); -int __init luo_session_setup_incoming(void *fdt); +void __init luo_session_setup_outgoing(u64 *sessions_pa); +int __init luo_session_setup_incoming(u64 sessions_pa); int luo_session_serialize(void); int luo_session_deserialize(void); @@ -104,8 +102,8 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh); void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh); void luo_flb_file_finish(struct liveupdate_file_handler *fh); void luo_flb_unregister_all(struct liveupdate_file_handler *fh); -int __init luo_flb_setup_outgoing(void *fdt); -int __init luo_flb_setup_incoming(void *fdt); +int __init luo_flb_setup_outgoing(u64 *flbs_pa); +void __init luo_flb_setup_incoming(u64 flbs_pa); void luo_flb_serialize(void); #ifdef CONFIG_LIVEUPDATE_TEST diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 7a42385dabe2..b79b2a488974 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -24,10 +24,10 @@ * ioctls on /dev/liveupdate. * * - Serialization: Session metadata is preserved using the KHO framework. When - * a live update is triggered via kexec, an array of `struct luo_session_ser` - * is populated and placed in a preserved memory region. An FDT node is also - * created, containing the count of sessions and the physical address of this - * array. + * a live update is triggered via kexec, session metadata is serialized into + * a chain of linked-blocks and placed in a preserved memory region. The + * physical address of the first block header is stored in the centralized + * `struct luo_ser` structure. * * Session Lifecycle: * @@ -46,6 +46,38 @@ * 4. Retrieval: A userspace agent in the new kernel can then call * `luo_session_retrieve()` with a session name to get a new file * descriptor and access the preserved state. + * + * Locking: + * + * The LUO session subsystem uses a three-tier locking hierarchy to ensure thread + * safety and prevent deadlocks during concurrent session mutations and kexec + * serialization: + * + * 1. `luo_session_serialize_rwsem` (global rwsem): + * Protects session mutations (creation, retrieval, release, and ioctls) + * against the serialization process during reboot. + * + * - Readers: Taken by any path modifying or accessing session state (e.g., + * `luo_session_create()`, `luo_session_retrieve()`, `luo_session_release()`, + * and `luo_session_ioctl()`). + * - Writer: Taken by the serialization process (`luo_session_serialize()`) + * during reboot. On success, the write lock is held indefinitely to freeze + * the subsystem. On failure, it is released to allow recovery. + * + * 2. `luo_session_header->rwsem` (per-list rwsem): + * Synchronizes list-level operations for the incoming and outgoing session headers. + * + * - Writer: Taken during list mutation operations (inserting or removing a + * session from the list). + * - Reader: Taken when traversing the list (e.g., retrieving a session by name). + * + * 3. `luo_session->mutex` (per-session mutex): + * Protects the internal state and file sets of an individual session. It is + * acquired during per-session operations such as preserving, retrieving, + * or freezing files. + * + * Lock Hierarchy: + * `luo_session_serialize_rwsem` -> `luo_session_header->rwsem` -> `luo_session->mutex` */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -58,41 +90,34 @@ #include #include #include +#include #include -#include #include #include #include #include #include -#include #include #include "luo_internal.h" -/* 16 4K pages, give space for 744 sessions */ -#define LUO_SESSION_PGCNT 16ul -#define LUO_SESSION_MAX (((LUO_SESSION_PGCNT << PAGE_SHIFT) - \ - sizeof(struct luo_session_header_ser)) / \ - sizeof(struct luo_session_ser)) - +static DECLARE_RWSEM(luo_session_serialize_rwsem); /** * struct luo_session_header - Header struct for managing LUO sessions. - * @count: The number of sessions currently tracked in the @list. - * @list: The head of the linked list of `struct luo_session` instances. - * @rwsem: A read-write semaphore providing synchronized access to the - * session list and other fields in this structure. - * @header_ser: The header data of serialization array. - * @ser: The serialized session data (an array of - * `struct luo_session_ser`). - * @active: Set to true when first initialized. If previous kernel did not - * send session data, active stays false for incoming. + * @count: The number of sessions currently tracked in the @list. + * @list: The head of the linked list of `struct luo_session` instances. + * @rwsem: A read-write semaphore providing synchronized access to the + * session list and other fields in this structure. + * @block_set: The set of serialization blocks. + * @sessions_pa: Points to the location of sessions_pa within struct luo_ser. + * @active: Set to true when first initialized. If previous kernel did not + * send session data, active stays false for incoming. */ struct luo_session_header { long count; struct list_head list; struct rw_semaphore rwsem; - struct luo_session_header_ser *header_ser; - struct luo_session_ser *ser; + struct kho_block_set block_set; + u64 *sessions_pa; bool active; }; @@ -110,10 +135,14 @@ static struct luo_session_global luo_session_global = { .incoming = { .list = LIST_HEAD_INIT(luo_session_global.incoming.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.incoming.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.incoming.block_set, + sizeof(struct luo_session_ser)), }, .outgoing = { .list = LIST_HEAD_INIT(luo_session_global.outgoing.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.outgoing.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.outgoing.block_set, + sizeof(struct luo_session_ser)), }, }; @@ -144,6 +173,7 @@ static int luo_session_insert(struct luo_session_header *sh, struct luo_session *session) { struct luo_session *it; + int err; guard(rwsem_write)(&sh->rwsem); @@ -152,8 +182,9 @@ static int luo_session_insert(struct luo_session_header *sh, * for new session. */ if (sh == &luo_session_global.outgoing) { - if (sh->count == LUO_SESSION_MAX) - return -ENOMEM; + err = kho_block_set_grow(&sh->block_set, sh->count + 1); + if (err) + return err; } /* @@ -178,6 +209,8 @@ static void luo_session_remove(struct luo_session_header *sh, guard(rwsem_write)(&sh->rwsem); list_del(&session->list); sh->count--; + if (sh == &luo_session_global.outgoing) + kho_block_set_shrink(&sh->block_set, sh->count); } static int luo_session_finish_one(struct luo_session *session) @@ -205,6 +238,7 @@ static int luo_session_release(struct inode *inodep, struct file *filep) struct luo_session *session = filep->private_data; struct luo_session_header *sh; + guard(rwsem_read)(&luo_session_serialize_rwsem); /* If retrieved is set, it means this session is from incoming list */ if (session->retrieved) { int err = luo_session_finish_one(session); @@ -256,10 +290,11 @@ static int luo_session_retrieve_fd(struct luo_session *session, if (argp->fd < 0) return argp->fd; - guard(mutex)(&session->mutex); + mutex_lock(&session->mutex); err = luo_retrieve_file(&session->file_set, argp->token, &file); + mutex_unlock(&session->mutex); if (err < 0) - goto err_put_fd; + goto err_put_fd; err = luo_ucmd_respond(ucmd, sizeof(*argp)); if (err) @@ -289,38 +324,80 @@ static int luo_session_finish(struct luo_session *session, return luo_ucmd_respond(ucmd, sizeof(*argp)); } +static int luo_session_get_name(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_get_name *argp = ucmd->cmd; + + if (argp->reserved != 0) + return -EINVAL; + + strscpy((char *)argp->name, session->name, sizeof(argp->name)); + + return luo_ucmd_respond(ucmd, sizeof(*argp)); +} + union ucmd_buffer { struct liveupdate_session_finish finish; struct liveupdate_session_preserve_fd preserve; struct liveupdate_session_retrieve_fd retrieve; + struct liveupdate_session_get_name get_name; +}; + +/* Type of sessions the ioctl applies to. */ +enum luo_ioctl_type { + LUO_IOCTL_INCOMING, + LUO_IOCTL_OUTGOING, + LUO_IOCTL_ALL, }; struct luo_ioctl_op { unsigned int size; unsigned int min_size; unsigned int ioctl_num; + enum luo_ioctl_type type; int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd); }; -#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ +#define IOCTL_OP(_ioctl, _fn, _struct, _last, _type) \ [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \ .size = sizeof(_struct) + \ BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ sizeof(_struct)), \ .min_size = offsetofend(_struct, _last), \ .ioctl_num = _ioctl, \ + .type = _type, \ .execute = _fn, \ } static const struct luo_ioctl_op luo_session_ioctl_ops[] = { IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish, - struct liveupdate_session_finish, reserved), + struct liveupdate_session_finish, reserved, LUO_IOCTL_INCOMING), IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd, - struct liveupdate_session_preserve_fd, token), + struct liveupdate_session_preserve_fd, token, LUO_IOCTL_OUTGOING), IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, - struct liveupdate_session_retrieve_fd, token), + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING), + IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name, + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_ALL), }; +static bool luo_ioctl_type_valid(struct luo_session *session, + const struct luo_ioctl_op *op) +{ + switch (op->type) { + case LUO_IOCTL_INCOMING: + /* Retrieved is only set on incoming sessions */ + return session->retrieved; + case LUO_IOCTL_OUTGOING: + return !session->retrieved; + case LUO_IOCTL_ALL: + return true; + } + + /* Catch-all. */ + return false; +} + static long luo_session_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -345,6 +422,8 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE]; if (op->ioctl_num != cmd) return -ENOIOCTLCMD; + if (!luo_ioctl_type_valid(session, op)) + return -EINVAL; if (ucmd.user_size < op->min_size) return -EINVAL; @@ -354,6 +433,7 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, if (ret) return ret; + guard(rwsem_read)(&luo_session_serialize_rwsem); return op->execute(session, &ucmd); } @@ -382,21 +462,28 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep) int luo_session_create(const char *name, struct file **filep) { + size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH); struct luo_session *session; int err; + if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1) + return -EINVAL; + session = luo_session_alloc(name); if (IS_ERR(session)) return PTR_ERR(session); + down_read(&luo_session_serialize_rwsem); err = luo_session_insert(&luo_session_global.outgoing, session); if (err) goto err_free; - scoped_guard(mutex, &session->mutex) - err = luo_session_getfile(session, filep); + mutex_lock(&session->mutex); + err = luo_session_getfile(session, filep); + mutex_unlock(&session->mutex); if (err) goto err_remove; + up_read(&luo_session_serialize_rwsem); return 0; @@ -404,6 +491,7 @@ int luo_session_create(const char *name, struct file **filep) luo_session_remove(&luo_session_global.outgoing, session); err_free: luo_session_free(session); + up_read(&luo_session_serialize_rwsem); return err; } @@ -415,12 +503,12 @@ int luo_session_retrieve(const char *name, struct file **filep) struct luo_session *it; int err; - scoped_guard(rwsem_read, &sh->rwsem) { - list_for_each_entry(it, &sh->list, list) { - if (!strncmp(it->name, name, sizeof(it->name))) { - session = it; - break; - } + guard(rwsem_read)(&luo_session_serialize_rwsem); + guard(rwsem_read)(&sh->rwsem); + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, name, sizeof(it->name))) { + session = it; + break; } } @@ -438,74 +526,58 @@ int luo_session_retrieve(const char *name, struct file **filep) return err; } -int __init luo_session_setup_outgoing(void *fdt_out) +void __init luo_session_setup_outgoing(u64 *sessions_pa) { - struct luo_session_header_ser *header_ser; - u64 header_ser_pa; - int err; + luo_session_global.outgoing.sessions_pa = sessions_pa; + luo_session_global.outgoing.active = true; +} - header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); - if (IS_ERR(header_ser)) - return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); +int __init luo_session_setup_incoming(u64 sessions_pa) +{ + struct luo_session_header *sh = &luo_session_global.incoming; + int err; - err = fdt_begin_node(fdt_out, LUO_FDT_SESSION_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_SESSION_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_SESSION_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); + if (!sessions_pa) + return 0; + err = kho_block_set_restore(&sh->block_set, sessions_pa); if (err) - goto err_unpreserve; - - luo_session_global.outgoing.header_ser = header_ser; - luo_session_global.outgoing.ser = (void *)(header_ser + 1); - luo_session_global.outgoing.active = true; + return err; + sh->active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - return err; } -int __init luo_session_setup_incoming(void *fdt_in) +static int luo_session_deserialize_one(struct luo_session_header *sh, + struct luo_session_ser *ser) { - struct luo_session_header_ser *header_ser; - int err, header_size, offset; - u64 header_ser_pa; - const void *ptr; - - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_SESSION_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get session node: [%s]\n", - LUO_FDT_SESSION_NODE_NAME); - return -EINVAL; + struct luo_session *session; + int err; + + session = luo_session_alloc(ser->name); + if (IS_ERR(session)) { + pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", + (int)sizeof(ser->name), ser->name, session); + return PTR_ERR(session); } - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_SESSION_COMPATIBLE); + err = luo_session_insert(sh, session); if (err) { - pr_err("Session node incompatible [%s]\n", - LUO_FDT_SESSION_COMPATIBLE); - return -EINVAL; + pr_warn("Failed to insert session [%s] %pe\n", + session->name, ERR_PTR(err)); + luo_session_free(session); + return err; } - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_SESSION_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get session header '%s' [%d]\n", - LUO_FDT_SESSION_HEADER, header_size); - return -EINVAL; + scoped_guard(mutex, &session->mutex) { + err = luo_file_deserialize(&session->file_set, + &ser->file_set_ser); + } + if (err) { + pr_warn("Failed to deserialize files for session [%s] %pe\n", + session->name, ERR_PTR(err)); + return err; } - - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); - - luo_session_global.incoming.header_ser = header_ser; - luo_session_global.incoming.ser = (void *)(header_ser + 1); - luo_session_global.incoming.active = true; return 0; } @@ -514,6 +586,8 @@ int luo_session_deserialize(void) { struct luo_session_header *sh = &luo_session_global.incoming; static bool is_deserialized; + struct luo_session_ser *ser; + struct kho_block_set_it it; static int saved_err; int err; @@ -540,43 +614,19 @@ int luo_session_deserialize(void) * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - for (int i = 0; i < sh->header_ser->count; i++) { - struct luo_session *session; - - session = luo_session_alloc(sh->ser[i].name); - if (IS_ERR(session)) { - pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", - (int)sizeof(sh->ser[i].name), - sh->ser[i].name, session); - err = PTR_ERR(session); - goto save_err; - } - - err = luo_session_insert(sh, session); - if (err) { - pr_warn("Failed to insert session [%s] %pe\n", - session->name, ERR_PTR(err)); - luo_session_free(session); - goto save_err; - } - - scoped_guard(mutex, &session->mutex) { - err = luo_file_deserialize(&session->file_set, - &sh->ser[i].file_set_ser); - } - if (err) { - pr_warn("Failed to deserialize files for session [%s] %pe\n", - session->name, ERR_PTR(err)); + kho_block_set_it_init(&it, &sh->block_set); + while ((ser = kho_block_set_it_read_entry(&it))) { + err = luo_session_deserialize_one(sh, ser); + if (err) goto save_err; - } } - kho_restore_free(sh->header_ser); - sh->header_ser = NULL; - sh->ser = NULL; + kho_block_set_destroy(&sh->block_set); return 0; + save_err: + kho_block_set_destroy(&sh->block_set); saved_err = err; return err; } @@ -585,30 +635,48 @@ int luo_session_serialize(void) { struct luo_session_header *sh = &luo_session_global.outgoing; struct luo_session *session; - int i = 0; + struct kho_block_set_it it; int err; - guard(rwsem_write)(&sh->rwsem); + down_write(&luo_session_serialize_rwsem); + down_write(&sh->rwsem); + *sh->sessions_pa = 0; + + kho_block_set_it_init(&it, &sh->block_set); + list_for_each_entry(session, &sh->list, list) { - err = luo_session_freeze_one(session, &sh->ser[i]); - if (err) + struct luo_session_ser *ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!ser)) { + err = -ENOSPC; + goto err_undo; + } + + err = luo_session_freeze_one(session, ser); + if (err) { + kho_block_set_it_prev(&it); goto err_undo; + } - strscpy(sh->ser[i].name, session->name, - sizeof(sh->ser[i].name)); - i++; + strscpy(ser->name, session->name, sizeof(ser->name)); } - sh->header_ser->count = sh->count; + + if (sh->count > 0) + *sh->sessions_pa = kho_block_set_head_pa(&sh->block_set); + up_write(&sh->rwsem); return 0; err_undo: list_for_each_entry_continue_reverse(session, &sh->list, list) { - i--; - luo_session_unfreeze_one(session, &sh->ser[i]); - memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); + struct luo_session_ser *ser = kho_block_set_it_prev(&it); + + luo_session_unfreeze_one(session, ser); + memset(ser->name, 0, sizeof(ser->name)); } + up_write(&sh->rwsem); + up_write(&luo_session_serialize_rwsem); return err; } - diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..e9ca4b26efec 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -796,8 +796,10 @@ static void lockdep_print_held_locks(struct task_struct *p) * It's not reliable to print a task's held locks if it's not sleeping * and it's not the current task. */ +#ifndef CONFIG_DEBUG_AID_FOR_SYZBOT if (p != current && task_is_running(p)) return; +#endif for (i = 0; i < depth; i++) { printk(" #%d: ", i); print_lock(p->held_locks + i); @@ -5077,7 +5079,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read); static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, int hardirqs_off, struct lockdep_map *nest_lock, unsigned long ip, - int references, int pin_count, int sync) + int references, int pin_count, int sync, int seq) { struct task_struct *curr = current; struct lock_class *class = NULL; @@ -5183,6 +5185,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, hlock->holdtime_stamp = lockstat_clock(); #endif hlock->pin_count = pin_count; + hlock->seq_count = seq; if (check_wait_context(curr, hlock)) return 0; @@ -5388,7 +5391,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth, hlock->read, hlock->check, hlock->hardirqs_off, hlock->nest_lock, hlock->acquire_ip, - hlock->references, hlock->pin_count, 0)) { + hlock->references, hlock->pin_count, 0, hlock->seq_count)) { case 0: return 1; case 1: @@ -5669,14 +5672,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie struct held_lock *hlock = curr->held_locks + i; if (match_held_lock(hlock, lock)) { + int pin_count; + if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n")) return; - hlock->pin_count -= cookie.val; + pin_count = hlock->pin_count - cookie.val; - if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n")) - hlock->pin_count = 0; + if (WARN(pin_count < 0, "pin count corrupted\n")) + pin_count = 0; + hlock->pin_count = pin_count; return; } } @@ -5684,6 +5690,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie WARN(1, "unpinning an unheld lock\n"); } +static u32 __lock_sequence(struct lockdep_map *lock) +{ + struct task_struct *curr = current; + int i; + + if (unlikely(!debug_locks)) + return ~0; + + for (i = 0; i < curr->lockdep_depth; i++) { + struct held_lock *hlock = curr->held_locks + i; + + if (match_held_lock(hlock, lock)) + return hlock->seq_count; + } + + return ~0; +} + /* * Check whether we follow the irq-flags state precisely: */ @@ -5866,7 +5890,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass, lockdep_recursion_inc(); __lock_acquire(lock, subclass, trylock, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0, + ++current->lockdep_seq); lockdep_recursion_finish(); raw_local_irq_restore(flags); } @@ -5914,7 +5939,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read, lockdep_recursion_inc(); __lock_acquire(lock, subclass, 0, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1, + ++current->lockdep_seq); check_chain_key(current); lockdep_recursion_finish(); raw_local_irq_restore(flags); @@ -6000,6 +6026,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) } EXPORT_SYMBOL_GPL(lock_unpin_lock); +u32 lock_sequence(struct lockdep_map *lock) +{ + unsigned long flags; + u32 seq = ~0; + + if (unlikely(!lockdep_enabled())) + return seq; + + raw_local_irq_save(flags); + check_flags(flags); + + lockdep_recursion_inc(); + seq = __lock_sequence(lock); + lockdep_recursion_finish(); + raw_local_irq_restore(flags); + + return seq; +} +EXPORT_SYMBOL_GPL(lock_sequence); + #ifdef CONFIG_LOCK_STAT static void print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, diff --git a/kernel/locking/spinlock_rt.c b/kernel/locking/spinlock_rt.c index db1e11b45de6..1d5e1b3c60bf 100644 --- a/kernel/locking/spinlock_rt.c +++ b/kernel/locking/spinlock_rt.c @@ -79,10 +79,27 @@ void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU) { spin_release(&lock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) rt_mutex_slowunlock(&lock->lock); + + /* + * This must be last to prevent the following UAF: + * + * T1 T2 + * spin_lock(&p->lock); rcu_read_lock(); + * invalidate(p); p = rcu_dereference(ptr); + * rcu_assign_pointer(ptr, NULL); if (!p) return; + * spin_unlock(&p->lock); spin_lock(&p->lock); + * kfree_rcu(p); rcu_read_unlock(); + * .... + * spin_unlock(&p->lock) + * rcu_read_unlock(); // Ends grace period + * rcu_do_batch() + * kfree(p); + * UAF -> rt_mutex_cmpxchg_release(&p->lock.lock...) + */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_spin_unlock); @@ -262,17 +279,21 @@ void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); migrate_enable(); - rcu_read_unlock(); rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_read_unlock); void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU) { rwlock_release(&rwlock->dep_map, _RET_IP_); - rcu_read_unlock(); migrate_enable(); rwbase_write_unlock(&rwlock->rwbase); + + /* This must be last. See comment in rt_spin_unlock() */ + rcu_read_unlock(); } EXPORT_SYMBOL(rt_write_unlock); diff --git a/kernel/panic.c b/kernel/panic.c index 213725b612aa..90e21f15fd7e 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -412,7 +412,12 @@ static bool panic_try_force_cpu(const char *fmt, va_list args) * fall back to static message for early boot panics or allocation failure. */ if (panic_force_buf) { - vsnprintf(panic_force_buf, PANIC_MSG_BUFSZ, fmt, args); + va_list ap; + + /* Do not consume args, the caller reuses it if we fail */ + va_copy(ap, args); + vsnprintf(panic_force_buf, PANIC_MSG_BUFSZ, fmt, ap); + va_end(ap); msg = panic_force_buf; } else { msg = "Redirected panic (buffer unavailable)"; diff --git a/kernel/params.c b/kernel/params.c index a668863a4bb6..5c8a05921e28 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -136,8 +136,8 @@ static int parse_one(char *param, if (!val && !(params[i].ops->flags & KERNEL_PARAM_OPS_FL_NOARG)) return -EINVAL; - pr_debug("handling %s with %p\n", param, - params[i].ops->set); + pr_debug("handling %s with value '%s'\n", param, + val ? val : "no-arg"); kernel_param_lock(params[i].mod); if (param_check_unsafe(¶ms[i])) err = params[i].ops->set(val, ¶ms[i]); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..415583c35f8c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4900,6 +4900,9 @@ static void __init rcu_dump_rcu_node_tree(void) struct workqueue_struct *rcu_gp_wq; +static struct cpumask rcu_stall_cpumask; +static struct cpumask rcu_exp_stall_cpumask; + void __init rcu_init(void) { int cpu = smp_processor_id(); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..46b6907f1b09 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -578,6 +578,7 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne if (!(READ_ONCE(rnp->expmask) & mask)) continue; ndetected++; + cpumask_set_cpu(cpu, &rcu_exp_stall_cpumask); rdp = per_cpu_ptr(&rcu_data, cpu); pr_cont(" %d-%c%c%c%c", cpu, "O."[!!cpu_online(cpu)], @@ -665,6 +666,8 @@ static void synchronize_rcu_expedited_wait(void) if (rcu_stall_is_suppressed()) continue; + cpumask_clear(&rcu_exp_stall_cpumask); + nbcon_cpu_emergency_enter(); j = jiffies; @@ -675,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_exp_stall_cpumask); } } diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..ebf381936eb1 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -159,7 +159,7 @@ static int __init check_cpu_stall_init(void) early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ -static void panic_on_rcu_stall(void) +static void panic_on_rcu_stall(const struct cpumask *stalled_mask) { static int cpu_stall; @@ -167,7 +167,7 @@ static void panic_on_rcu_stall(void) * Attempt to kick out the BPF scheduler if it's installed and defer * the panic to give the system a chance to recover. */ - if (scx_rcu_cpu_stall()) + if (scx_rcu_cpu_stall(stalled_mask)) return; if (++cpu_stall < sysctl_max_rcu_stall_to_panic) @@ -644,6 +644,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) if (rcu_stall_is_suppressed()) return; + cpumask_clear(&rcu_stall_cpumask); + nbcon_cpu_emergency_enter(); /* @@ -659,6 +661,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) for_each_leaf_node_possible_cpu(rnp, cpu) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { print_cpu_stall_info(cpu); + cpumask_set_cpu(cpu, &rcu_stall_cpumask); ndetected++; } } @@ -700,7 +703,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + panic_on_rcu_stall(&rcu_stall_cpumask); rcu_force_quiescent_state(); /* Kick them all. */ } @@ -753,7 +756,9 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(); + cpumask_clear(&rcu_stall_cpumask); + cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask); /* * Attempt to revive the RCU machinery by forcing a context switch. diff --git a/kernel/resource.c b/kernel/resource.c index d02a53fb95d8..e60539a55541 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -1238,7 +1238,7 @@ reserve_region_with_split(struct resource *root, resource_size_t start, * * Returns alignment on success, 0 (invalid alignment) on failure. */ -resource_size_t resource_alignment(struct resource *res) +resource_size_t resource_alignment(const struct resource *res) { switch (res->flags & (IORESOURCE_SIZEALIGN | IORESOURCE_STARTALIGN)) { case IORESOURCE_SIZEALIGN: @@ -1859,7 +1859,7 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size) if (p->flags & IORESOURCE_BUSY) continue; - pr_warn("resource sanity check: requesting [mem %pa-%pa], which spans more than %s %pR\n", + pr_debug("resource sanity check: requesting [mem %pa-%pa], which spans more than %s %pR\n", &addr, &end, p->name, p); err = -1; break; diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c index 067979a7b69e..01dc7bf89af8 100644 --- a/kernel/sched/build_policy.c +++ b/kernel/sched/build_policy.c @@ -61,15 +61,17 @@ # include # include # include -# include "ext_types.h" -# include "ext_internal.h" -# include "ext_cid.h" -# include "ext_arena.h" -# include "ext_idle.h" -# include "ext.c" -# include "ext_cid.c" -# include "ext_arena.c" -# include "ext_idle.c" +# include "ext/types.h" +# include "ext/internal.h" +# include "ext/cid.h" +# include "ext/arena.h" +# include "ext/idle.h" +# include "ext/sub.h" +# include "ext/ext.c" +# include "ext/cid.c" +# include "ext/arena.c" +# include "ext/idle.c" +# include "ext/sub.c" #endif #include "syscalls.c" diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b791e9e9f67..2e7cde033a31 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5368,6 +5368,12 @@ static struct rq *finish_task_switch(struct task_struct *prev) */ kmap_local_sched_in(); + /* + * Any cached block-layer timestamp (plug->cur_ktime) is stale now, + * invalidate it. + */ + blk_plug_invalidate_ts(); + fire_sched_in_preempt_notifiers(current); /* * When switching through a kernel thread, the loop in @@ -5651,11 +5657,8 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat); */ static inline void prefetch_curr_exec_start(struct task_struct *p) { -#ifdef CONFIG_FAIR_GROUP_SCHED - struct sched_entity *curr = p->se.cfs_rq->curr; -#else struct sched_entity *curr = task_rq(p)->cfs.curr; -#endif + prefetch(curr); prefetch(&curr->exec_start); } @@ -7290,12 +7293,10 @@ static inline void sched_submit_work(struct task_struct *tsk) static void sched_update_worker(struct task_struct *tsk) { - if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { - if (tsk->flags & PF_BLOCK_TS) - blk_plug_invalidate_ts(tsk); + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { if (tsk->flags & PF_WQ_WORKER) wq_worker_running(tsk); - else if (tsk->flags & PF_IO_WORKER) + else io_wq_worker_running(tsk); } } @@ -10909,8 +10910,19 @@ static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm) } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) { unsigned int cid = rq->curr->mm_cid.cid; - /* Ensure it has the transition bit set */ - if (!cid_in_transit(cid)) { + /* + * Set the transition bit only on a genuine task-owned + * CID. A running active task can legitimately have + * MM_CID_UNSET here: in per-CPU mode CIDs are assigned + * lazily on schedule-in, so the fork()/execve() window + * leaves the task active with no owned CID. Setting the + * transition bit on MM_CID_UNSET would later feed + * clear_bit() an out-of-bounds bit number via + * mm_cid_schedout(), so exclude it. A CPU-owned + * (MM_CID_ONCPU) CID is handled by the cid_on_cpu() + * branch above and never reaches here. + */ + if (cid != MM_CID_UNSET && !cid_in_transit(cid)) { cid = cid_to_transit_cid(cid); rq->curr->mm_cid.cid = cid; pcp->cid = cid; diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index ae9fd211cec1..a4e689eefdfb 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -486,6 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, sg_cpu->util, max_cap); + sg_policy->need_freq_update = false; sg_policy->last_freq_update_time = time; } diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 679ac65be6b0..06bddaa738e5 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -421,7 +421,7 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ int nr_ticks) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ -#ifdef CONFIG_NO_HZ_COMMON +#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { u64 *cpustat = kc->cpustat; @@ -560,7 +560,7 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, { return kcpustat_cpu(cpu).cpustat[idx]; } -#endif /* CONFIG_NO_HZ_COMMON */ +#endif /* CONFIG_NO_HZ_COMMON && !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0f858b98c9aa..a3003b0f4522 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2733,15 +2733,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf) */ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags) { + struct task_struct *donor = rq->donor; /* * Can only get preempted by stop-class, and those should be * few and short lived, doesn't really make sense to push * anything away for that. */ - if (p->sched_class != &dl_sched_class) + if (p->sched_class != &dl_sched_class || + donor->sched_class != &dl_sched_class) return; - if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { + if (dl_entity_preempt(&p->dl, &donor->dl)) { resched_curr(rq); return; } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 40584b27ea0c..72236db67983 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -633,6 +633,82 @@ static void debugfs_fair_server_init(void) } } +#ifdef CONFIG_FAIR_GROUP_SCHED +static int cgroup_mode = 2; + +/* See __sched_cgroup_mode_update(). */ +static const char *cgroup_mode_str[] = { + "up", + "smp", + "concur", + "max", + "tasks", +}; + +static int sched_cgroup_mode(const char *str) +{ + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (!strcmp(str, cgroup_mode_str[i])) + return i; + } + return -EINVAL; +} + +static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char buf[16]; + int mode; + + if (cnt > 15) + cnt = 15; + + if (copy_from_user(buf, ubuf, cnt)) + return -EFAULT; + + buf[cnt] = 0; + mode = sched_cgroup_mode(strstrip(buf)); + if (mode < 0) + return mode; + + __sched_cgroup_mode_update(mode); + WRITE_ONCE(cgroup_mode, mode); + + *ppos += cnt; + return cnt; +} + +static int sched_cgroup_show(struct seq_file *m, void *v) +{ + int mode = READ_ONCE(cgroup_mode); + + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (mode == i) + seq_puts(m, "("); + seq_puts(m, cgroup_mode_str[i]); + if (mode == i) + seq_puts(m, ")"); + + seq_puts(m, " "); + } + seq_puts(m, "\n"); + return 0; +} + +static int sched_cgroup_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_cgroup_show, NULL); +} + +static const struct file_operations sched_cgroup_fops = { + .open = sched_cgroup_open, + .write = sched_cgroup_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; @@ -686,6 +762,10 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); +#ifdef CONFIG_FAIR_GROUP_SCHED + debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); +#endif + debugfs_fair_server_init(); #ifdef CONFIG_SCHED_CLASS_EXT debugfs_ext_server_init(); @@ -895,10 +975,11 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), + p->se.h_load.weight, SPLIT_NS(p->se.vruntime), - entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', + entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', SPLIT_NS(p->se.deadline), p->se.custom_slice ? 'S' : ' ', SPLIT_NS(p->se.slice), @@ -927,7 +1008,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID vruntime eligible " + SEQ_printf(m, " S task PID weight vruntime eligible " "deadline slice sum-exec switches " "prio wait-time sum-sleep sum-block" #ifdef CONFIG_NUMA_BALANCING @@ -1035,6 +1116,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->tg_load_avg_contrib); SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", atomic_long_read(&cfs_rq->tg->load_avg)); + SEQ_printf(m, " .%-30s: %lu\n", "h_load", + cfs_rq->h_load); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "throttled", @@ -1359,7 +1442,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(wait_count); PN_SCHEDSTAT(iowait_sum); P_SCHEDSTAT(iowait_count); - P_SCHEDSTAT(nr_migrations_cold); P_SCHEDSTAT(nr_failed_migrations_affine); P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); @@ -1371,8 +1453,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_wakeups_remote); P_SCHEDSTAT(nr_wakeups_affine); P_SCHEDSTAT(nr_wakeups_affine_attempts); - P_SCHEDSTAT(nr_wakeups_passive); - P_SCHEDSTAT(nr_wakeups_idle); avg_atom = p->se.sum_exec_runtime; if (nr_switches) diff --git a/kernel/sched/ext_arena.c b/kernel/sched/ext/arena.c similarity index 90% rename from kernel/sched/ext_arena.c rename to kernel/sched/ext/arena.c index 493c2424f842..b0e6a0be4913 100644 --- a/kernel/sched/ext_arena.c +++ b/kernel/sched/ext/arena.c @@ -15,6 +15,10 @@ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. * Copyright (c) 2026 Tejun Heo */ +#include + +#include "internal.h" +#include "arena.h" enum scx_arena_consts { SCX_ARENA_MIN_ORDER = 3, /* 8-byte minimum sub-allocation */ @@ -66,8 +70,6 @@ void scx_arena_pool_destroy(struct scx_sched *sch) */ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) { - u64 kern_vm_start; - u32 uaddr32; void *p; int ret; @@ -79,15 +81,8 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt) if (!p) return -ENOMEM; - uaddr32 = (u32)(unsigned long)p; - /* arena.o, which defines these, is built only on MMU && 64BIT */ -#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) - kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map); -#else - kern_vm_start = 0; -#endif - - ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32, + ret = gen_pool_add(sch->arena_pool, + (unsigned long)scx_arena_to_kaddr(sch, p), page_cnt * PAGE_SIZE, NUMA_NO_NODE); if (ret) { bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt); diff --git a/kernel/sched/ext_arena.h b/kernel/sched/ext/arena.h similarity index 95% rename from kernel/sched/ext_arena.h rename to kernel/sched/ext/arena.h index 4f3610160102..c378ae5fbc02 100644 --- a/kernel/sched/ext_arena.h +++ b/kernel/sched/ext/arena.h @@ -8,6 +8,8 @@ #ifndef _KERNEL_SCHED_EXT_ARENA_H #define _KERNEL_SCHED_EXT_ARENA_H +#include + struct scx_sched; s32 scx_arena_pool_init(struct scx_sched *sch); diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext/cid.c similarity index 99% rename from kernel/sched/ext_cid.c rename to kernel/sched/ext/cid.c index 66944a7ef79d..af83084ec740 100644 --- a/kernel/sched/ext_cid.c +++ b/kernel/sched/ext/cid.c @@ -7,6 +7,9 @@ */ #include +#include "internal.h" +#include "cid.h" + /* * cid tables. * @@ -71,7 +74,7 @@ static s32 scx_cid_arrays_alloc(void) * scx_cid_init - build the cid mapping * @sch: the scx_sched being initialized; used as the scx_error() target * - * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by + * See "Topological CPU IDs" in cid.h for the model. Walk online cpus by * intersection at each level (parent_scratch & this_level_mask), which keeps * containment correct by construction and naturally splits a physical LLC * straddling two NUMA nodes into two LLC units. The caller must hold diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext/cid.h similarity index 92% rename from kernel/sched/ext_cid.h rename to kernel/sched/ext/cid.h index 5745e5785e89..9c4f4b907f12 100644 --- a/kernel/sched/ext_cid.h +++ b/kernel/sched/ext/cid.h @@ -33,6 +33,8 @@ #ifndef _KERNEL_SCHED_EXT_CID_H #define _KERNEL_SCHED_EXT_CID_H +#include "internal.h" + struct scx_sched; /* @@ -43,7 +45,7 @@ struct scx_sched; * possible-but-not-online cpus and carries all-(-1) topo info (see * scx_cid_topo); callers detect it via the -1 sentinels. * - * See the comment above the table definitions in ext_cid.c for the + * See the comment above the table definitions in cid.c for the * memory-ordering and visibility contract. */ extern s16 *scx_cid_to_cpu_tbl; @@ -268,4 +270,25 @@ static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \ __w &= __w - 1) +/* + * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form + * schedulers it resolves to the matching cid; for cpu-form it passes @cpu + * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op + * (currently only ops.select_cpu); it validates the BPF-supplied cid and + * triggers scx_error() on @sch if invalid. + */ +static inline s32 scx_cpu_arg(s32 cpu) +{ + if (scx_is_cid_type()) + return __scx_cpu_to_cid(cpu); + return cpu; +} + +static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) +{ + if (cpu_or_cid < 0 || !scx_is_cid_type()) + return cpu_or_cid; + return scx_cid_to_cpu(sch, cpu_or_cid); +} + #endif /* _KERNEL_SCHED_EXT_CID_H */ diff --git a/kernel/sched/ext.c b/kernel/sched/ext/ext.c similarity index 87% rename from kernel/sched/ext.c rename to kernel/sched/ext/ext.c index 0db6fa2daea3..1a0ec985da77 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext/ext.c @@ -6,8 +6,22 @@ * Copyright (c) 2022 Tejun Heo * Copyright (c) 2022 David Vernet */ - -static DEFINE_RAW_SPINLOCK(scx_sched_lock); +#include +#include +#include +#include +#include +#include +#include + +#include "../pelt.h" +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "idle.h" +#include "sub.h" + +DEFINE_RAW_SPINLOCK(scx_sched_lock); /* * NOTE: sched_ext is in the process of growing multiple scheduler support and @@ -26,14 +40,14 @@ struct scx_sched __rcu *scx_root; static LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED -static const struct rhashtable_params scx_sched_hash_params = { +const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; -static struct rhashtable scx_sched_hash; +struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ @@ -55,9 +69,9 @@ static DEFINE_RAW_SPINLOCK(scx_tasks_lock); static LIST_HEAD(scx_tasks); /* ops enable/disable */ -static DEFINE_MUTEX(scx_enable_mutex); +DEFINE_MUTEX(scx_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_enabled); -DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); +DEFINE_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); static DEFINE_RAW_SPINLOCK(scx_bypass_lock); static bool scx_init_task_enabled; @@ -88,7 +102,7 @@ static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1); * tasks for the sub-sched being enabled. Use a global variable instead of a * per-task field as all enables are serialized. */ -static struct scx_sched *scx_enabling_sub_sched; +struct scx_sched *scx_enabling_sub_sched; #else #define scx_enabling_sub_sched (struct scx_sched *)NULL #endif /* CONFIG_EXT_SUB_SCHED */ @@ -229,7 +243,6 @@ MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microsecond static void run_deferred(struct rq *rq); static bool task_dead_and_done(struct task_struct *p); -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind); __printf(5, 6) bool __scx_exit(struct scx_sched *sch, @@ -246,8 +259,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, return ret; } -#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) - static long jiffies_delta_msecs(unsigned long at, unsigned long now) { if (time_after(at, now)) @@ -261,73 +272,6 @@ static bool u32_before(u32 a, u32 b) return (s32)(a - b) < 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_parent - Find the parent sched - * @sch: sched to find the parent of - * - * Returns the parent scheduler or %NULL if @sch is root. - */ -static struct scx_sched *scx_parent(struct scx_sched *sch) -{ - if (sch->level) - return sch->ancestors[sch->level - 1]; - else - return NULL; -} - -/** - * scx_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: sched whose descendants to walk - * - * To be used by scx_for_each_descendant_pre(). Find the next descendant to - * visit for pre-order traversal of @root's descendants. @root is included in - * the iteration and the first node to be visited. - */ -static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, - struct scx_sched *root) -{ - struct scx_sched *next; - - lockdep_assert(lockdep_is_held(&scx_enable_mutex) || - lockdep_is_held(&scx_sched_lock)); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) - return list_next_entry(pos, sibling); - pos = scx_parent(pos); - } - - return NULL; -} - -static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) -{ - return rhashtable_lookup(&scx_sched_hash, &cgroup_id, - scx_sched_hash_params); -} - -static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) -{ - rcu_assign_pointer(p->scx.sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } -static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } -static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /** * scx_is_descendant - Test whether sched is a descendant * @sch: sched to test @@ -342,19 +286,6 @@ static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor) return sch->ancestors[ancestor->level] == ancestor; } -/** - * scx_for_each_descendant_pre - pre-order walk of a sched's descendants - * @pos: iteration cursor - * @root: sched to walk the descendants of - * - * Walk @root's descendants. @root is included in the iteration and the first - * node to be visited. Must be called with either scx_enable_mutex or - * scx_sched_lock held. - */ -#define scx_for_each_descendant_pre(pos, root) \ - for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ - (pos) = scx_next_descendant_pre((pos), (root))) - static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu) { return &sch->pnode[cpu_to_node(cpu)]->global_dsq; @@ -373,11 +304,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu) -{ - return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; -} - static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { #ifdef CONFIG_EXT_SUB_SCHED @@ -396,27 +322,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c sch = scx_parent(sch); #endif /* CONFIG_EXT_SUB_SCHED */ - return bypass_dsq(sch, cpu); -} - -/** - * bypass_dsp_enabled - Check if bypass dispatch path is enabled - * @sch: scheduler to check - * - * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled - * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors - * are bypassing. In the former case, the ancestor is not itself bypassing but - * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, - * the ancestor's bypass dispatch path must be active even though its own - * bypass_depth remains zero. - * - * This function checks bypass_dsp_enable_depth which is managed separately from - * bypass_depth to enable this decoupling. See enable_bypass_dsp() and - * disable_bypass_dsp(). - */ -static bool bypass_dsp_enabled(struct scx_sched *sch) -{ - return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); + return scx_bypass_dsq(sch, cpu); } /** @@ -483,123 +389,12 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); -static inline void update_locked_rq(struct rq *rq) -{ - /* - * Check whether @rq is actually locked. This can help expose bugs - * or incorrect assumptions about the context in which a kfunc or - * callback is executed. - */ - if (rq) - lockdep_assert_rq_held(rq); - __this_cpu_write(scx_locked_rq_state, rq); -} - /* - * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not - * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. - */ -#define SCX_CALL_OP(sch, op, locked_rq, args...) \ -do { \ - struct rq *__prev_locked_rq; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ -} while (0) - -/* - * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so + * Flipped on enable per sch->is_cid_type. Declared in internal.h so * subsystem inlines can read it. */ DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); -/* - * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form - * schedulers it resolves to the matching cid; for cpu-form it passes @cpu - * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op - * (currently only ops.select_cpu); it validates the BPF-supplied cid and - * triggers scx_error() on @sch if invalid. - */ -static s32 scx_cpu_arg(s32 cpu) -{ - if (scx_is_cid_type()) - return __scx_cpu_to_cid(cpu); - return cpu; -} - -static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) -{ - if (cpu_or_cid < 0 || !scx_is_cid_type()) - return cpu_or_cid; - return scx_cid_to_cpu(sch, cpu_or_cid); -} - -#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ -({ \ - struct rq *__prev_locked_rq; \ - __typeof__((sch)->ops.op(args)) __ret; \ - \ - if (locked_rq) { \ - __prev_locked_rq = scx_locked_rq(); \ - update_locked_rq(locked_rq); \ - } \ - __ret = (sch)->ops.op(args); \ - if (locked_rq) \ - update_locked_rq(__prev_locked_rq); \ - __ret; \ -}) - -/* - * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments - * and records them in current->scx.kf_tasks[] for the duration of the call. A - * kfunc invoked from inside such an op can then use - * scx_kf_arg_task_ok() to verify that its task argument is one of - * those subject tasks. - * - * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - - * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's - * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. - * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. - * - * kf_tasks[] can not stack, so task-based SCX ops must not nest. The - * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants - * while a previous one is still in progress. - */ -#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ -do { \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ -} while (0) - -#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ -({ \ - __typeof__((sch)->ops.op(task, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - __ret; \ -}) - -#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ -({ \ - __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ - WARN_ON_ONCE(current->scx.kf_tasks[0]); \ - current->scx.kf_tasks[0] = task0; \ - current->scx.kf_tasks[1] = task1; \ - __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ - current->scx.kf_tasks[0] = NULL; \ - current->scx.kf_tasks[1] = NULL; \ - __ret; \ -}) - /** * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task * @sch: scx_sched being invoked @@ -608,7 +403,7 @@ do { \ * @cpumask: new cpumask * * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller + * scratch in cid.c and dispatch through the ops_cid union view. Caller * must hold @rq's rq lock so this_cpu_ptr is stable across the call. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, @@ -638,19 +433,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, current->scx.kf_tasks[0] = NULL; } -/* see SCX_CALL_OP_TASK() */ -static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, - struct task_struct *p) -{ - if (unlikely((p != current->scx.kf_tasks[0] && - p != current->scx.kf_tasks[1]))) { - scx_error(sch, "called on a task not being operated on"); - return false; - } - - return true; -} - enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, @@ -804,12 +586,12 @@ struct bpf_iter_scx_dsq { } __attribute__((aligned(8))); -static u32 scx_get_task_state(const struct task_struct *p) +u32 scx_get_task_state(const struct task_struct *p) { return p->scx.flags & SCX_TASK_STATE_MASK; } -static void scx_set_task_state(struct task_struct *p, u32 state) +void scx_set_task_state(struct task_struct *p, u32 state) { u32 prev_state = scx_get_task_state(p); bool warn = false; @@ -849,23 +631,6 @@ static void scx_set_task_state(struct task_struct *p, u32 state) p->scx.flags |= state; } -/* - * SCX task iterator. - */ -struct scx_task_iter { - struct sched_ext_entity cursor; - struct task_struct *locked_task; - struct rq *rq; - struct rq_flags rf; - u32 cnt; - bool list_locked; -#ifdef CONFIG_EXT_SUB_SCHED - struct cgroup *cgrp; - struct cgroup_subsys_state *css_pos; - struct css_task_iter css_iter; -#endif -}; - /** * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration * @iter: iterator to init @@ -894,7 +659,7 @@ struct scx_task_iter { * All tasks which existed when the iteration started are guaranteed to be * visited as long as they are not dead. */ -static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp) { memset(iter, 0, sizeof(*iter)); @@ -933,7 +698,7 @@ static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) * This function can be safely called anytime during an iteration. The next * iterator operation will automatically restore the necessary locking. */ -static void scx_task_iter_unlock(struct scx_task_iter *iter) +void scx_task_iter_unlock(struct scx_task_iter *iter) { __scx_task_iter_rq_unlock(iter); if (iter->list_locked) { @@ -976,7 +741,7 @@ static void scx_task_iter_relock(struct scx_task_iter *iter, * which is released on return. If the iterator holds a task's rq lock, that rq * lock is also released. See scx_task_iter_start() for details. */ -static void scx_task_iter_stop(struct scx_task_iter *iter) +void scx_task_iter_stop(struct scx_task_iter *iter) { #ifdef CONFIG_EXT_SUB_SCHED if (iter->cgrp) { @@ -1051,7 +816,7 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) * whether they would like to filter out dead tasks. See scx_task_iter_start() * for details. */ -static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) { struct task_struct *p; @@ -1106,42 +871,6 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) return NULL; } -/** - * scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This can be used when preemption is not disabled. - */ -#define scx_add_event(sch, name, cnt) do { \ - this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, (cnt)); \ -} while(0) - -/** - * __scx_add_event - Increase an event counter for 'name' by 'cnt' - * @sch: scx_sched to account events for - * @name: an event name defined in struct scx_event_stats - * @cnt: the number of the event occurred - * - * This should be used only when preemption is disabled. - */ -#define __scx_add_event(sch, name, cnt) do { \ - __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ - trace_sched_ext_event(#name, cnt); \ -} while(0) - -/** - * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e' - * @dst_e: destination event stats - * @src_e: source event stats - * @kind: a kind of event to be aggregated - */ -#define scx_agg_event(dst_e, src_e, kind) do { \ - (dst_e)->kind += READ_ONCE((src_e)->kind); \ -} while(0) - /** * scx_dump_event - Dump an event 'kind' in 'events' to 's' * @s: output seq_buf @@ -1216,28 +945,6 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where) } } -/** - * ops_sanitize_err - Sanitize a -errno value - * @sch: scx_sched to error out on error - * @ops_name: operation to blame on failure - * @err: -errno value to sanitize - * - * Verify @err is a valid -errno. If not, trigger scx_error() and return - * -%EPROTO. This is necessary because returning a rogue -errno up the chain can - * cause misbehaviors. For an example, a large negative return from - * ops.init_task() triggers an oops when passed up the call chain because the - * value fails IS_ERR() test after being encoded with ERR_PTR() and then is - * handled as a pointer. - */ -static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) -{ - if (err < 0 && err >= -MAX_ERRNO) - return err; - - scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); - return -EPROTO; -} - static void deferred_bal_cb_workfn(struct rq *rq) { run_deferred(rq); @@ -1324,8 +1031,8 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } -static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, - u64 reenq_flags, struct rq *locked_rq) +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq) { struct rq *rq; @@ -1389,16 +1096,6 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq schedule_deferred(rq); } -static void schedule_reenq_local(struct rq *rq, u64 reenq_flags) -{ - struct scx_sched *root = rcu_dereference_sched(scx_root); - - if (WARN_ON_ONCE(!root)) - return; - - schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); -} - /** * touch_core_sched - Update timestamp used for core-sched task ordering * @rq: rq to read clock from, must be locked @@ -1485,8 +1182,8 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * to the CPU or dequeued. In both cases, the only way @p can go back to * the BPF sched is through enqueueing. If being inserted into a local * DSQ with IMMED, persist the state until the next enqueueing event in - * do_enqueue_task() so that we can maintain IMMED protection through - * e.g. SAVE/RESTORE cycles and slice extensions. + * scx_do_enqueue_task() so that we can maintain IMMED protection + * through e.g. SAVE/RESTORE cycles and slice extensions. */ if (enq_flags & SCX_ENQ_IMMED) { if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { @@ -1509,7 +1206,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en * done yet, @p can't go on the CPU immediately. Re-enqueue. */ if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags))) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } } @@ -1626,9 +1323,9 @@ static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq } } -static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, struct task_struct *p, - u64 enq_flags) +static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, struct task_struct *p, + u64 enq_flags) { bool is_local = dsq->id == SCX_DSQ_LOCAL; @@ -1776,7 +1473,7 @@ static void task_unlink_from_dsq(struct task_struct *p, } } -static void dispatch_dequeue(struct rq *rq, struct task_struct *p) +static void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) { struct scx_dispatch_q *dsq = p->scx.dsq; bool is_local = dsq == &rq->scx.local_dsq; @@ -1830,8 +1527,8 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p) } /* - * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq - * and dsq are locked. + * Abbreviated version of scx_dispatch_dequeue() that can be used when both + * @p's rq and dsq are locked. */ static void dispatch_dequeue_locked(struct task_struct *p, struct scx_dispatch_q *dsq) @@ -1912,10 +1609,10 @@ static void mark_direct_dispatch(struct scx_sched *sch, * - direct_dispatch(): cleared on the synchronous enqueue path, deferred * dispatch keeps the state until consumed * - process_ddsp_deferred_locals(): cleared after consuming deferred state, - * - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch + * - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch * verdict is ignored (local/global/bypass) - * - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred - * cancellation and holding_cpu races + * - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering + * deferred cancellation and holding_cpu races * - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by * the scx_bypass() loop, so that stale state is not reused by a subsequent * scheduler instance @@ -1976,10 +1673,10 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, ddsp_enq_flags = p->scx.ddsp_enq_flags; clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); } -static bool scx_rq_online(struct rq *rq) +bool scx_rq_online(struct rq *rq) { /* * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates @@ -1991,8 +1688,8 @@ static bool scx_rq_online(struct rq *rq) return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } -static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, - int sticky_cpu) +static void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, + int sticky_cpu) { struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; @@ -2079,7 +1776,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, direct_dispatch(sch, p, enq_flags); return; local_norefill: - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags); return; local: dsq = &rq->scx.local_dsq; @@ -2100,7 +1797,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, touch_core_sched(rq, p); refill_task_slice_dfl(sch, p); clear_direct_dispatch(p); - dispatch_enqueue(sch, rq, dsq, p, enq_flags); + scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags); } static bool task_runnable(const struct task_struct *p) @@ -2169,7 +1866,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ if (rq->scx.nr_running == 1) dl_server_start(&rq->ext_server); - do_enqueue_task(rq, p, enq_flags, sticky_cpu); + scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu); if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; @@ -2305,7 +2002,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ rq->scx.nr_running--; sub_nr_running(rq, 1); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); clear_direct_dispatch(p); return true; } @@ -2353,7 +2050,7 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl * - A higher-priority wakes up while SCX dispatch is in progress. */ if (rq->scx.nr_immed) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); } static void move_local_task_to_local_dsq(struct scx_sched *sch, @@ -2435,6 +2132,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, * no to the BPF scheduler initiated migrations while offline. * * The caller must ensure that @p and @rq are on different CPUs. + * If enforce == true, caller must hold @p's rq lock. */ static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, @@ -2442,6 +2140,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, { s32 cpu = cpu_of(rq); + /* + * To prevent races with @p still running on its old CPU while switching + * out, make sure we're holding @p's rq lock so as not to risk + * erroneously killing the BPF scheduler. + */ + if (enforce) + lockdep_assert_rq_held(task_rq(p)); + WARN_ON_ONCE(task_cpu(p) == cpu); /* @@ -2509,7 +2215,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * values afterwards, as this operation can't be preempted or recurse, the * holding_cpu can never become this CPU again before we're done. Thus, we can * tell whether we lost to dequeue by testing whether the holding_cpu still - * points to this CPU. See dispatch_dequeue() for the counterpart. + * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is * still valid. %false if lost to dequeue. @@ -2614,14 +2320,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); - dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); + scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags); } return dst_rq; } -static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, - struct scx_dispatch_q *dsq, u64 enq_flags) +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags) { struct task_struct *p; retry: @@ -2667,11 +2373,11 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq, return false; } -static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq) +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq) { int node = cpu_to_node(cpu_of(rq)); - return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); + return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0); } /** @@ -2704,15 +2410,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, * If dispatching to @rq that @p is already on, no lock dancing needed. */ if (rq == src_rq && rq == dst_rq) { - dispatch_enqueue(sch, rq, dst_dsq, p, - enq_flags | SCX_ENQ_CLEAR_OPSS); - return; - } - - if (src_rq != dst_rq && - unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p, - enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK); + scx_dispatch_enqueue(sch, rq, dst_dsq, p, + enq_flags | SCX_ENQ_CLEAR_OPSS); return; } @@ -2742,6 +2441,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, /* task_rq couldn't have changed if we're still the holding cpu */ if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + bool fallback = false; /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by @@ -2749,8 +2449,13 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, */ if (src_rq == dst_rq) { p->scx.holding_cpu = -1; - dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, - enq_flags); + scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, + enq_flags); + } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { + p->scx.holding_cpu = -1; + fallback = true; + scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), + p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); } else { move_remote_task_to_local_dsq(p, enq_flags, src_rq, dst_rq); @@ -2759,7 +2464,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* if the destination CPU is idle, wake it up */ - if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) resched_curr(dst_rq); } @@ -2838,10 +2543,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, goto retry; case SCX_OPSS_QUEUEING: /* - * do_enqueue_task() is in the process of transferring the task - * to the BPF scheduler while holding @p's rq lock. As we aren't - * holding any kernel or BPF resource that the enqueue path may - * depend upon, it's safe to wait. + * scx_do_enqueue_task() is in the process of transferring the + * task to the BPF scheduler while holding @p's rq lock. As we + * aren't holding any kernel or BPF resource that the enqueue + * path may depend upon, it's safe to wait. */ wait_ops_state(p, opss); goto retry; @@ -2854,10 +2559,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, if (dsq->id == SCX_DSQ_LOCAL) dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags); else - dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); } -static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq) { struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; u32 u; @@ -2886,115 +2591,6 @@ static inline void maybe_queue_balance_callback(struct rq *rq) rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; } -/* - * One user of this function is scx_bpf_dispatch() which can be called - * recursively as sub-sched dispatches nest. Always inline to reduce stack usage - * from the call frame. - */ -static __always_inline bool -scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, - struct task_struct *prev, bool nested) -{ - struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - int nr_loops = SCX_DSP_MAX_LOOPS; - s32 cpu = cpu_of(rq); - bool prev_on_sch = (prev->sched_class == &ext_sched_class) && - scx_task_on_sched(sch, prev); - - if (consume_global_dsq(sch, rq)) - return true; - - if (bypass_dsp_enabled(sch)) { - /* if @sch is bypassing, only the bypass DSQs are active */ - if (scx_bypassing(sch, cpu)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - -#ifdef CONFIG_EXT_SUB_SCHED - /* - * If @sch isn't bypassing but its children are, @sch is - * responsible for making forward progress for both its own - * tasks that aren't bypassing and the bypassing descendants' - * tasks. The following implements a simple built-in behavior - - * let each CPU try to run the bypass DSQ every Nth time. - * - * Later, if necessary, we can add an ops flag to suppress the - * auto-consumption and a kfunc to consume the bypass DSQ and, - * so that the BPF scheduler can fully control scheduling of - * bypassed tasks. - */ - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) { - __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); - return true; - } -#endif /* CONFIG_EXT_SUB_SCHED */ - } - - if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) - return false; - - dspc->rq = rq; - - /* - * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, - * the local DSQ might still end up empty after a successful - * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() - * produced some tasks, retry. The BPF scheduler may depend on this - * looping behavior to simplify its implementation. - */ - do { - dspc->nr_tasks = 0; - - if (nested) { - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - } else { - /* stash @prev so that nested invocations can access it */ - rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), - prev_on_sch ? prev : NULL); - rq->scx.sub_dispatch_prev = NULL; - } - - flush_dispatch_buf(sch, rq); - - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { - rq->scx.flags |= SCX_RQ_BAL_KEEP; - return true; - } - if (rq->scx.local_dsq.nr) - return true; - if (consume_global_dsq(sch, rq)) - return true; - - /* - * ops.dispatch() can trap us in this loop by repeatedly - * dispatching ineligible tasks. Break out once in a while to - * allow the watchdog to run. As IRQ can't be enabled in - * balance(), we want to complete this scheduling cycle and then - * start a new one. IOW, we want to call resched_curr() on the - * next, most likely idle, task, not the current one. Use - * __scx_bpf_kick_cpu() for deferred kicking. - */ - if (unlikely(!--nr_loops)) { - scx_kick_cpu(sch, cpu, 0); - break; - } - } while (dspc->nr_tasks); - - /* - * Prevent the CPU from going idle while bypassed descendants have tasks - * queued. Without this fallback, bypassed tasks could stall if the host - * scheduler's ops.dispatch() doesn't yield any tasks. - */ - if (bypass_dsp_enabled(sch)) - return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0); - - return false; -} - static int balance_one(struct rq *rq, struct task_struct *prev) { struct scx_sched *sch = scx_root; @@ -3069,7 +2665,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev) * between the IMMED queueing and the subsequent scheduling event. */ if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) - schedule_reenq_local(rq, 0); + scx_schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_BALANCE; return true; @@ -3085,7 +2681,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) * dispatched. Call ops_dequeue() to notify the BPF scheduler. */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); } p->se.exec_start = rq_clock_task(rq); @@ -3197,10 +2793,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { - dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD); } goto switch_class; } @@ -3210,12 +2806,17 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. + * + * Core scheduling can force this CPU idle while @p stays + * runnable. @p's cookie then won't match the core's, so skip + * the warning in that case. */ if (next && sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); - do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); + WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && + !(sch->ops.flags & SCX_OPS_ENQ_LAST)); + scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { - do_enqueue_task(rq, p, 0, -1); + scx_do_enqueue_task(rq, p, 0, -1); } } @@ -3673,7 +3274,7 @@ static struct cgroup *tg_cgrp(struct task_group *tg) #endif /* CONFIG_EXT_GROUP_SCHED */ -static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork) { int ret; @@ -3687,7 +3288,7 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args); if (unlikely(ret)) { - ret = ops_sanitize_err(sch, "init_task", ret); + ret = scx_ops_sanitize_err(sch, "init_task", ret); return ret; } } @@ -3756,7 +3357,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); } -static void scx_enable_task(struct scx_sched *sch, struct task_struct *p) +void scx_enable_task(struct scx_sched *sch, struct task_struct *p) { __scx_enable_task(sch, p); scx_set_task_state(p, SCX_TASK_ENABLED); @@ -3775,6 +3376,13 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY); + /* + * Reset the SCX-managed fields when @p leaves the BPF scheduler's + * control, after ops.disable() has observed their final values. + */ + p->scx.dsq_vtime = 0; + p->scx.slice = 0; + /* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear @@ -3783,8 +3391,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY); } -static void __scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = false, @@ -3818,7 +3425,7 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch, * ran. The task state has not been transitioned, so this mirrors the * SCX_TASK_INIT branch in __scx_disable_and_exit_task(). */ -static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p) { struct scx_exit_task_args args = { .cancelled = true }; @@ -3829,8 +3436,7 @@ static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct * SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args); } -static void scx_disable_and_exit_task(struct scx_sched *sch, - struct task_struct *p) +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) { __scx_disable_and_exit_task(sch, p); @@ -4225,7 +3831,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(p, &reenq_flags, &reason)) continue; - dispatch_dequeue(rq, p); + scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; @@ -4237,7 +3843,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { list_del_init(&p->scx.dsq_list.node); - do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; @@ -4352,7 +3958,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; p->scx.flags |= reason; - do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); + scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; @@ -4472,7 +4078,7 @@ int scx_tg_online(struct task_group *tg) ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); if (ret) - ret = ops_sanitize_err(sch, "cgroup_init", ret); + ret = scx_ops_sanitize_err(sch, "cgroup_init", ret); } if (ret == 0) tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED; @@ -4540,7 +4146,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset) p->scx.cgrp_moving_from = NULL; } - return ops_sanitize_err(sch, "cgroup_prep_move", ret); + return scx_ops_sanitize_err(sch, "cgroup_prep_move", ret); } void scx_cgroup_move_task(struct task_struct *p) @@ -4643,7 +4249,7 @@ static struct cgroup *root_cgroup(void) return &cgrp_dfl_root.cgrp; } -static void scx_cgroup_lock(void) +void scx_cgroup_lock(void) { #ifdef CONFIG_EXT_GROUP_SCHED percpu_down_write(&scx_cgroup_ops_rwsem); @@ -4651,7 +4257,7 @@ static void scx_cgroup_lock(void) cgroup_lock(); } -static void scx_cgroup_unlock(void) +void scx_cgroup_unlock(void) { cgroup_unlock(); #ifdef CONFIG_EXT_GROUP_SCHED @@ -4664,26 +4270,6 @@ static inline void scx_cgroup_lock(void) {} static inline void scx_cgroup_unlock(void) {} #endif /* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */ -#ifdef CONFIG_EXT_SUB_SCHED -static struct cgroup *sch_cgroup(struct scx_sched *sch) -{ - return sch->cgrp; -} - -/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ -static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) -{ - struct cgroup *pos; - struct cgroup_subsys_state *css; - - cgroup_for_each_live_descendant_pre(pos, css, cgrp) - rcu_assign_pointer(pos->scx_sched, sch); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } -static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} -#endif /* CONFIG_EXT_SUB_SCHED */ - /* * Omitted operations: * @@ -4818,7 +4404,7 @@ static void destroy_dsq(struct scx_sched *sch, u64 dsq_id) goto out_unlock_dsq; /* - * Mark dead by invalidating ->id to prevent dispatch_enqueue() from + * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from * queueing more tasks. As this function can be called from anywhere, * freeing is bounced through an irq work to avoid nesting RCU * operations inside scheduler locks. @@ -4966,8 +4552,10 @@ static const struct attribute_group scx_global_attr_group = { static void free_pnode(struct scx_sched_pnode *pnode); static void free_exit_info(struct scx_exit_info *ei); +static const char *scx_exit_reason(enum scx_exit_kind kind); +static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind); -static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) { size_t size = struct_size_t(struct scx_cmask, bits, SCX_CMASK_NR_WORDS(num_possible_cpus())); @@ -5022,6 +4610,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #ifdef CONFIG_EXT_SUB_SCHED kfree(sch->cgrp_path); @@ -5029,6 +4618,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) cgroup_put(sch_cgroup(sch)); if (sch->sub_kset) kobject_put(&sch->sub_kset->kobj); + if (scx_parent(sch)) + kobject_put(&scx_parent(sch)->kobj); #endif /* CONFIG_EXT_SUB_SCHED */ for_each_possible_cpu(cpu) { @@ -5041,7 +4632,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) */ WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - exit_dsq(bypass_dsq(sch, cpu)); + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); @@ -5099,19 +4690,9 @@ static ssize_t scx_attr_events_show(struct kobject *kobj, int at = 0; scx_read_events(sch, &events); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING); - at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT); - at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH); - at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE); - at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED); - at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) (at += scx_attr_event_show(buf, at, &events, name)) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT return at; } SCX_ATTR(events); @@ -5205,6 +4786,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) /** * handle_lockup - sched_ext common lockup handler + * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current. * @fmt: format string * * Called on system stall or lockup condition and initiates abort of sched_ext @@ -5214,7 +4796,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p) * resolve the lockup. %false if sched_ext is not enabled or abort was already * initiated by someone else. */ -static __printf(1, 2) bool handle_lockup(const char *fmt, ...) +static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...) { struct scx_sched *sch; va_list args; @@ -5230,7 +4812,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) case SCX_ENABLING: case SCX_ENABLED: va_start(args, fmt); - ret = scx_verror(sch, fmt, args); + ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args); va_end(args); return ret; default: @@ -5250,9 +4832,46 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...) * resolve the reported RCU stall. %false if sched_ext is not enabled or someone * else already initiated abort. */ -bool scx_rcu_cpu_stall(void) +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { - return handle_lockup("RCU CPU stall detected!"); + struct scx_sched *sch; + struct scx_exit_info *ei; + int exit_cpu; + + guard(rcu)(); + + sch = rcu_dereference(scx_root); + if (unlikely(!sch)) + return false; + + switch (scx_enable_state()) { + case SCX_ENABLING: + case SCX_ENABLED: + break; + default: + return false; + } + + exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask); + ei = sch->exit_info; + + guard(preempt)(); + + if (!scx_claim_exit(sch, SCX_EXIT_ERROR)) + return false; + +#ifdef CONFIG_STACKTRACE + ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); +#endif + scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)", + cpumask_pr_args(stalled_mask)); + ei->kind = SCX_EXIT_ERROR; + ei->reason = scx_exit_reason(SCX_EXIT_ERROR); + ei->exit_cpu = exit_cpu; + cpumask_copy(sch->stall_cpus, stalled_mask); + + irq_work_queue(&sch->disable_irq_work); + return true; } /** @@ -5267,11 +4886,13 @@ bool scx_rcu_cpu_stall(void) */ void scx_softlockup(u32 dur_s) { - if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s)) + int cpu = smp_processor_id(); + + if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s)) return; printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n", - smp_processor_id(), dur_s); + cpu, dur_s); } /* @@ -5286,7 +4907,7 @@ static void scx_hardlockup_irq_workfn(struct irq_work *work) { int cpu = atomic_xchg(&scx_hardlockup_cpu, -1); - if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu)) + if (cpu >= 0 && handle_lockup(cpu, "hard lockup - CPU %d", cpu)) printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n", cpu); } @@ -5322,7 +4943,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, u32 nr_donor_target, u32 nr_donee_target) { struct rq *donor_rq = cpu_rq(donor); - struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor); + struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor); struct task_struct *p, *n; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0); s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target; @@ -5370,7 +4991,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (donee >= nr_cpu_ids) continue; - donee_dsq = bypass_dsq(sch, donee); + donee_dsq = scx_bypass_dsq(sch, donee); /* * $p's rq is not locked but $p's DSQ lock protects its @@ -5391,7 +5012,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, * between bypass DSQs. */ dispatch_dequeue_locked(p, donor_dsq); - dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); + scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED); /* * $donee might have been idle and need to be woken up. No need @@ -5434,7 +5055,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* count the target tasks and CPUs */ for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); nr_tasks += nr; nr_cpus++; @@ -5456,7 +5077,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) cpumask_set_cpu(cpu, donee_mask); } @@ -5467,7 +5088,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) break; if (cpumask_test_cpu(cpu, donee_mask)) continue; - if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) continue; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, @@ -5478,7 +5099,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) resched_cpu(cpu); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr); + u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); after_min = min(nr, after_min); after_max = max(nr, after_max); @@ -5504,7 +5125,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer) int node; u32 intv_us; - if (!bypass_dsp_enabled(sch)) + if (!scx_bypass_dsp_enabled(sch)) return; for_each_node_with_cpus(node) @@ -5570,9 +5191,9 @@ static void enable_bypass_dsp(struct scx_sched *sch) * dispatch enabled while a descendant is bypassing, which is all that's * required. * - * bypass_dsp_enabled() test is used to determine whether to enter the - * bypass dispatch handling path from both bypassing and hosting scheds. - * Bump enable depth on both @sch and bypass dispatch host. + * scx_bypass_dsp_enabled() test is used to determine whether to enter + * the bypass dispatch handling path from both bypassing and hosting + * scheds. Bump enable depth on both @sch and bypass dispatch host. */ ret = atomic_inc_return(&sch->bypass_dsp_enable_depth); WARN_ON_ONCE(ret <= 0); @@ -5592,7 +5213,7 @@ static void enable_bypass_dsp(struct scx_sched *sch) } /* may be called without holding scx_bypass_lock */ -static void disable_bypass_dsp(struct scx_sched *sch) +void scx_disable_bypass_dsp(struct scx_sched *sch) { s32 ret; @@ -5640,7 +5261,7 @@ static void disable_bypass_dsp(struct scx_sched *sch) * * - scx_prio_less() reverts to the default core_sched_at order. */ -static void scx_bypass(struct scx_sched *sch, bool bypass) +void scx_bypass(struct scx_sched *sch, bool bypass) { struct scx_sched *pos; unsigned long flags; @@ -5737,7 +5358,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass) /* disarming must come after moving all tasks out of the bypass DSQs */ if (!bypass) - disable_bypass_dsp(sch); + scx_disable_bypass_dsp(sch); unlock: raw_spin_unlock_irqrestore(&scx_bypass_lock, flags); } @@ -5800,7 +5421,7 @@ static void free_kick_syncs(void) int cpu; for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *to_free; to_free = rcu_replace_pointer(*ksyncs, NULL, true); @@ -5829,7 +5450,7 @@ static void refresh_watchdog(void) cancel_delayed_work_sync(&scx_watchdog_work); } -static s32 scx_link_sched(struct scx_sched *sch) +s32 scx_link_sched(struct scx_sched *sch) { const char *err_msg = ""; s32 ret = 0; @@ -5878,7 +5499,7 @@ static s32 scx_link_sched(struct scx_sched *sch) return 0; } -static void scx_unlink_sched(struct scx_sched *sch) +void scx_unlink_sched(struct scx_sched *sch) { scoped_guard(raw_spinlock_irq, &scx_sched_lock) { #ifdef CONFIG_EXT_SUB_SCHED @@ -5899,13 +5520,13 @@ static void scx_unlink_sched(struct scx_sched *sch) * @sch. Once @sch becomes empty during disable, there's no point in dumping it. * This prevents calling dump ops on a dead sch. */ -static void scx_disable_dump(struct scx_sched *sch) +void scx_disable_dump(struct scx_sched *sch) { guard(raw_spinlock_irqsave)(&scx_dump_lock); sch->dump_disabled = true; } -static void scx_log_sched_disable(struct scx_sched *sch) +void scx_log_sched_disable(struct scx_sched *sch) { struct scx_exit_info *ei = sch->exit_info; const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler"; @@ -5925,237 +5546,41 @@ static void scx_log_sched_disable(struct scx_sched *sch) } } -#ifdef CONFIG_EXT_SUB_SCHED -static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); - -static void drain_descendants(struct scx_sched *sch) -{ - /* - * Child scheds that finished the critical part of disabling will take - * themselves off @sch->children. Wait for it to drain. As propagation - * is recursive, empty @sch->children means that all proper descendant - * scheds reached unlinking stage. - */ - wait_event(scx_unlink_waitq, list_empty(&sch->children)); -} - -static void scx_fail_parent(struct scx_sched *sch, - struct task_struct *failed, s32 fail_code) -{ - struct scx_sched *parent = scx_parent(sch); - struct scx_task_iter sti; - struct task_struct *p; - - scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", - fail_code, failed->comm, failed->pid); - - /* - * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into - * it. This may cause downstream failures on the BPF side but $parent is - * dying anyway. - */ - scx_bypass(parent, true); - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (scx_task_on_sched(parent, p)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - scx_disable_and_exit_task(sch, p); - scx_set_task_sched(p, parent); - } - } - scx_task_iter_stop(&sti); -} - -static void scx_sub_disable(struct scx_sched *sch) +static void scx_root_disable(struct scx_sched *sch) { - struct scx_sched *parent = scx_parent(sch); struct scx_task_iter sti; struct task_struct *p; - int ret; + bool was_switched_all; + int cpu; - /* - * Guarantee forward progress and wait for descendants to be disabled. - * To limit disruptions, $parent is not bypassed. Tasks are fully - * prepped and then inserted back into $parent. - */ + /* guarantee forward progress and wait for descendants to be disabled */ scx_bypass(sch, true); drain_descendants(sch); + switch (scx_set_enable_state(SCX_DISABLING)) { + case SCX_DISABLING: + WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); + break; + case SCX_DISABLED: + pr_warn("sched_ext: ops error detected without ops (%s)\n", + sch->exit_info->msg); + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); + goto done; + default: + break; + } + /* * Here, every runnable task is guaranteed to make forward progress and * we can safely use blocking synchronization constructs. Actually * disable ops. */ mutex_lock(&scx_enable_mutex); - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - set_cgroup_sched(sch_cgroup(sch), parent); - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; + was_switched_all = scx_switched_all(); - /* filter out duplicate visits */ - if (scx_task_on_sched(parent, p)) - continue; - - /* - * By the time control reaches here, all descendant schedulers - * should already have been disabled. - */ - WARN_ON_ONCE(!scx_task_on_sched(sch, p)); - - /* - * @p is pinned by the iter: css_task_iter_next() takes a - * reference and holds it until the next iter_next() call, so - * @p->usage is guaranteed > 0. - */ - get_task_struct(p); - - scx_task_iter_unlock(&sti); - - /* - * $p is READY or ENABLED on @sch. Initialize for $parent, - * disable and exit from @sch, and then switch over to $parent. - * - * If a task fails to initialize for $parent, the only available - * action is disabling $parent too. While this allows disabling - * of a child sched to cause the parent scheduler to fail, the - * failure can only originate from ops.init_task() of the - * parent. A child can't directly affect the parent through its - * own failures. - */ - ret = __scx_init_task(parent, p, false); - if (ret) { - scx_fail_parent(sch, p, ret); - put_task_struct(p); - break; - } - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on @sch (the - * sched @p was on at that point), not on $parent. - * $parent's just-completed init is owed an exit_task() - * and we issue it here. - */ - scx_sub_init_cancel_task(parent, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p is initialized for $parent and still attached to - * @sch. Disable and exit for @sch, switch over to - * $parent, override the state to READY to account for - * $p having already been initialized, and then enable. - */ - scx_disable_and_exit_task(sch, p); - scx_set_task_state(p, SCX_TASK_INIT_BEGIN); - scx_set_task_state(p, SCX_TASK_INIT); - scx_set_task_sched(p, parent); - scx_set_task_state(p, SCX_TASK_READY); - scx_enable_task(parent, p); - } - - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - scx_disable_dump(sch); - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - /* - * All tasks are moved off of @sch but there may still be on-going - * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use - * the expedited version as ancestors may be waiting in bypass mode. - * Also, tell the parent that there is no need to keep running bypass - * DSQs for us. - */ - synchronize_rcu_expedited(); - disable_bypass_dsp(sch); - - scx_unlink_sched(sch); - - mutex_unlock(&scx_enable_mutex); - - /* - * @sch is now unlinked from the parent's children list. Notify and call - * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called - * after unlinking and releasing all locks. See scx_claim_exit(). - */ - wake_up_all(&scx_unlink_waitq); - - if (parent->ops.sub_detach && sch->sub_attached) { - struct scx_sub_detach_args sub_detach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - SCX_CALL_OP(parent, sub_detach, NULL, - &sub_detach_args); - } - - scx_log_sched_disable(sch); - - if (sch->ops.exit) - SCX_CALL_OP(sch, exit, NULL, sch->exit_info); - if (sch->sub_kset) - kobject_del(&sch->sub_kset->kobj); - kobject_del(&sch->kobj); -} -#else /* CONFIG_EXT_SUB_SCHED */ -static inline void drain_descendants(struct scx_sched *sch) { } -static inline void scx_sub_disable(struct scx_sched *sch) { } -#endif /* CONFIG_EXT_SUB_SCHED */ - -static void scx_root_disable(struct scx_sched *sch) -{ - struct scx_task_iter sti; - struct task_struct *p; - bool was_switched_all; - int cpu; - - /* guarantee forward progress and wait for descendants to be disabled */ - scx_bypass(sch, true); - drain_descendants(sch); - - switch (scx_set_enable_state(SCX_DISABLING)) { - case SCX_DISABLING: - WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); - break; - case SCX_DISABLED: - pr_warn("sched_ext: ops error detected without ops (%s)\n", - sch->exit_info->msg); - WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); - goto done; - default: - break; - } - - /* - * Here, every runnable task is guaranteed to make forward progress and - * we can safely use blocking synchronization constructs. Actually - * disable ops. - */ - mutex_lock(&scx_enable_mutex); - - was_switched_all = scx_switched_all(); - - static_branch_disable(&__scx_switched_all); - WRITE_ONCE(scx_switching_all, false); + static_branch_disable(&__scx_switched_all); + WRITE_ONCE(scx_switching_all, false); /* * Shut down cgroup support before tasks so that the cgroup attach path @@ -6368,7 +5793,7 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) * as a noop. Syncing the irq_work first is required to guarantee the * kthread work has been queued before waiting for it. */ -static void scx_flush_disable_work(struct scx_sched *sch) +void scx_flush_disable_work(struct scx_sched *sch) { int kind; @@ -6697,14 +6122,23 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "----------"); /* - * Dump the exit CPU first so it isn't lost to dump truncation, then - * walk the rest in order, skipping the one already dumped. + * Dump stalled CPUs first so they aren't lost to dump truncation, then + * walk the rest in order. Fall back to exit_cpu if no stall mask set. */ - if (ei->exit_cpu >= 0) - scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); - for_each_possible_cpu(cpu) { - if (cpu != ei->exit_cpu) + if (!cpumask_empty(sch->stall_cpus)) { + for_each_cpu(cpu, sch->stall_cpus) scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (!cpumask_test_cpu(cpu, sch->stall_cpus)) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } + } else { + if (ei->exit_cpu >= 0) + scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks); + for_each_possible_cpu(cpu) { + if (cpu != ei->exit_cpu) + scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks); + } } dump_newline(&s); @@ -6712,19 +6146,9 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei, dump_line(&s, "--------------"); scx_read_events(sch, &events); - scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK); - scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING); - scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_dump_event(s, &events, SCX_EV_REENQ_IMMED); - scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT); - scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL); - scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION); - scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH); - scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE); - scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED); - scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH); +#define SCX_EVENT(name) scx_dump_event(s, &events, name) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) memcpy(ei->dump + dump_len - sizeof(trunc_marker), @@ -6781,7 +6205,7 @@ static int alloc_kick_syncs(void) * can exceed percpu allocator limits on large machines. */ for_each_possible_cpu(cpu) { - struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); + struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu); struct scx_kick_syncs *new_ksyncs; WARN_ON_ONCE(rcu_access_pointer(*ksyncs)); @@ -6823,31 +6247,13 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node) return pnode; } -/* - * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid - * starvation. During the READY -> ENABLED task switching loop, the calling - * thread's sched_class gets switched from fair to ext. As fair has higher - * priority than ext, the calling thread can be indefinitely starved under - * fair-class saturation, leading to a system hang. - */ -struct scx_enable_cmd { - struct kthread_work work; - union { - struct sched_ext_ops *ops; - struct sched_ext_ops_cid *ops_cid; - }; - bool is_cid_type; - struct bpf_map *arena_map; /* arena ref to transfer to sch */ - int ret; -}; - /* * Allocate and initialize a new scx_sched. @cgrp's reference is always * consumed whether the function succeeds or fails. */ -static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, - struct cgroup *cgrp, - struct scx_sched *parent) +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent) { struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; @@ -6894,7 +6300,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, } for_each_possible_cpu(cpu) { - ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); + ret = init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch); if (ret) { bypass_fail_cpu = cpu; goto err_free_pcpu; @@ -6941,6 +6347,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, ret = -ENOMEM; goto err_free_lb_cpumask; } + if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free_lb_resched_cpumask; + } /* * Copy ops through the right union view. For cid-form the source is * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/ @@ -6976,12 +6386,19 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, INIT_LIST_HEAD(&sch->children); INIT_LIST_HEAD(&sch->sibling); - if (parent) + if (parent) { + /* + * Pin @parent for @sch's lifetime. The kobject hierarchy pins + * it only via @parent->sub_kset, which is dropped during + * disable. Released in scx_sched_free_rcu_work(). + */ + kobject_get(&parent->kobj); ret = kobject_init_and_add(&sch->kobj, &scx_ktype, &parent->sub_kset->kobj, "sub-%llu", cgroup_id(cgrp)); - else + } else { ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root"); + } if (ret < 0) { RCU_INIT_POINTER(ops->priv, NULL); @@ -7024,8 +6441,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, #ifdef CONFIG_EXT_SUB_SCHED err_free_lb_resched: RCU_INIT_POINTER(ops->priv, NULL); - free_cpumask_var(sch->bypass_lb_resched_cpumask); + free_cpumask_var(sch->stall_cpus); #endif +err_free_lb_resched_cpumask: + free_cpumask_var(sch->bypass_lb_resched_cpumask); err_free_lb_cpumask: free_cpumask_var(sch->bypass_lb_donee_cpumask); err_stop_helper: @@ -7034,7 +6453,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, for_each_possible_cpu(cpu) { if (cpu == bypass_fail_cpu) break; - exit_dsq(bypass_dsq(sch, cpu)); + exit_dsq(scx_bypass_dsq(sch, cpu)); } free_percpu(sch->pcpu); err_free_pnode: @@ -7078,7 +6497,7 @@ static int check_hotplug_seq(struct scx_sched *sch, return 0; } -static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) { /* * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the @@ -7116,7 +6535,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) * run past the BPF allocation. Skip for cid-form. */ if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release)) - pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); + pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n"); /* * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched @@ -7241,7 +6660,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (sch->ops.init) { ret = SCX_CALL_OP_RET(sch, init, NULL); if (ret) { - ret = ops_sanitize_err(sch, "init", ret); + ret = scx_ops_sanitize_err(sch, "init", ret); cpus_read_unlock(); scx_error(sch, "ops.init() failed (%d)", ret); goto err_disable; @@ -7274,7 +6693,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) cpus_read_unlock(); - ret = validate_ops(sch, ops); + ret = scx_validate_ops(sch, ops); if (ret) goto err_disable; @@ -7516,347 +6935,6 @@ static void scx_root_enable_workfn(struct kthread_work *work) cmd->ret = 0; } -#ifdef CONFIG_EXT_SUB_SCHED -/* verify that a scheduler can be attached to @cgrp and return the parent */ -static struct scx_sched *find_parent_sched(struct cgroup *cgrp) -{ - struct scx_sched *parent = cgrp->scx_sched; - struct scx_sched *pos; - - lockdep_assert_held(&scx_sched_lock); - - /* can't attach twice to the same cgroup */ - if (parent->cgrp == cgrp) - return ERR_PTR(-EBUSY); - - /* does $parent allow sub-scheds? */ - if (!parent->ops.sub_attach) - return ERR_PTR(-EOPNOTSUPP); - - /* can't insert between $parent and its exiting children */ - list_for_each_entry(pos, &parent->children, sibling) - if (cgroup_is_descendant(pos->cgrp, cgrp)) - return ERR_PTR(-EBUSY); - - return parent; -} - -static bool assert_task_ready_or_enabled(struct task_struct *p) -{ - u32 state = scx_get_task_state(p); - - switch (state) { - case SCX_TASK_READY: - case SCX_TASK_ENABLED: - return true; - default: - WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", - state, p->comm, p->pid); - return false; - } -} - -static void scx_sub_enable_workfn(struct kthread_work *work) -{ - struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); - struct sched_ext_ops *ops = cmd->ops; - struct cgroup *cgrp; - struct scx_sched *parent, *sch; - struct scx_task_iter sti; - struct task_struct *p; - s32 i, ret; - - mutex_lock(&scx_enable_mutex); - - if (!scx_enabled()) { - ret = -ENODEV; - goto out_unlock; - } - - /* See scx_root_enable_workfn() for the @ops->priv check. */ - if (rcu_access_pointer(ops->priv)) { - ret = -EBUSY; - goto out_unlock; - } - - cgrp = cgroup_get_from_id(ops->sub_cgroup_id); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - raw_spin_lock_irq(&scx_sched_lock); - parent = find_parent_sched(cgrp); - if (IS_ERR(parent)) { - raw_spin_unlock_irq(&scx_sched_lock); - ret = PTR_ERR(parent); - goto out_put_cgrp; - } - kobject_get(&parent->kobj); - raw_spin_unlock_irq(&scx_sched_lock); - - /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ - sch = scx_alloc_and_add_sched(cmd, cgrp, parent); - kobject_put(&parent->kobj); - if (IS_ERR(sch)) { - ret = PTR_ERR(sch); - goto out_unlock; - } - - ret = scx_link_sched(sch); - if (ret) - goto err_disable; - - if (sch->level >= SCX_SUB_MAX_DEPTH) { - scx_error(sch, "max nesting depth %d violated", - SCX_SUB_MAX_DEPTH); - goto err_disable; - } - - if (sch->ops.init) { - ret = SCX_CALL_OP_RET(sch, init, NULL); - if (ret) { - ret = ops_sanitize_err(sch, "init", ret); - scx_error(sch, "ops.init() failed (%d)", ret); - goto err_disable; - } - sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; - } - - ret = scx_arena_pool_init(sch); - if (ret) - goto err_disable; - - ret = scx_set_cmask_scratch_alloc(sch); - if (ret) - goto err_disable; - - if (validate_ops(sch, ops)) - goto err_disable; - - struct scx_sub_attach_args sub_attach_args = { - .ops = &sch->ops, - .cgroup_path = sch->cgrp_path, - }; - - ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, - &sub_attach_args); - if (ret) { - ret = ops_sanitize_err(sch, "sub_attach", ret); - scx_error(sch, "parent rejected (%d)", ret); - goto err_disable; - } - sch->sub_attached = true; - - scx_bypass(sch, true); - - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) - if (((void (**)(void))ops)[i]) - set_bit(i, sch->has_op); - - percpu_down_write(&scx_fork_rwsem); - scx_cgroup_lock(); - - /* - * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see - * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. - */ - set_cgroup_sched(sch_cgroup(sch), sch); - if (!(cgrp->self.flags & CSS_ONLINE)) { - scx_error(sch, "cgroup is not online"); - goto err_unlock_and_disable; - } - - /* - * Initialize tasks for the new child $sch without exiting them for - * $parent so that the tasks can always be reverted back to $parent - * sched on child init failure. - */ - WARN_ON_ONCE(scx_enabling_sub_sched); - scx_enabling_sub_sched = sch; - - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - struct rq *rq; - struct rq_flags rf; - - /* - * Task iteration may visit the same task twice when racing - * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which - * finished __scx_init_task() and skip if set. - * - * A task may exit and get freed between __scx_init_task() - * completion and scx_enable_task(). In such cases, - * scx_disable_and_exit_task() must exit the task for both the - * parent and child scheds. - */ - if (p->scx.flags & SCX_TASK_SUB_INIT) - continue; - - /* @p is pinned by the iter; see scx_sub_disable() */ - get_task_struct(p); - - if (!assert_task_ready_or_enabled(p)) { - ret = -EINVAL; - goto abort; - } - - scx_task_iter_unlock(&sti); - - /* - * As $p is still on $parent, it can't be transitioned to INIT. - * Let's worry about task state later. Use __scx_init_task(). - */ - ret = __scx_init_task(sch, p, false); - if (ret) - goto abort; - - rq = task_rq_lock(p, &rf); - - if (scx_get_task_state(p) == SCX_TASK_DEAD) { - /* - * sched_ext_dead() raced us between __scx_init_task() - * and this rq lock and ran exit_task() on $parent (the - * sched @p was on at that point), not on @sch. @sch's - * just-completed init is owed an exit_task() and we - * issue it here. - */ - scx_sub_init_cancel_task(sch, p); - task_rq_unlock(rq, p, &rf); - put_task_struct(p); - continue; - } - - p->scx.flags |= SCX_TASK_SUB_INIT; - task_rq_unlock(rq, p, &rf); - - put_task_struct(p); - } - scx_task_iter_stop(&sti); - - /* - * All tasks are prepped. Disable/exit tasks for $parent and enable for - * the new @sch. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - /* - * Use clearing of %SCX_TASK_SUB_INIT to detect and skip - * duplicate iterations. - */ - if (!(p->scx.flags & SCX_TASK_SUB_INIT)) - continue; - - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* - * $p must be either READY or ENABLED. If ENABLED, - * __scx_disabled_and_exit_task() first disables and - * makes it READY. However, after exiting $p, it will - * leave $p as READY. - */ - assert_task_ready_or_enabled(p); - __scx_disable_and_exit_task(parent, p); - - /* - * $p is now only initialized for @sch and READY, which - * is what we want. Assign it to @sch and enable. - */ - scx_set_task_sched(p, sch); - scx_enable_task(sch, p); - - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - - scx_enabling_sub_sched = NULL; - - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); - - scx_bypass(sch, false); - - pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); - kobject_uevent(&sch->kobj, KOBJ_ADD); - ret = 0; - goto out_unlock; - -out_put_cgrp: - cgroup_put(cgrp); -out_unlock: - mutex_unlock(&scx_enable_mutex); - cmd->ret = ret; - return; - -abort: - put_task_struct(p); - scx_task_iter_stop(&sti); - - /* - * Undo __scx_init_task() for tasks we marked. scx_enable_task() never - * ran for @sch on them, so calling scx_disable_task() here would invoke - * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched - * must stay set until SUB_INIT is cleared from every marked task - - * scx_disable_and_exit_task() reads it when a task exits concurrently. - */ - scx_task_iter_start(&sti, sch->cgrp); - while ((p = scx_task_iter_next_locked(&sti))) { - if (p->scx.flags & SCX_TASK_SUB_INIT) { - scx_sub_init_cancel_task(sch, p); - p->scx.flags &= ~SCX_TASK_SUB_INIT; - } - } - scx_task_iter_stop(&sti); - scx_enabling_sub_sched = NULL; -err_unlock_and_disable: - /* we'll soon enter disable path, keep bypass on */ - scx_cgroup_unlock(); - percpu_up_write(&scx_fork_rwsem); -err_disable: - mutex_unlock(&scx_enable_mutex); - scx_flush_disable_work(sch); - cmd->ret = 0; -} - -static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cgroup *cgrp = data; - struct cgroup *parent = cgroup_parent(cgrp); - - if (!cgroup_on_dfl(cgrp)) - return NOTIFY_OK; - - switch (action) { - case CGROUP_LIFETIME_ONLINE: - /* inherit ->scx_sched from $parent */ - if (parent) - rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); - break; - case CGROUP_LIFETIME_OFFLINE: - /* if there is a sched attached, shoot it down */ - if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) - scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, - SCX_ECODE_RSN_CGROUP_OFFLINE, - "cgroup %llu going offline", cgroup_id(cgrp)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block scx_cgroup_lifetime_nb = { - .notifier_call = scx_cgroup_lifetime_notify, -}; - -static s32 __init scx_cgroup_lifetime_notifier_init(void) -{ - return blocking_notifier_chain_register(&cgroup_lifetime_notifier, - &scx_cgroup_lifetime_nb); -} -core_initcall(scx_cgroup_lifetime_notifier_init); -#endif /* CONFIG_EXT_SUB_SCHED */ - static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link) { static struct kthread_worker *helper; @@ -7934,7 +7012,7 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, off + size <= offsetofend(struct task_struct, scx.slice)) || (off >= offsetof(struct task_struct, scx.dsq_vtime) && off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) { - pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()"); + pr_warn_ratelimited("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()\n"); return SCALAR_VALUE; } @@ -8003,20 +7081,6 @@ static int bpf_scx_init_member(const struct btf_type *t, return 0; } -#ifdef CONFIG_EXT_SUB_SCHED -static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) -{ - struct scx_sched *sch; - - guard(rcu)(); - sch = scx_prog_sched(prog->aux); - if (unlikely(!sch)) - return; - - scx_error(sch, "dispatch recursion detected"); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - static int bpf_scx_check_member(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog) @@ -8901,7 +7965,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, /* * If the BPF scheduler keeps calling this function repeatedly, it can - * cause similar live-lock conditions as consume_dispatch_q(). + * cause similar live-lock conditions as scx_consume_dispatch_q(). */ if (unlikely(READ_ONCE(sch->aborting))) return false; @@ -9062,7 +8126,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; - flush_dispatch_buf(sch, dspc->rq); + scx_flush_dispatch_buf(sch, dspc->rq); dsq = find_user_dsq(sch, dsq_id); if (unlikely(!dsq)) { @@ -9070,7 +8134,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags, return false; } - if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { + if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) { /* * A successfully consumed task can be dequeued before it starts * running while the CPU is trying to migrate other dispatched @@ -9187,45 +8251,6 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); } -#ifdef CONFIG_EXT_SUB_SCHED -/** - * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler - * @cgroup_id: cgroup ID of the child scheduler to dispatch - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - * - * Allows a parent scheduler to trigger dispatching on one of its direct - * child schedulers. The child scheduler runs its dispatch operation to - * move tasks from dispatch queues to the local runqueue. - * - * Returns: true on success, false if cgroup_id is invalid, not a direct - * child, or caller lacks dispatch permission. - */ -__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) -{ - struct rq *this_rq = this_rq(); - struct scx_sched *parent, *child; - - guard(rcu)(); - parent = scx_prog_sched(aux); - if (unlikely(!parent)) - return false; - - child = scx_find_sub_sched(cgroup_id); - - if (unlikely(!child)) - return false; - - if (unlikely(scx_parent(child) != parent)) { - scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", - cgroup_id); - return false; - } - - return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, - true); -} -#endif /* CONFIG_EXT_SUB_SCHED */ - __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_dispatch) @@ -9413,7 +8438,7 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } -static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct rq *this_rq; unsigned long irq_flags; @@ -10222,34 +9247,6 @@ __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p) return tbl[task_cpu(p)]; } -/** - * scx_bpf_cpu_rq - Fetch the rq of a CPU - * @cpu: CPU of the rq - * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs - */ -__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux) -{ - struct scx_sched *sch; - - guard(rcu)(); - - sch = scx_prog_sched(aux); - if (unlikely(!sch)) - return NULL; - - if (!scx_cpu_valid(sch, cpu, NULL)) - return NULL; - - if (!sch->warned_deprecated_rq) { - printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; " - "use scx_bpf_locked_rq() when holding rq lock " - "or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__); - sch->warned_deprecated_rq = true; - } - - return cpu_rq(cpu); -} - /** * scx_bpf_locked_rq - Return the rq currently locked by SCX * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs @@ -10416,26 +9413,15 @@ __bpf_kfunc u64 scx_bpf_now(void) static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events) { - struct scx_event_stats *e_cpu; int cpu; /* Aggregate per-CPU event counters into @events. */ memset(events, 0, sizeof(*events)); for_each_possible_cpu(cpu) { - e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; - scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); - scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING); - scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED); - scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT); - scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH); - scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE); - scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED); - scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH); + struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats; +#define SCX_EVENT(name) (events->name += READ_ONCE(e_cpu->name)) + SCX_EVENTS_LIST(SCX_EVENT); +#undef SCX_EVENT } } @@ -10540,7 +9526,6 @@ BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) @@ -10575,7 +9560,6 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { BTF_KFUNCS_START(scx_kfunc_ids_cpu_only) BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) -BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED) BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS) @@ -10795,7 +9779,7 @@ static int __init scx_init(void) /* @priv tail must align since both share the same data block */ CID_OFFSET_MATCH(priv, priv); /* - * cid-form must end exactly at @priv - validate_ops() skips + * cid-form must end exactly at @priv - scx_validate_ops() skips * cpu_acquire/cpu_release for cid-form because reading those fields * past the BPF allocation would be UB. */ diff --git a/kernel/sched/ext.h b/kernel/sched/ext/ext.h similarity index 100% rename from kernel/sched/ext.h rename to kernel/sched/ext/ext.h diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext/idle.c similarity index 99% rename from kernel/sched/ext_idle.c rename to kernel/sched/ext/idle.c index 2077373d8da3..8e8c6201b7df 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext/idle.c @@ -9,6 +9,9 @@ * Copyright (c) 2022 David Vernet * Copyright (c) 2024 Andrea Righi */ +#include "internal.h" +#include "cid.h" +#include "idle.h" /* Enable/disable built-in idle CPU selection policy */ static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); diff --git a/kernel/sched/ext_idle.h b/kernel/sched/ext/idle.h similarity index 93% rename from kernel/sched/ext_idle.h rename to kernel/sched/ext/idle.h index 8d169d3bbdf9..87a0e58f1eb7 100644 --- a/kernel/sched/ext_idle.h +++ b/kernel/sched/ext/idle.h @@ -10,7 +10,11 @@ #ifndef _KERNEL_SCHED_EXT_IDLE_H #define _KERNEL_SCHED_EXT_IDLE_H +#include + +struct cpumask; struct sched_ext_ops; +struct task_struct; extern struct btf_id_set8 scx_kfunc_ids_idle; extern struct btf_id_set8 scx_kfunc_ids_select_cpu; diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext/internal.h similarity index 82% rename from kernel/sched/ext_internal.h rename to kernel/sched/ext/internal.h index b04701190b23..f9fe7c6ebc4b 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext/internal.h @@ -5,6 +5,40 @@ * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. * Copyright (c) 2025 Tejun Heo */ +#ifndef _KERNEL_SCHED_EXT_INTERNAL_H +#define _KERNEL_SCHED_EXT_INTERNAL_H + +#include "../sched.h" +#include "types.h" + +#include + +/** + * scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This can be used when preemption is not disabled. + */ +#define scx_add_event(sch, name, cnt) do { \ + this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, (cnt)); \ +} while(0) + +/** + * __scx_add_event - Increase an event counter for 'name' by 'cnt' + * @sch: scx_sched to account events for + * @name: an event name defined in struct scx_event_stats + * @cnt: the number of the event occurred + * + * This should be used only when preemption is disabled. + */ +#define __scx_add_event(sch, name, cnt) do { \ + __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \ + trace_sched_ext_event(#name, cnt); \ +} while(0) + #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) #define SCX_MOFF_IDX(moff) ((moff) / sizeof(void (*)(void))) @@ -1045,6 +1079,21 @@ struct scx_event_stats { s64 SCX_EV_SUB_BYPASS_DISPATCH; }; +#define SCX_EVENTS_LIST(SCX_EVENT) \ + SCX_EVENT(SCX_EV_SELECT_CPU_FALLBACK); \ + SCX_EVENT(SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE); \ + SCX_EVENT(SCX_EV_DISPATCH_KEEP_LAST); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING); \ + SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED); \ + SCX_EVENT(SCX_EV_REENQ_IMMED); \ + SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT); \ + SCX_EVENT(SCX_EV_REFILL_SLICE_DFL); \ + SCX_EVENT(SCX_EV_BYPASS_DURATION); \ + SCX_EVENT(SCX_EV_BYPASS_DISPATCH); \ + SCX_EVENT(SCX_EV_BYPASS_ACTIVATE); \ + SCX_EVENT(SCX_EV_INSERT_NOT_OWNED); \ + SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH) + struct scx_sched; enum scx_sched_pcpu_flags { @@ -1151,7 +1200,7 @@ struct scx_sched { u64 bypass_timestamp; s32 bypass_depth; - /* bypass dispatch path enable state, see bypass_dsp_enabled() */ + /* bypass dispatch path enable state, see scx_bypass_dsp_enabled() */ unsigned long bypass_dsp_claim; atomic_t bypass_dsp_enable_depth; @@ -1165,7 +1214,6 @@ struct scx_sched { * but it doesn't really matter. */ bool warned_zero_slice:1; - bool warned_deprecated_rq:1; bool warned_unassoc_progs:1; struct list_head all; @@ -1200,6 +1248,7 @@ struct scx_sched { struct timer_list bypass_lb_timer; cpumask_var_t bypass_lb_donee_cpumask; cpumask_var_t bypass_lb_resched_cpumask; + cpumask_var_t stall_cpus; struct rcu_work rcu_work; /* all ancestors including self */ @@ -1463,21 +1512,24 @@ static const char *scx_enable_state_str[] = { * The sched_ext core uses a "lock dancing" protocol coordinated by * p->scx.holding_cpu. When moving a task to a different rq: * - * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) - * 2. Set p->scx.holding_cpu to the current CPU - * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING + * 1. Set p->scx.holding_cpu to the current CPU + * 2. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING * is set, so clearing DISPATCHING first prevents the circular wait * (safe to lock the rq we need) - * 4. Unlock the current CPU's rq - * 5. Lock src_rq (where the task currently lives) - * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the + * 3. Unlock the current CPU's rq + * 4. Lock src_rq (where the task currently lives) + * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the * race (dequeue clears holding_cpu to -1 when it takes the task), in * this case migration is aborted - * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly + * 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly * into dst_rq's local DSQ (no lock swap needed) - * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases - * src_rq, locks dst_rq, and performs the deactivate/activate - * migration cycle (dst_rq is held on return) + * 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq + * (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu, + * leave the task on src_rq, and enqueue it on the fallback DSQ. + * 8. Otherwise (i.e. if the task can be moved to dst_rq), call + * move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq, + * and performs the deactivate/activate migration cycle + * (dst_rq is held on return) * 9. Unlock dst_rq and re-lock the current CPU's rq to restore * the lock state expected by the caller * @@ -1511,6 +1563,41 @@ enum scx_ops_state { #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) +/* + * SCX task iterator. + */ +struct scx_task_iter { + struct sched_ext_entity cursor; + struct task_struct *locked_task; + struct rq *rq; + struct rq_flags rf; + u32 cnt; + bool list_locked; +#ifdef CONFIG_EXT_SUB_SCHED + struct cgroup *cgrp; + struct cgroup_subsys_state *css_pos; + struct css_task_iter css_iter; +#endif +}; + +/* + * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid + * starvation. During the READY -> ENABLED task switching loop, the calling + * thread's sched_class gets switched from fair to ext. As fair has higher + * priority than ext, the calling thread can be indefinitely starved under + * fair-class saturation, leading to a system hang. + */ +struct scx_enable_cmd { + struct kthread_work work; + union { + struct sched_ext_ops *ops; + struct sched_ext_ops_cid *ops_cid; + }; + bool is_cid_type; + struct bpf_map *arena_map; /* arena ref to transfer to sch */ + int ret; +}; + extern struct scx_sched __rcu *scx_root; DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); @@ -1531,12 +1618,112 @@ __printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu, const char *fmt, ...); +u32 scx_get_task_state(const struct task_struct *p); +void scx_set_task_state(struct task_struct *p, u32 state); +void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp); +void scx_task_iter_unlock(struct scx_task_iter *iter); +void scx_task_iter_stop(struct scx_task_iter *iter); +struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); +bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, + struct scx_dispatch_q *dsq, u64 enq_flags); +bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); +bool scx_rq_online(struct rq *rq); +void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); +void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, + u64 reenq_flags, struct rq *locked_rq); +int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork); +void scx_enable_task(struct scx_sched *sch, struct task_struct *p); +void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p); +void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p); +#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED) +void scx_cgroup_lock(void); +void scx_cgroup_unlock(void); +#endif +s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch); +void scx_disable_bypass_dsp(struct scx_sched *sch); +void scx_bypass(struct scx_sched *sch, bool bypass); +s32 scx_link_sched(struct scx_sched *sch); +void scx_unlink_sched(struct scx_sched *sch); +void scx_disable_dump(struct scx_sched *sch); +void scx_log_sched_disable(struct scx_sched *sch); +void scx_flush_disable_work(struct scx_sched *sch); +struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, + struct cgroup *cgrp, + struct scx_sched *parent); +int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops); + +extern raw_spinlock_t scx_sched_lock; +extern struct mutex scx_enable_mutex; +extern struct percpu_rw_semaphore scx_fork_rwsem; +#ifdef CONFIG_EXT_SUB_SCHED +extern const struct rhashtable_params scx_sched_hash_params; +extern struct rhashtable scx_sched_hash; +extern struct scx_sched *scx_enabling_sub_sched; +#endif + #define scx_exit(sch, kind, exit_code, fmt, args...) \ __scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args) #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) -#define scx_verror(sch, fmt, args) \ - scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args) + +static inline struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu) +{ + return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq; +} + +/** + * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled + * @sch: scheduler to check + * + * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled + * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors + * are bypassing. In the former case, the ancestor is not itself bypassing but + * its bypass DSQs will be populated with bypassed tasks from descendants. Thus, + * the ancestor's bypass dispatch path must be active even though its own + * bypass_depth remains zero. + * + * This function checks bypass_dsp_enable_depth which is managed separately from + * bypass_depth to enable this decoupling. See enable_bypass_dsp() and + * scx_disable_bypass_dsp(). + */ +static inline bool scx_bypass_dsp_enabled(struct scx_sched *sch) +{ + return unlikely(atomic_read(&sch->bypass_dsp_enable_depth)); +} + +/** + * scx_ops_sanitize_err - Sanitize a -errno value + * @sch: scx_sched to error out on error + * @ops_name: operation to blame on failure + * @err: -errno value to sanitize + * + * Verify @err is a valid -errno. If not, trigger scx_error() and return + * -%EPROTO. This is necessary because returning a rogue -errno up the chain can + * cause misbehaviors. For an example, a large negative return from + * ops.init_task() triggers an oops when passed up the call chain because the + * value fails IS_ERR() test after being encoded with ERR_PTR() and then is + * handled as a pointer. + */ +static inline int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err) +{ + if (err < 0 && err >= -MAX_ERRNO) + return err; + + scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err); + return -EPROTO; +} + +static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) +{ + struct scx_sched *root = rcu_dereference_sched(scx_root); + + if (WARN_ON_ONCE(!root)) + return; + + schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq); +} /* * Return the rq currently locked from an scx callback, or NULL if no rq is @@ -1547,6 +1734,111 @@ static inline struct rq *scx_locked_rq(void) return __this_cpu_read(scx_locked_rq_state); } +static inline void update_locked_rq(struct rq *rq) +{ + /* + * Check whether @rq is actually locked. This can help expose bugs + * or incorrect assumptions about the context in which a kfunc or + * callback is executed. + */ + if (rq) + lockdep_assert_rq_held(rq); + __this_cpu_write(scx_locked_rq_state, rq); +} + +#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) + +/* + * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not + * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit. + */ +#define SCX_CALL_OP(sch, op, locked_rq, args...) \ +do { \ + struct rq *__prev_locked_rq; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ +} while (0) + +#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \ +({ \ + struct rq *__prev_locked_rq; \ + __typeof__((sch)->ops.op(args)) __ret; \ + \ + if (locked_rq) { \ + __prev_locked_rq = scx_locked_rq(); \ + update_locked_rq(locked_rq); \ + } \ + __ret = (sch)->ops.op(args); \ + if (locked_rq) \ + update_locked_rq(__prev_locked_rq); \ + __ret; \ +}) + +/* + * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments + * and records them in current->scx.kf_tasks[] for the duration of the call. A + * kfunc invoked from inside such an op can then use + * scx_kf_arg_task_ok() to verify that its task argument is one of + * those subject tasks. + * + * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held - + * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's + * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. + * So if kf_tasks[] is set, @p's scheduler-protected fields are stable. + * + * kf_tasks[] can not stack, so task-based SCX ops must not nest. The + * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants + * while a previous one is still in progress. + */ +#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \ +do { \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + SCX_CALL_OP((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ +} while (0) + +#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \ +({ \ + __typeof__((sch)->ops.op(task, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + __ret; \ +}) + +#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \ +({ \ + __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ + WARN_ON_ONCE(current->scx.kf_tasks[0]); \ + current->scx.kf_tasks[0] = task0; \ + current->scx.kf_tasks[1] = task1; \ + __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \ + current->scx.kf_tasks[0] = NULL; \ + current->scx.kf_tasks[1] = NULL; \ + __ret; \ +}) + +/* see SCX_CALL_OP_TASK() */ +static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch, + struct task_struct *p) +{ + if (unlikely((p != current->scx.kf_tasks[0] && + p != current->scx.kf_tasks[1]))) { + scx_error(sch, "called on a task not being operated on"); + return false; + } + + return true; +} + static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) { return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags & @@ -1627,6 +1919,20 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) return NULL; } + +/** + * scx_parent - Find the parent sched + * @sch: sched to find the parent of + * + * Returns the parent scheduler or %NULL if @sch is root. + */ +static inline struct scx_sched *scx_parent(struct scx_sched *sch) +{ + if (sch->level) + return sch->ancestors[sch->level - 1]; + else + return NULL; +} #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_task_sched(const struct task_struct *p) { @@ -1650,4 +1956,8 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) { return rcu_dereference_all(scx_root); } + +static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; } #endif /* CONFIG_EXT_SUB_SCHED */ + +#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c new file mode 100644 index 000000000000..050420427273 --- /dev/null +++ b/kernel/sched/ext/sub.c @@ -0,0 +1,668 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * A sub-scheduler is an scx_sched attached to a cgroup subtree under another + * scx_sched. This file holds the sub-scheduler implementation: the scheduler + * tree walk, capability delegation, per-shard cap state and its sync, and the + * sub-scheduler enable/disable paths. The core dispatch/enqueue machinery it + * builds on lives in ext.c. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#include +#include "internal.h" +#include "cid.h" +#include "arena.h" +#include "sub.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +/** + * scx_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: sched whose descendants to walk + * + * To be used by scx_for_each_descendant_pre(). Find the next descendant to + * visit for pre-order traversal of @root's descendants. @root is included in + * the iteration and the first node to be visited. + */ +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) +{ + struct scx_sched *next; + + lockdep_assert(lockdep_is_held(&scx_enable_mutex) || + lockdep_is_held(&scx_sched_lock)); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + while (pos != root) { + if (!list_is_last(&pos->sibling, &scx_parent(pos)->children)) + return list_next_entry(pos, sibling); + pos = scx_parent(pos); + } + + return NULL; +} + +static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) +{ + return rhashtable_lookup(&scx_sched_hash, &cgroup_id, + scx_sched_hash_params); +} + +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) +{ + rcu_assign_pointer(p->scx.sched, sch); +} + +struct cgroup *sch_cgroup(struct scx_sched *sch) +{ + return sch->cgrp; +} + +/* for each descendant of @cgrp including self, set ->scx_sched to @sch */ +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) +{ + struct cgroup *pos; + struct cgroup_subsys_state *css; + + cgroup_for_each_live_descendant_pre(pos, css, cgrp) + rcu_assign_pointer(pos->scx_sched, sch); +} + +static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq); + +void drain_descendants(struct scx_sched *sch) +{ + /* + * Child scheds that finished the critical part of disabling will take + * themselves off @sch->children. Wait for it to drain. As propagation + * is recursive, empty @sch->children means that all proper descendant + * scheds reached unlinking stage. + */ + wait_event(scx_unlink_waitq, list_empty(&sch->children)); +} + +static void scx_fail_parent(struct scx_sched *sch, + struct task_struct *failed, s32 fail_code) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + + scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler", + fail_code, failed->comm, failed->pid); + + /* + * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into + * it. This may cause downstream failures on the BPF side but $parent is + * dying anyway. + */ + scx_bypass(parent, true); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (scx_task_on_sched(parent, p)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_disable_and_exit_task(sch, p); + scx_set_task_sched(p, parent); + } + } + scx_task_iter_stop(&sti); +} + +void scx_sub_disable(struct scx_sched *sch) +{ + struct scx_sched *parent = scx_parent(sch); + struct scx_task_iter sti; + struct task_struct *p; + int ret; + + /* + * Guarantee forward progress and wait for descendants to be disabled. + * To limit disruptions, $parent is not bypassed. Tasks are fully + * prepped and then inserted back into $parent. + */ + scx_bypass(sch, true); + drain_descendants(sch); + + /* + * Here, every runnable task is guaranteed to make forward progress and + * we can safely use blocking synchronization constructs. Actually + * disable ops. + */ + mutex_lock(&scx_enable_mutex); + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + set_cgroup_sched(sch_cgroup(sch), parent); + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* filter out duplicate visits */ + if (scx_task_on_sched(parent, p)) + continue; + + /* + * By the time control reaches here, all descendant schedulers + * should already have been disabled. + */ + WARN_ON_ONCE(!scx_task_on_sched(sch, p)); + + /* + * @p is pinned by the iter: css_task_iter_next() takes a + * reference and holds it until the next iter_next() call, so + * @p->usage is guaranteed > 0. + */ + get_task_struct(p); + + scx_task_iter_unlock(&sti); + + /* + * $p is READY or ENABLED on @sch. Initialize for $parent, + * disable and exit from @sch, and then switch over to $parent. + * + * If a task fails to initialize for $parent, the only available + * action is disabling $parent too. While this allows disabling + * of a child sched to cause the parent scheduler to fail, the + * failure can only originate from ops.init_task() of the + * parent. A child can't directly affect the parent through its + * own failures. + */ + ret = __scx_init_task(parent, p, false); + if (ret) { + scx_fail_parent(sch, p, ret); + put_task_struct(p); + break; + } + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on @sch (the + * sched @p was on at that point), not on $parent. + * $parent's just-completed init is owed an exit_task() + * and we issue it here. + */ + scx_sub_init_cancel_task(parent, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p is initialized for $parent and still attached to + * @sch. Disable and exit for @sch, switch over to + * $parent, override the state to READY to account for + * $p having already been initialized, and then enable. + */ + scx_disable_and_exit_task(sch, p); + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); + scx_set_task_state(p, SCX_TASK_INIT); + scx_set_task_sched(p, parent); + scx_set_task_state(p, SCX_TASK_READY); + scx_enable_task(parent, p); + } + + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + scx_disable_dump(sch); + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + /* + * All tasks are moved off of @sch but there may still be on-going + * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use + * the expedited version as ancestors may be waiting in bypass mode. + * Also, tell the parent that there is no need to keep running bypass + * DSQs for us. + */ + synchronize_rcu_expedited(); + scx_disable_bypass_dsp(sch); + + scx_unlink_sched(sch); + + mutex_unlock(&scx_enable_mutex); + + /* + * @sch is now unlinked from the parent's children list. Notify and call + * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called + * after unlinking and releasing all locks. See scx_claim_exit(). + */ + wake_up_all(&scx_unlink_waitq); + + if (parent->ops.sub_detach && sch->sub_attached) { + struct scx_sub_detach_args sub_detach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + SCX_CALL_OP(parent, sub_detach, NULL, + &sub_detach_args); + } + + scx_log_sched_disable(sch); + + if (sch->ops.exit) + SCX_CALL_OP(sch, exit, NULL, sch->exit_info); + if (sch->sub_kset) + kobject_del(&sch->sub_kset->kobj); + kobject_del(&sch->kobj); +} + +/* verify that a scheduler can be attached to @cgrp and return the parent */ +static struct scx_sched *find_parent_sched(struct cgroup *cgrp) +{ + struct scx_sched *parent = cgrp->scx_sched; + struct scx_sched *pos; + + lockdep_assert_held(&scx_sched_lock); + + /* can't attach twice to the same cgroup */ + if (parent->cgrp == cgrp) + return ERR_PTR(-EBUSY); + + /* does $parent allow sub-scheds? */ + if (!parent->ops.sub_attach) + return ERR_PTR(-EOPNOTSUPP); + + /* can't insert between $parent and its exiting children */ + list_for_each_entry(pos, &parent->children, sibling) + if (cgroup_is_descendant(pos->cgrp, cgrp)) + return ERR_PTR(-EBUSY); + + return parent; +} + +static bool assert_task_ready_or_enabled(struct task_struct *p) +{ + u32 state = scx_get_task_state(p); + + switch (state) { + case SCX_TASK_READY: + case SCX_TASK_ENABLED: + return true; + default: + WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched", + state, p->comm, p->pid); + return false; + } +} + +void scx_sub_enable_workfn(struct kthread_work *work) +{ + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); + struct sched_ext_ops *ops = cmd->ops; + struct cgroup *cgrp; + struct scx_sched *parent, *sch; + struct scx_task_iter sti; + struct task_struct *p; + s32 i, ret; + + mutex_lock(&scx_enable_mutex); + + if (!scx_enabled()) { + ret = -ENODEV; + goto out_unlock; + } + + /* See scx_root_enable_workfn() for the @ops->priv check. */ + if (rcu_access_pointer(ops->priv)) { + ret = -EBUSY; + goto out_unlock; + } + + cgrp = cgroup_get_from_id(ops->sub_cgroup_id); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + raw_spin_lock_irq(&scx_sched_lock); + parent = find_parent_sched(cgrp); + if (IS_ERR(parent)) { + raw_spin_unlock_irq(&scx_sched_lock); + ret = PTR_ERR(parent); + goto out_put_cgrp; + } + kobject_get(&parent->kobj); + raw_spin_unlock_irq(&scx_sched_lock); + + /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */ + sch = scx_alloc_and_add_sched(cmd, cgrp, parent); + kobject_put(&parent->kobj); + if (IS_ERR(sch)) { + ret = PTR_ERR(sch); + goto out_unlock; + } + + ret = scx_link_sched(sch); + if (ret) + goto err_disable; + + if (sch->level >= SCX_SUB_MAX_DEPTH) { + scx_error(sch, "max nesting depth %d violated", + SCX_SUB_MAX_DEPTH); + goto err_disable; + } + + if (sch->ops.init) { + ret = SCX_CALL_OP_RET(sch, init, NULL); + if (ret) { + ret = scx_ops_sanitize_err(sch, "init", ret); + scx_error(sch, "ops.init() failed (%d)", ret); + goto err_disable; + } + sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + } + + ret = scx_arena_pool_init(sch); + if (ret) + goto err_disable; + + ret = scx_set_cmask_scratch_alloc(sch); + if (ret) + goto err_disable; + + if (scx_validate_ops(sch, ops)) + goto err_disable; + + struct scx_sub_attach_args sub_attach_args = { + .ops = &sch->ops, + .cgroup_path = sch->cgrp_path, + }; + + ret = SCX_CALL_OP_RET(parent, sub_attach, NULL, + &sub_attach_args); + if (ret) { + ret = scx_ops_sanitize_err(sch, "sub_attach", ret); + scx_error(sch, "parent rejected (%d)", ret); + goto err_disable; + } + sch->sub_attached = true; + + scx_bypass(sch, true); + + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) + if (((void (**)(void))ops)[i]) + set_bit(i, sch->has_op); + + percpu_down_write(&scx_fork_rwsem); + scx_cgroup_lock(); + + /* + * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see + * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down. + */ + set_cgroup_sched(sch_cgroup(sch), sch); + if (!(cgrp->self.flags & CSS_ONLINE)) { + scx_error(sch, "cgroup is not online"); + goto err_unlock_and_disable; + } + + /* + * Initialize tasks for the new child $sch without exiting them for + * $parent so that the tasks can always be reverted back to $parent + * sched on child init failure. + */ + WARN_ON_ONCE(scx_enabling_sub_sched); + scx_enabling_sub_sched = sch; + + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + struct rq *rq; + struct rq_flags rf; + + /* + * Task iteration may visit the same task twice when racing + * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which + * finished __scx_init_task() and skip if set. + * + * A task may exit and get freed between __scx_init_task() + * completion and scx_enable_task(). In such cases, + * scx_disable_and_exit_task() must exit the task for both the + * parent and child scheds. + */ + if (p->scx.flags & SCX_TASK_SUB_INIT) + continue; + + /* @p is pinned by the iter; see scx_sub_disable() */ + get_task_struct(p); + + if (!assert_task_ready_or_enabled(p)) { + ret = -EINVAL; + goto abort; + } + + scx_task_iter_unlock(&sti); + + /* + * As $p is still on $parent, it can't be transitioned to INIT. + * Let's worry about task state later. Use __scx_init_task(). + */ + ret = __scx_init_task(sch, p, false); + if (ret) + goto abort; + + rq = task_rq_lock(p, &rf); + + if (scx_get_task_state(p) == SCX_TASK_DEAD) { + /* + * sched_ext_dead() raced us between __scx_init_task() + * and this rq lock and ran exit_task() on $parent (the + * sched @p was on at that point), not on @sch. @sch's + * just-completed init is owed an exit_task() and we + * issue it here. + */ + scx_sub_init_cancel_task(sch, p); + task_rq_unlock(rq, p, &rf); + put_task_struct(p); + continue; + } + + p->scx.flags |= SCX_TASK_SUB_INIT; + task_rq_unlock(rq, p, &rf); + + put_task_struct(p); + } + scx_task_iter_stop(&sti); + + /* + * All tasks are prepped. Disable/exit tasks for $parent and enable for + * the new @sch. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + /* + * Use clearing of %SCX_TASK_SUB_INIT to detect and skip + * duplicate iterations. + */ + if (!(p->scx.flags & SCX_TASK_SUB_INIT)) + continue; + + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + /* + * $p must be either READY or ENABLED. If ENABLED, + * __scx_disabled_and_exit_task() first disables and + * makes it READY. However, after exiting $p, it will + * leave $p as READY. + */ + assert_task_ready_or_enabled(p); + __scx_disable_and_exit_task(parent, p); + + /* + * $p is now only initialized for @sch and READY, which + * is what we want. Assign it to @sch and enable. + */ + scx_set_task_sched(p, sch); + scx_enable_task(sch, p); + + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + + scx_enabling_sub_sched = NULL; + + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); + + scx_bypass(sch, false); + + pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); + kobject_uevent(&sch->kobj, KOBJ_ADD); + ret = 0; + goto out_unlock; + +out_put_cgrp: + cgroup_put(cgrp); +out_unlock: + mutex_unlock(&scx_enable_mutex); + cmd->ret = ret; + return; + +abort: + put_task_struct(p); + scx_task_iter_stop(&sti); + + /* + * Undo __scx_init_task() for tasks we marked. scx_enable_task() never + * ran for @sch on them, so calling scx_disable_task() here would invoke + * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched + * must stay set until SUB_INIT is cleared from every marked task - + * scx_disable_and_exit_task() reads it when a task exits concurrently. + */ + scx_task_iter_start(&sti, sch->cgrp); + while ((p = scx_task_iter_next_locked(&sti))) { + if (p->scx.flags & SCX_TASK_SUB_INIT) { + scx_sub_init_cancel_task(sch, p); + p->scx.flags &= ~SCX_TASK_SUB_INIT; + } + } + scx_task_iter_stop(&sti); + scx_enabling_sub_sched = NULL; +err_unlock_and_disable: + /* we'll soon enter disable path, keep bypass on */ + scx_cgroup_unlock(); + percpu_up_write(&scx_fork_rwsem); +err_disable: + mutex_unlock(&scx_enable_mutex); + scx_flush_disable_work(sch); + cmd->ret = 0; +} + +static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cgroup *cgrp = data; + struct cgroup *parent = cgroup_parent(cgrp); + + if (!cgroup_on_dfl(cgrp)) + return NOTIFY_OK; + + switch (action) { + case CGROUP_LIFETIME_ONLINE: + /* inherit ->scx_sched from $parent */ + if (parent) + rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched); + break; + case CGROUP_LIFETIME_OFFLINE: + /* if there is a sched attached, shoot it down */ + if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp) + scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN, + SCX_ECODE_RSN_CGROUP_OFFLINE, + "cgroup %llu going offline", cgroup_id(cgrp)); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block scx_cgroup_lifetime_nb = { + .notifier_call = scx_cgroup_lifetime_notify, +}; + +static s32 __init scx_cgroup_lifetime_notifier_init(void) +{ + return blocking_notifier_chain_register(&cgroup_lifetime_notifier, + &scx_cgroup_lifetime_nb); +} +core_initcall(scx_cgroup_lifetime_notifier_init); + +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(prog->aux); + if (unlikely(!sch)) + return; + + scx_error(sch, "dispatch recursion detected"); +} + +__bpf_kfunc_start_defs(); + +/** + * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler + * @cgroup_id: cgroup ID of the child scheduler to dispatch + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Allows a parent scheduler to trigger dispatching on one of its direct + * child schedulers. The child scheduler runs its dispatch operation to + * move tasks from dispatch queues to the local runqueue. + * + * Returns: true on success, false if cgroup_id is invalid, not a direct + * child, or caller lacks dispatch permission. + */ +__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux) +{ + struct rq *this_rq = this_rq(); + struct scx_sched *parent, *child; + + guard(rcu)(); + parent = scx_prog_sched(aux); + if (unlikely(!parent)) + return false; + + child = scx_find_sub_sched(cgroup_id); + + if (unlikely(!child)) + return false; + + if (unlikely(scx_parent(child) != parent)) { + scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu", + cgroup_id); + return false; + } + + return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev, + true); +} + +__bpf_kfunc_end_defs(); + +#endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h new file mode 100644 index 000000000000..460a9fd196dc --- /dev/null +++ b/kernel/sched/ext/sub.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst + * + * Sub-scheduler hierarchy support. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef _KERNEL_SCHED_EXT_SUB_H +#define _KERNEL_SCHED_EXT_SUB_H + +#include "internal.h" +#include "cid.h" + +#ifdef CONFIG_EXT_SUB_SCHED + +struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); +struct cgroup *sch_cgroup(struct scx_sched *sch); +void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); +void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog); +void drain_descendants(struct scx_sched *sch); +void scx_sub_disable(struct scx_sched *sch); +void scx_sub_enable_workfn(struct kthread_work *work); +bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); + +#else /* CONFIG_EXT_SUB_SCHED */ + +static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } +static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} +static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } +static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {} +static inline void drain_descendants(struct scx_sched *sch) { } +static inline void scx_sub_disable(struct scx_sched *sch) { } + +#endif /* CONFIG_EXT_SUB_SCHED */ + +/** + * scx_for_each_descendant_pre - pre-order walk of a sched's descendants + * @pos: iteration cursor + * @root: sched to walk the descendants of + * + * Walk @root's descendants. @root is included in the iteration and the first + * node to be visited. Must be called with either scx_enable_mutex or + * scx_sched_lock held. + */ +#define scx_for_each_descendant_pre(pos, root) \ + for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos); \ + (pos) = scx_next_descendant_pre((pos), (root))) + +/* + * One user of this function is scx_bpf_dispatch() which can be called + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage + * from the call frame. + */ +static __always_inline bool +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, + struct task_struct *prev, bool nested) +{ + struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx; + int nr_loops = SCX_DSP_MAX_LOOPS; + s32 cpu = cpu_of(rq); + bool prev_on_sch = (prev->sched_class == &ext_sched_class) && + scx_task_on_sched(sch, prev); + + if (scx_consume_global_dsq(sch, rq)) + return true; + + if (scx_bypass_dsp_enabled(sch)) { + /* if @sch is bypassing, only the bypass DSQs are active */ + if (scx_bypassing(sch, cpu)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + +#ifdef CONFIG_EXT_SUB_SCHED + /* + * If @sch isn't bypassing but its children are, @sch is + * responsible for making forward progress for both its own + * tasks that aren't bypassing and the bypassing descendants' + * tasks. The following implements a simple built-in behavior - + * let each CPU try to run the bypass DSQ every Nth time. + * + * Later, if necessary, we can add an ops flag to suppress the + * auto-consumption and a kfunc to consume the bypass DSQ and, + * so that the BPF scheduler can fully control scheduling of + * bypassed tasks. + */ + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); + return true; + } +#endif /* CONFIG_EXT_SUB_SCHED */ + } + + if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq)) + return false; + + dspc->rq = rq; + + /* + * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq + * lock, the local DSQ might still end up empty after a successful + * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() + * produced some tasks, retry. The BPF scheduler may depend on this + * looping behavior to simplify its implementation. + */ + do { + dspc->nr_tasks = 0; + + if (nested) { + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + } else { + /* stash @prev so that nested invocations can access it */ + rq->scx.sub_dispatch_prev = prev; + SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu), + prev_on_sch ? prev : NULL); + rq->scx.sub_dispatch_prev = NULL; + } + + scx_flush_dispatch_buf(sch, rq); + + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { + rq->scx.flags |= SCX_RQ_BAL_KEEP; + return true; + } + if (rq->scx.local_dsq.nr) + return true; + if (scx_consume_global_dsq(sch, rq)) + return true; + + /* + * ops.dispatch() can trap us in this loop by repeatedly + * dispatching ineligible tasks. Break out once in a while to + * allow the watchdog to run. As IRQ can't be enabled in + * balance(), we want to complete this scheduling cycle and then + * start a new one. IOW, we want to call resched_curr() on the + * next, most likely idle, task, not the current one. Use + * __scx_bpf_kick_cpu() for deferred kicking. + */ + if (unlikely(!--nr_loops)) { + scx_kick_cpu(sch, cpu, 0); + break; + } + } while (dspc->nr_tasks); + + /* + * Prevent the CPU from going idle while bypassed descendants have tasks + * queued. Without this fallback, bypassed tasks could stall if the host + * scheduler's ops.dispatch() doesn't yield any tasks. + */ + if (scx_bypass_dsp_enabled(sch)) + return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0); + + return false; +} + +#endif /* _KERNEL_SCHED_EXT_SUB_H */ diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext/types.h similarity index 97% rename from kernel/sched/ext_types.h rename to kernel/sched/ext/types.h index 8b3527e21fca..bc74eafd43f1 100644 --- a/kernel/sched/ext_types.h +++ b/kernel/sched/ext/types.h @@ -8,6 +8,12 @@ #ifndef _KERNEL_SCHED_EXT_TYPES_H #define _KERNEL_SCHED_EXT_TYPES_H +#include +#include +#include +#include +#include + enum scx_consts { SCX_DSP_DFL_MAX_BATCH = 32, SCX_DSP_MAX_LOOPS = 32, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d78467ec6ee1..df8c9c2c7918 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -72,7 +73,10 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; /* - * Minimal preemption granularity for CPU-bound tasks: + * Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH: + * + * Under EEVDF this is the request size used to compute the virtual + * deadline; see update_deadline(). * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ @@ -296,8 +300,8 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = __calc_delta(delta, NICE_0_LOAD, &se->load); + if (se->h_load.weight != NICE_0_LOAD) + delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); return delta; } @@ -427,38 +431,6 @@ static inline struct sched_entity *parent_entity(const struct sched_entity *se) return se->parent; } -static void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ - int se_depth, pse_depth; - - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ - - /* First walk up until both entities are at same depth */ - se_depth = (*se)->depth; - pse_depth = (*pse)->depth; - - while (se_depth > pse_depth) { - se_depth--; - *se = parent_entity(*se); - } - - while (pse_depth > se_depth) { - pse_depth--; - *pse = parent_entity(*pse); - } - - while (!is_same_group(*se, *pse)) { - *se = parent_entity(*se); - *pse = parent_entity(*pse); - } -} - static int tg_is_idle(struct task_group *tg) { return tg->idle > 0; @@ -502,11 +474,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return NULL; } -static inline void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ -} - static inline int tg_is_idle(struct task_group *tg) { return 0; @@ -685,7 +652,7 @@ static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned static inline void __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 w_vruntime, key = entity_key(cfs_rq, se); w_vruntime = key * weight; @@ -702,7 +669,7 @@ sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se) s64 key, tmp; again: - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); key = entity_key(cfs_rq, se); if (check_mul_overflow(key, weight, &key)) @@ -748,7 +715,7 @@ sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) static void sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 key = entity_key(cfs_rq, se); cfs_rq->sum_w_vruntime -= key * weight; @@ -790,7 +757,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) s64 runtime = cfs_rq->sum_w_vruntime; if (curr) { - unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); runtime += entity_key(cfs_rq, curr) * w; weight += w; @@ -813,6 +780,67 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) return cfs_rq->zero_vruntime; } +/* + * \Sum (v_i - v0)*w_i + * V = ------------------- + v0 + * \Sum w_i + * + * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus: + * + * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j) + * + * \Sum_i!=j (v_i - v0)*w_i + * v_j = ------------------------ + v0 + * W - w_j + * + * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i' + * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr + * is not included in the sum. + */ +static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) +{ + struct sched_entity *curr = cfs_rq->curr; + long weight = cfs_rq->sum_weight; + s64 delta = 0; + + if (curr && !curr->on_rq) + curr = NULL; + + /* + * This is called from set_next_task_fair(.first=true) / + * set_protect_slice() so curr had better be set and on_rq. + */ + WARN_ON_ONCE(!curr); + + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + /* + * Do not add @curr to obtain the effective '- w_j' terms. + */ + + /* sign flips effective floor / ceiling */ + if (runtime < 0) + runtime -= (weight - 1); + + delta = div64_long(runtime, weight); + } + + return cfs_rq->zero_vruntime + delta + 1; +} + static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); /* @@ -861,8 +889,6 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) u64 avruntime = avg_vruntime(cfs_rq); s64 vlag = entity_lag(cfs_rq, se, avruntime); - WARN_ON_ONCE(!se->on_rq); - if (se->sched_delayed) { /* previous vlag < 0 otherwise se would not be delayed */ vlag = max(vlag, se->vlag); @@ -898,7 +924,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { - unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); avg += entity_key(cfs_rq, curr) * weight; load += weight; @@ -1039,6 +1065,9 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; @@ -1048,6 +1077,9 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); sum_w_vruntime_sub(cfs_rq, se); @@ -1090,8 +1122,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity slice = cfs_rq_min_slice(cfs_rq); slice = min(slice, se->slice); - if (slice != se->slice) - vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + + /* If there are shorter slices than se's one */ + if (slice != se->slice) { + if (sched_feat(PREEMPT_SHORT)) + vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); + else + vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + } se->vprot = vprot; } @@ -1099,8 +1137,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { u64 slice = cfs_rq_min_slice(cfs_rq); + u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); - se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); + se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); } static inline bool protect_slice(struct sched_entity *se) @@ -1144,7 +1183,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) * We can safely skip eligibility check if there is only one entity * in this cfs_rq, saving some cycles. */ - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return curr && curr->on_rq ? curr : se; /* @@ -1394,8 +1433,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) return delta_exec; } -static void set_next_buddy(struct sched_entity *se); - #ifdef CONFIG_SCHED_CACHE /* @@ -1990,7 +2027,7 @@ static void update_curr(struct cfs_rq *cfs_rq) * not necessarily be the actual task running * (rq->curr.se). This is easy to confuse! */ - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); s64 delta_exec; bool resched; @@ -2002,26 +2039,29 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; + account_cfs_rq_runtime(cfs_rq, delta_exec); + + if (!entity_is_task(curr)) + return; + + cfs_rq = &rq->cfs; + curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - if (entity_is_task(curr)) { - /* - * If the fair_server is active, we need to account for the - * fair_server time whether or not the task is running on - * behalf of fair_server or not: - * - If the task is running on behalf of fair_server, we need - * to limit its time based on the assigned runtime. - * - Fair task that runs outside of fair_server should account - * against fair_server such that it can account for this time - * and possibly avoid running this period. - */ - dl_server_update(&rq->fair_server, delta_exec); - } - - account_cfs_rq_runtime(cfs_rq, delta_exec); + /* + * If the fair_server is active, we need to account for the + * fair_server time whether or not the task is running on + * behalf of fair_server or not: + * - If the task is running on behalf of fair_server, we need + * to limit its time based on the assigned runtime. + * - Fair task that runs outside of fair_server should account + * against fair_server such that it can account for this time + * and possibly avoid running this period. + */ + dl_server_update(&rq->fair_server, delta_exec); - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return; if (resched || !protect_slice(curr)) { @@ -2032,7 +2072,10 @@ static void update_curr(struct cfs_rq *cfs_rq) static void update_curr_fair(struct rq *rq) { - update_curr(cfs_rq_of(&rq->donor->se)); + struct sched_entity *se = &rq->donor->se; + + for_each_sched_entity(se) + update_curr(cfs_rq_of(se)); } static inline void @@ -2108,7 +2151,7 @@ update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_start_fair(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) @@ -2126,7 +2169,7 @@ update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Mark the end of the wait period if dequeueing a * waiting task: */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_end_fair(cfs_rq, se); if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { @@ -4467,6 +4510,7 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4482,6 +4526,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4563,7 +4608,7 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) static void rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) { - unsigned long old_weight = se->load.weight; + long old_weight = se->h_load.weight; /* * VRUNTIME @@ -4663,16 +4708,17 @@ rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) se->vprot = div64_long(se->vprot * old_weight, weight); } -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) +static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight, bool on_rq) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; u64 avruntime = 0; - if (se->on_rq) { - /* commit outstanding execution time */ - update_curr(cfs_rq); + if (se->h_load.weight == weight) + return; + + if (on_rq) { avruntime = avg_vruntime(cfs_rq); se->vlag = entity_lag(cfs_rq, se, avruntime); se->deadline -= avruntime; @@ -4682,46 +4728,90 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, rel_vprot = true; } - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); - update_load_sub(&cfs_rq->load, se->load.weight); } - dequeue_load_avg(cfs_rq, se); rescale_entity(se, weight, rel_vprot); - update_load_set(&se->load, weight); + update_load_set(&se->h_load, weight); - do { - u32 divider = get_pelt_divider(&se->avg); - se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); - } while (0); - - enqueue_load_avg(cfs_rq, se); - if (se->on_rq) { + if (on_rq) { if (rel_vprot) se->vprot += avruntime; se->deadline += avruntime; se->rel_deadline = 0; se->vruntime = avruntime - se->vlag; - update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } } +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + if (se->load.weight == weight) + return; + + if (se->on_rq) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); + update_load_sub(&cfs_rq->load, se->load.weight); + } + dequeue_load_avg(cfs_rq, se); + + update_load_set(&se->load, weight); + + do { + u32 divider = get_pelt_divider(&se->avg); + se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); + } while (0); + + enqueue_load_avg(cfs_rq, se); + + if (se->on_rq) + update_load_add(&cfs_rq->load, se->load.weight); +} + +/* + * weight = NICE_0_LOAD; + * for_each_entity_se(se) + * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + */ +static __always_inline +unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + weight *= se->load.weight; + if (parent_entity(se)) + weight /= cfs_rq->load.weight; + else + weight /= NICE_0_LOAD; + + return max(weight, MIN_SHARES); +} + static void reweight_task_fair(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - struct load_weight *load = &se->load; + unsigned long weight = NICE_0_LOAD; + + if (se->on_rq) + update_curr_fair(rq); + + reweight_entity(cfs_rq_of(se), se, lw->weight); + se->load.inv_weight = lw->inv_weight; - reweight_entity(cfs_rq, se, lw->weight); - load->inv_weight = lw->inv_weight; + if (!se->on_rq) + return; + + for_each_sched_entity(se) + weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + + reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); @@ -4800,12 +4890,10 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); * * hence icky! */ -static long calc_group_shares(struct cfs_rq *cfs_rq) +static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) { - long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - - tg_shares = READ_ONCE(tg->shares); + long tg_weight, load, shares; load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -4831,7 +4919,115 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, tg_shares); + return clamp_t(long, shares, MIN_SHARES, shares_max); +} + +static int tg_cpus(struct task_group *tg) +{ + int nr = num_online_cpus(); + + if (cpusets_enabled()) { + struct cgroup *cgrp = tg->css.cgroup; + if (cgrp) + nr = cpuset_num_cpus(cgrp); + } + + return nr; +} + +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + +/* + * Func: fraction(nr_tasks * tg->shares) + * + * Scale tg->shares by the number of tasks. + */ +static long calc_tasks_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_tasks(tg); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: min(fraction(nr_cpus * tg->shares), nice -20) + * + * Scale tg->shares by the maximal number of CPUs; but clip the max shares at + * nice -20, otherwise a single spinner on a 512 CPU machine would result in + * 512*NICE_0_LOAD, which is also crazy. + */ +static long calc_max_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_cpus(tg); + long tg_shares = READ_ONCE(tg->shares); + long max_shares = scale_load(sched_prio_to_weight[0]); + return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); +} + +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: fraction(tg->shares) + * + * This infamously results in tiny shares when you have many CPUs. + */ +static long calc_smp_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); +} + +/* + * Ignore this pesky SMP stuff, use (4). + */ +static long calc_up_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + return READ_ONCE(tg->shares); +} + +DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); + +void __sched_cgroup_mode_update(int mode) +{ + long (*func)(struct cfs_rq *); + switch (mode) { + case 0: + func = &calc_up_shares; + break; + case 1: + func = &calc_smp_shares; + break; + case 2: + default: + func = &calc_concur_shares; + break; + case 3: + func = &calc_max_shares; + break; + case 4: + func = &calc_tasks_shares; + break; + } + static_call_update(calc_group_shares, func); } /* @@ -4850,9 +5046,8 @@ static void update_cfs_group(struct sched_entity *se) if (!gcfs_rq || !gcfs_rq->load.weight) return; - shares = calc_group_shares(gcfs_rq); - if (unlikely(se->load.weight != shares)) - reweight_entity(cfs_rq_of(se), se, shares); + shares = static_call(calc_group_shares)(gcfs_rq); + reweight_entity(cfs_rq_of(se), se, shares); } #else /* !CONFIG_FAIR_GROUP_SCHED: */ @@ -4970,11 +5165,11 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) * differential update where we store the last value we propagated. This in * turn allows skipping updates if the differential is 'small'. * - * Updating tg's load_avg is necessary before update_cfs_share(). + * Updating tg's load_avg is necessary before update_cfs_group(). */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -4995,17 +5190,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5015,9 +5214,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } @@ -5174,7 +5376,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -5188,7 +5391,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -5430,7 +5633,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {} * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) * avg. The immediate corollary is that all (fair) tasks must be attached. * - * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. + * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. * * Return: true if the load decayed or we removed load. * @@ -5968,6 +6171,7 @@ static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { u64 vslice, vruntime = avg_vruntime(cfs_rq); + unsigned int nr_queued = cfs_rq->h_nr_queued; bool update_zero = false; s64 lag = 0; @@ -5975,6 +6179,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) se->slice = sysctl_sched_base_slice; vslice = calc_delta_fair(se->slice, se); + if (flags & ENQUEUE_QUEUED) + nr_queued -= 1; + /* * Due to how V is constructed as the weighted average of entities, * adding tasks with positive lag, or removing tasks with negative lag @@ -5983,7 +6190,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ - if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { + if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { struct sched_entity *curr = cfs_rq->curr; long load, weight; @@ -6043,9 +6250,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ load = cfs_rq->sum_weight; if (curr && curr->on_rq) - load += avg_vruntime_weight(cfs_rq, curr->load.weight); + load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; @@ -6103,23 +6310,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) static void check_enqueue_throttle(struct cfs_rq *cfs_rq); static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); -static void -requeue_delayed_entity(struct sched_entity *se); - static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool curr = cfs_rq->curr == se; - - /* - * If we're the current task, we must renormalise before calling - * update_curr(). - */ - if (curr) - place_entity(cfs_rq, se, flags); - - update_curr(cfs_rq); - /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. @@ -6138,13 +6331,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ update_cfs_group(se); - /* - * XXX now that the entity has been re-weighted, and it's lag adjusted, - * we can place the entity. - */ - if (!curr) - place_entity(cfs_rq, se, flags); - account_entity_enqueue(cfs_rq, se); /* Entity has migrated, no longer consider this task hot */ @@ -6153,8 +6339,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); - if (!curr) - __enqueue_entity(cfs_rq, se); se->on_rq = 1; if (cfs_rq->nr_queued == 1) { @@ -6172,21 +6356,19 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } } -static void __clear_buddies_next(struct sched_entity *se) +static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next != se) - break; - - cfs_rq->next = NULL; - } + if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) + return; + if (se_is_idle(se)) + return; + cfs_rq->next = se; } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->next == se) - __clear_buddies_next(se); + cfs_rq->next = NULL; } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -6197,7 +6379,7 @@ static void set_delayed(struct sched_entity *se) /* * Delayed se of cfs_rq have no tasks queued on them. - * Do not adjust h_nr_runnable since dequeue_entities() + * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. */ if (!entity_is_task(se)) @@ -6230,45 +6412,16 @@ static void clear_delayed(struct sched_entity *se) } } -static bool +static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool sleep = flags & DEQUEUE_SLEEP; - int action = 0; - - update_curr(cfs_rq); - clear_buddies(cfs_rq, se); - - if (flags & DEQUEUE_DELAYED) { - WARN_ON_ONCE(!se->sched_delayed); - } else { - bool delay = sleep; - /* - * DELAY_DEQUEUE relies on spurious wakeups, special task - * states must not suffer spurious wakeups, excempt them. - */ - if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) - delay = false; - - WARN_ON_ONCE(delay && se->sched_delayed); - - if (sched_feat(DELAY_DEQUEUE) && delay && - !entity_eligible(cfs_rq, se)) { - if (entity_is_task(se)) - action |= UPDATE_UTIL_EST; - update_load_avg(cfs_rq, se, action); - update_entity_lag(cfs_rq, se); - set_delayed(se); - return false; - } - } + int action = UPDATE_TG; - action = UPDATE_TG; if (entity_is_task(se)) { if (task_on_rq_migrating(task_of(se))) action |= DO_DETACH; - if (sleep && !(flags & DEQUEUE_DELAYED)) + if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) action |= UPDATE_UTIL_EST; } @@ -6286,14 +6439,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_stats_dequeue_fair(cfs_rq, se, flags); - update_entity_lag(cfs_rq, se); - if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { - se->deadline -= se->vruntime; - se->rel_deadline = 1; - } - - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); se->on_rq = 0; account_entity_dequeue(cfs_rq, se); @@ -6302,9 +6447,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_cfs_group(se); - if (flags & DEQUEUE_DELAYED) - clear_delayed(se); - if (cfs_rq->nr_queued == 0) { update_idle_cfs_rq_clock_pelt(cfs_rq); #ifdef CONFIG_CFS_BANDWIDTH @@ -6317,15 +6459,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } #endif } - - return true; } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - clear_buddies(cfs_rq, se); - /* 'current' is not kept within the tree. */ if (se->on_rq) { /* @@ -6334,16 +6472,12 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) * runqueue. */ update_stats_wait_end_fair(cfs_rq, se); - __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - - if (first) - set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); - WARN_ON_ONCE(cfs_rq->curr); - cfs_rq->curr = se; + WARN_ON_ONCE(cfs_rq->h_curr); + cfs_rq->h_curr = se; /* * Track our maximum slice length, if the CPU's load is at @@ -6363,23 +6497,17 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) se->prev_sum_exec_runtime = se->sum_exec_runtime; } -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); -/* - * Pick the next process, keeping these things in mind, in this order: - * 1) keep things fair between processes/task groups - * 2) pick the "next" process, since someone really wants that to run - * 3) pick the "last" process, for cache locality - * 4) do not run the "skip" process, if something else is available - */ static struct sched_entity * -pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) +pick_next_entity(struct rq *rq, bool protect) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* * Must not reference @se again, see __block_task(). */ @@ -6399,13 +6527,11 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); - /* Put 'current' back into the tree. */ - __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ update_load_avg(cfs_rq, prev, 0); } - WARN_ON_ONCE(cfs_rq->curr != prev); - cfs_rq->curr = NULL; + WARN_ON_ONCE(cfs_rq->h_curr != prev); + cfs_rq->h_curr = NULL; } static void @@ -6960,7 +7086,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ - if (rq->curr == rq->idle && rq->cfs.nr_queued) + if (rq->curr == rq->idle && rq->cfs.h_nr_queued) resched_curr(rq); } @@ -7295,7 +7421,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) return; /* an active group must be handled by the update_curr() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) + if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) return; /* ensure the group is not already throttled */ @@ -7667,7 +7793,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) resched_curr(rq); return; } - delta = (se->load.weight * vdelta) / NICE_0_LOAD; + delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; /* * Correct for instantaneous load of other classes. @@ -7767,10 +7893,8 @@ static int choose_idle_cpu(int cpu, struct task_struct *p) } static void -requeue_delayed_entity(struct sched_entity *se) +requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - /* * se->sched_delayed should imply: se->on_rq == 1. * Because a delayed entity is one that is still on @@ -7780,34 +7904,72 @@ requeue_delayed_entity(struct sched_entity *se) WARN_ON_ONCE(!se->on_rq); if (update_entity_lag(cfs_rq, se)) { - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); place_entity(cfs_rq, se, 0); if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } update_load_avg(cfs_rq, se, 0); clear_delayed(se); } +static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) +{ + unsigned long weight = NICE_0_LOAD; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + int h_nr_idle = task_has_idle_policy(p); + int h_nr_runnable = 1; + + if (task_new && se->sched_delayed) + h_nr_runnable = 0; + + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + update_curr(cfs_rq); + + if (!se->on_rq) { + enqueue_entity(cfs_rq, se, flags); + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); + } + + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; + + weight = __calc_prop_weight(cfs_rq, se, weight); + + flags = ENQUEUE_WAKEUP; + } + + return weight; +} + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and * then put the task into the rbtree: */ static void -enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) -{ - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - int h_nr_idle = task_has_idle_policy(p); - int h_nr_runnable = 1; - int task_new = !(flags & ENQUEUE_WAKEUP); +enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) +{ int rq_h_nr_queued = rq->cfs.h_nr_queued; - u64 slice = 0; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight; + bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -7819,10 +7981,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) * estimated utilization, before we update schedutil. */ if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) - util_est_enqueue(&rq->cfs, p); + util_est_enqueue(cfs_rq, p); if (flags & ENQUEUE_DELAYED) { - requeue_delayed_entity(se); + requeue_delayed_entity(cfs_rq, se); return; } @@ -7834,57 +7996,22 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - if (task_new && se->sched_delayed) - h_nr_runnable = 0; - - for_each_sched_entity(se) { - if (se->on_rq) { - if (se->sched_delayed) - requeue_delayed_entity(se); - break; - } - cfs_rq = cfs_rq_of(se); - - /* - * Basically set the slice of group entries to the min_slice of - * their respective cfs_rq. This ensures the group can service - * its entities in the desired time-frame. - */ - if (slice) { - se->slice = slice; - se->custom_slice = 1; - } - enqueue_entity(cfs_rq, se, flags); - slice = cfs_rq_min_slice(cfs_rq); - - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; - - flags = ENQUEUE_WAKEUP; - } - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + /* + * XXX comment on the curr thing + */ + curr = (cfs_rq->curr == se); + if (curr) + place_entity(cfs_rq, se, flags); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + if (se->on_rq && se->sched_delayed) + requeue_delayed_entity(cfs_rq, se); - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; + weight = enqueue_hierarchy(p, flags); - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; + if (!curr) { + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) @@ -7915,105 +8042,109 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) hrtick_update(rq); } -/* - * Basically dequeue_task_fair(), except it can deal with dequeue_entity() - * failing half-way through and resume the dequeue later. - * - * Returns: - * -1 - dequeue delayed - * 0 - dequeue throttled - * 1 - dequeue complete - */ -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) +static void dequeue_hierarchy(struct task_struct *p, int flags) { - bool was_sched_idle = sched_idle_rq(rq); + struct sched_entity *se = &p->se; bool task_sleep = flags & DEQUEUE_SLEEP; bool task_delayed = flags & DEQUEUE_DELAYED; bool task_throttled = flags & DEQUEUE_THROTTLE; - struct task_struct *p = NULL; - int h_nr_idle = 0; - int h_nr_queued = 0; int h_nr_runnable = 0; - struct cfs_rq *cfs_rq; - u64 slice = 0; + int h_nr_idle = task_has_idle_policy(p); + bool dequeue = true; - if (entity_is_task(se)) { - p = task_of(se); - h_nr_queued = 1; - h_nr_idle = task_has_idle_policy(p); - if (task_sleep || task_delayed || !se->sched_delayed) - h_nr_runnable = 1; - } + if (task_sleep || task_delayed || !se->sched_delayed) + h_nr_runnable = 1; for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!dequeue_entity(cfs_rq, se, flags)) { - if (p && &p->se == se) - return -1; + update_curr(cfs_rq); - slice = cfs_rq_min_slice(cfs_rq); - break; + if (dequeue) { + dequeue_entity(cfs_rq, se, flags); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) + dequeue = false; + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); } cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; + cfs_rq->h_nr_queued--; cfs_rq->h_nr_idle -= h_nr_idle; if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + h_nr_idle = 1; if (throttled_hierarchy(cfs_rq) && task_throttled) record_throttle_clock(cfs_rq); - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) { - slice = cfs_rq_min_slice(cfs_rq); - - /* Avoid re-evaluating load for this entity: */ - se = parent_entity(se); - /* - * Bias pick_next to pick a task from this cfs_rq, as - * p is sleeping when it is within its sched_slice. - */ - if (task_sleep && se) - set_next_buddy(se); - break; - } flags |= DEQUEUE_SLEEP; flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); } +} - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); +/* + * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. + * + * Returns: + * true - dequeued + * false - delayed + */ +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) +{ + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + bool was_sched_idle = sched_idle_rq(rq); + bool task_sleep = flags & DEQUEUE_SLEEP; + bool task_delayed = flags & DEQUEUE_DELAYED; - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + clear_buddies(cfs_rq, se); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + update_curr(cfs_rq_of(se)); + update_entity_lag(cfs_rq, se); - cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; - cfs_rq->h_nr_idle -= h_nr_idle; + if (flags & DEQUEUE_DELAYED) { + WARN_ON_ONCE(!se->sched_delayed); + } else { + bool delay = task_sleep; + /* + * DELAY_DEQUEUE relies on spurious wakeups, special task + * states must not suffer spurious wakeups, excempt them. + */ + if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) + delay = false; - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + WARN_ON_ONCE(delay && se->sched_delayed); - if (throttled_hierarchy(cfs_rq) && task_throttled) - record_throttle_clock(cfs_rq); + if (sched_feat(DELAY_DEQUEUE) && delay && + !entity_eligible(cfs_rq, se)) { + update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); + set_delayed(se); + return false; + } + } + + dequeue_hierarchy(p, flags); + + if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { + se->deadline -= se->vruntime; + se->rel_deadline = 1; } + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); - sub_nr_running(rq, h_nr_queued); + sub_nr_running(rq, 1); /* balance early to pull high priority tasks */ if (unlikely(!was_sched_idle && sched_idle_rq(rq))) rq->next_balance = jiffies; - if (p && task_delayed) { + if (task_delayed) { + clear_delayed(se); + WARN_ON_ONCE(!task_sleep); WARN_ON_ONCE(p->on_rq != 1); @@ -8025,7 +8156,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) __block_task(rq, p); } - return 1; + return true; } /* @@ -8043,11 +8174,11 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - if (dequeue_entities(rq, &p->se, flags) < 0) + if (!__dequeue_task(rq, p, flags)) return false; /* - * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). + * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). */ return true; } @@ -9642,19 +9773,6 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) static void task_dead_fair(struct task_struct *p) { struct sched_entity *se = &p->se; - - if (se->sched_delayed) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - task_rq_unlock(rq, p, &rf); - } - remove_entity_load_avg(se); } @@ -9688,27 +9806,14 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context set_task_max_allowed_capacity(p); } -static void set_next_buddy(struct sched_entity *se) -{ - for_each_sched_entity(se) { - if (WARN_ON_ONCE(!se->on_rq)) - return; - if (se_is_idle(se)) - return; - cfs_rq_of(se)->next = se; - } -} - enum preempt_wakeup_action { PREEMPT_WAKEUP_NONE, /* No preemption. */ PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ - PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ + PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ }; -static inline bool -set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, - struct sched_entity *pse, struct sched_entity *se) +static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) { /* * Keep existing buddy if the deadline is sooner than pse. @@ -9719,7 +9824,16 @@ set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, if (cfs_rq->next && entity_before(cfs_rq->next, pse)) return false; - set_next_buddy(pse); + set_next_buddy(cfs_rq, pse); + return true; +} + +static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) +{ + if (cfs_rq->next && cfs_rq->next->slice < pse->slice) + return false; + + set_next_buddy(cfs_rq, pse); return true; } @@ -9772,13 +9886,14 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; struct sched_entity *nse, *se = &donor->se, *pse = &p->se; - struct cfs_rq *cfs_rq = task_cfs_rq(donor); + struct cfs_rq *cfs_rq = &rq->cfs; int cse_is_idle, pse_is_idle; /* * XXX Getting preempted by higher class, try and find idle CPU? */ - if (p->sched_class != &fair_sched_class) + if (p->sched_class != &fair_sched_class || + donor->sched_class != &fair_sched_class) return; if (unlikely(se == pse)) @@ -9803,42 +9918,43 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f * prevents us from potentially nominating it as a false LAST_BUDDY * below. */ - if (test_tsk_need_resched(rq->curr)) + if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr)) return; if (!sched_feat(WAKEUP_PREEMPTION)) return; - find_matching_se(&se, &pse); WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); pse_is_idle = se_is_idle(pse); + nse = se; /* * Preempt an idle entity in favor of a non-idle entity (and don't preempt * in the inverse case). */ - if (cse_is_idle && !pse_is_idle) { - /* - * When non-idle entity preempt an idle entity, - * don't give idle entity slice protection. - */ - preempt_action = PREEMPT_WAKEUP_SHORT; + if (cse_is_idle && !pse_is_idle) goto preempt; - } + + update_curr_fair(rq); if (cse_is_idle != pse_is_idle) - return; + goto update; /* * BATCH and IDLE tasks do not preempt others. */ if (unlikely(!normal_policy(p->policy))) - return; + goto update; + + /* + * Do not preempt for tasks that are sched_delayed as it would violate + * EEVDF to forcibly queue an ineligible task. + */ + if (pse->sched_delayed) + goto update; - cfs_rq = cfs_rq_of(se); - update_curr(cfs_rq); /* * If @p has a shorter slice than current and @p is eligible, override * current's slice protection in order to allow preemption. @@ -9850,17 +9966,13 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f /* * Ignore wakee preemption on WF_FORK as it is less likely that - * there is shared data as exec often follow fork. Do not - * preempt for tasks that are sched_delayed as it would violate - * EEVDF to forcibly queue an ineligible task. + * there is shared data as exec often follow fork. */ - if ((wake_flags & WF_FORK) || pse->sched_delayed) - return; + if (wake_flags & WF_FORK) + goto update; /* Prefer picking wakee soon if appropriate. */ - if (sched_feat(NEXT_BUDDY) && - set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { - + if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) { /* * Decide whether to obey WF_SYNC hint for a new buddy. Old * buddies are ignored as they may not be relevant to the @@ -9882,29 +9994,33 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f } pick: - nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); - /* If @p has become the most eligible task, force preemption */ - if (nse == pse) - goto preempt; + if (cfs_rq->h_nr_queued) { + nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); + if (unlikely(!nse)) + goto pick; + + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; + } /* - * Because p is enqueued, nse being null can only mean that we - * dequeued a delayed task. If there are still entities queued in - * cfs, check if the next one will be p. + * If @p is eligible but not the next task to run then cancel protection + * to prevent large scheduling latency */ - if (!nse && cfs_rq->nr_queued) - goto pick; - + if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse)) + goto preempt; +update: if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); return; preempt: - if (preempt_action == PREEMPT_WAKEUP_SHORT) { - cancel_protect_slice(se); - clear_buddies(cfs_rq, se); - } + cancel_protect_slice(se); + + if (preempt_action == PREEMPT_WAKEUP_SHORT) + set_short_buddy(cfs_rq, pse); resched_curr_lazy(rq); } @@ -9912,33 +10028,24 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; - struct cfs_rq *cfs_rq; struct task_struct *p; - bool throttled; int new_tasks; again: - cfs_rq = &rq->cfs; - if (!cfs_rq->nr_queued) + if (!cfs_rq->h_nr_queued) goto idle; - throttled = false; - - do { - /* Might not have done put_prev_entity() */ - if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + /* Might not have done put_prev_entity() */ + if (cfs_rq->curr && cfs_rq->curr->on_rq) + update_curr(cfs_rq); - se = pick_next_entity(rq, cfs_rq, true); - if (!se) - goto again; - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + se = pick_next_entity(rq, true); + if (!se) + goto again; p = task_of(se); - if (unlikely(throttled)) - task_throttle_setup_work(p); return p; idle: @@ -9975,7 +10082,7 @@ void fair_server_init(struct rq *rq) static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *nse = NULL; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -9985,7 +10092,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t while (se) { cfs_rq = cfs_rq_of(se); - if (!nse || cfs_rq->curr) + if (!nse || cfs_rq->h_curr) put_prev_entity(cfs_rq, se); #ifdef CONFIG_FAIR_GROUP_SCHED if (nse) { @@ -10004,6 +10111,14 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t #endif se = parent_entity(se); } + + /* Put 'current' back into the tree. */ + cfs_rq = &rq->cfs; + se = &prev->se; + WARN_ON_ONCE(cfs_rq->curr != se); + cfs_rq->curr = NULL; + if (se->on_rq) + __enqueue_entity(cfs_rq, se); } /* @@ -10012,8 +10127,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t static void yield_task_fair(struct rq *rq) { struct task_struct *curr = rq->donor; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq = &rq->cfs; /* * Are we the only task in the tree? @@ -10054,11 +10169,11 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) struct sched_entity *se = &p->se; /* !se->on_rq also covers throttled task */ - if (!se->on_rq) + if (!se->on_rq || se->sched_delayed) return false; /* Tell the scheduler that we'd really like se to run next. */ - set_next_buddy(se); + set_next_buddy(&task_rq(p)->cfs, se); yield_task_fair(rq); @@ -10397,15 +10512,10 @@ static inline long migrate_degrades_locality(struct task_struct *p, */ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) { - struct cfs_rq *dst_cfs_rq; + struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#ifdef CONFIG_FAIR_GROUP_SCHED - dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); -#else - dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#endif - if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && - !entity_eligible(task_cfs_rq(p), &p->se)) + if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && + !entity_eligible(&task_rq(p)->cfs, &p->se)) return 1; return 0; @@ -11188,7 +11298,7 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { load = cfs_rq->h_load; load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq_load_avg(cfs_rq) + 1); cfs_rq = group_cfs_rq(se); cfs_rq->h_load = load; cfs_rq->last_h_load_update = now; @@ -13437,75 +13547,78 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, } } - if (!ld_moved) { - schedstat_inc(sd->lb_failed[idle]); - /* - * Increment the failure counter only on periodic balance. - * We do not want newidle balance, which can be very - * frequent, pollute the failure counter causing - * excessive cache_hot migrations and active balances. - * - * Similarly for migration_misfit which is not related to - * load/util migration, don't pollute nr_balance_failed. - * - * The same for cache aware scheduling's allowance for - * load imbalance. If regular load balance does not - * migrate task due to LLC locality, it is a expected - * behavior and don't pollute nr_balance_failed. - * See can_migrate_task(). - */ - if (idle != CPU_NEWLY_IDLE && - env.migration_type != migrate_misfit && - !(env.flags & LBF_LLC_PINNED)) - sd->nr_balance_failed++; + if (ld_moved) { + sd->nr_balance_failed = 0; + goto out_unbalanced; + } - if (need_active_balance(&env)) { - unsigned long flags; + schedstat_inc(sd->lb_failed[idle]); + /* + * Increment the failure counter only on periodic balance. + * We do not want newidle balance, which can be very + * frequent, pollute the failure counter causing + * excessive cache_hot migrations and active balances. + * + * Similarly for migration_misfit which is not related to + * load/util migration, don't pollute nr_balance_failed. + * + * The same for cache aware scheduling's allowance for + * load imbalance. If regular load balance does not + * migrate task due to LLC locality, it is a expected + * behavior and don't pollute nr_balance_failed. + * See can_migrate_task(). + */ + if (idle != CPU_NEWLY_IDLE && + env.migration_type != migrate_misfit && + !(env.flags & LBF_LLC_PINNED)) + sd->nr_balance_failed++; - raw_spin_rq_lock_irqsave(busiest, flags); + if (!need_active_balance(&env)) + goto out_unbalanced; - /* - * Don't kick the active_load_balance_cpu_stop, - * if the curr task on busiest CPU can't be - * moved to this_cpu: - */ - if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { - raw_spin_rq_unlock_irqrestore(busiest, flags); - goto out_one_pinned; - } + scoped_guard (raw_spin_rq_lock_irqsave, busiest) { + /* + * Don't kick the active_load_balance_cpu_stop, + * if the curr task on busiest CPU can't be + * moved to this_cpu: + */ + if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) + goto out_one_pinned; - /* Record that we found at least one task that could run on this_cpu */ - env.flags &= ~LBF_ALL_PINNED; + /* Record that we found at least one task that could run on this_cpu */ + env.flags &= ~LBF_ALL_PINNED; - /* - * ->active_balance synchronizes accesses to - * ->active_balance_work. Once set, it's cleared - * only after active load balance is finished. - */ - if (!busiest->active_balance) { - busiest->active_balance = 1; - busiest->push_cpu = this_cpu; - active_balance = 1; - } + /* + * ->active_balance synchronizes accesses to + * ->active_balance_work. Once set, it's cleared + * only after active load balance is finished. + */ + if (busiest->active_balance) + goto out_unbalanced; - preempt_disable(); - raw_spin_rq_unlock_irqrestore(busiest, flags); - if (active_balance) { - stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, - &busiest->active_balance_work); - } - preempt_enable(); - } - } else { - sd->nr_balance_failed = 0; - } + /* + * @busiest dropped its rq_lock in the middle of + * scheduling out its ->curr task (->on_rq := 0), no + * need to forcefully punt it away with active balance. + */ + if (!busiest->curr->on_rq) + goto out_unbalanced; - if (likely(!active_balance) || need_active_balance(&env)) { - /* We were unbalanced, so reset the balancing interval */ - sd->balance_interval = sd->min_interval; + busiest->active_balance = 1; + busiest->push_cpu = this_cpu; + active_balance = 1; + preempt_disable(); + } + if (active_balance) { + stop_one_cpu_nowait(cpu_of(busiest), + active_load_balance_cpu_stop, busiest, + &busiest->active_balance_work); } + preempt_enable(); +out_unbalanced: + /* We were unbalanced, so reset the balancing interval */ + sd->balance_interval = sd->min_interval; goto out; out_balanced: @@ -14571,7 +14684,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * if the sibling is forced idle, then trigger schedule to * give forced idle task a chance. * - * sched_slice() considers only this active rq and it gets the + * __entity_slice_used() considers only this active rq and it gets the * whole slice. But during force idle, we have siblings acting * like a single runqueue and hence we need to consider runnable * tasks on this CPU and the forced idle CPU. Ideally, we should @@ -14580,7 +14693,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && + if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } @@ -14789,30 +14902,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, WARN_ON_ONCE(task_rq(b)->core != rq->core); -#ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Find an se in the hierarchy for tasks a and b, such that the se's - * are immediate siblings. - */ - while (sea->cfs_rq->tg != seb->cfs_rq->tg) { - int sea_depth = sea->depth; - int seb_depth = seb->depth; - - if (sea_depth >= seb_depth) - sea = parent_entity(sea); - if (sea_depth <= seb_depth) - seb = parent_entity(seb); - } - - se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); - se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); - - cfs_rqa = sea->cfs_rq; - cfs_rqb = seb->cfs_rq; -#else /* !CONFIG_FAIR_GROUP_SCHED: */ cfs_rqa = &task_rq(a)->cfs; cfs_rqb = &task_rq(b)->cfs; -#endif /* !CONFIG_FAIR_GROUP_SCHED */ /* * Find delta after normalizing se's vruntime with its cfs_rq's @@ -14851,11 +14942,20 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct sched_entity *se = &curr->se; - struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); + if (se->on_rq) { + unsigned long weight = NICE_0_LOAD; + struct cfs_rq *cfs_rq; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); + + weight = __calc_prop_weight(cfs_rq, se, weight); + } + + se = &curr->se; + reweight_eevdf(cfs_rq, se, weight, se->on_rq); } if (queued) @@ -14895,7 +14995,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) if (p->prio == oldprio) return; - if (rq->cfs.nr_queued == 1) + if (rq->cfs.h_nr_queued == 1) return; /* @@ -15024,33 +15124,44 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -/* - * Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; bool throttled = false; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight = NICE_0_LOAD; + bool on_rq = se->on_rq; + + clear_buddies(cfs_rq, se); + + if (on_rq) + __dequeue_entity(cfs_rq, se); for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + cfs_rq = cfs_rq_of(se); - if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && - first && cfs_rq->curr) - break; + if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || + !first || !cfs_rq->h_curr) + set_next_entity(cfs_rq, se); - set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ throttled |= account_cfs_rq_runtime(cfs_rq, 0); + + if (on_rq) + weight = __calc_prop_weight(cfs_rq, se, weight); } if (throttled) task_throttle_setup_work(p); se = &p->se; + cfs_rq->curr = se; + + if (on_rq) { + reweight_eevdf(cfs_rq, se, weight, se->on_rq); + if (first) + set_protect_slice(cfs_rq, se); + } if (task_on_rq_queued(p)) { /* @@ -15163,17 +15274,8 @@ void unregister_fair_sched_group(struct task_group *tg) struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); - if (se) { - if (se->sched_delayed) { - guard(rq_lock_irqsave)(rq); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - list_del_leaf_cfs_rq(cfs_rq); - } + if (se) remove_entity_load_avg(se); - } /* * Only empty task groups can be destroyed; so we can speculatively diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 897790889ba3..779eb58a4261 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -206,7 +206,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa, /* * running is a subset of runnable (weight) so running can't be set if * runnable is clear. But there are some corner cases where the current - * se has been already dequeued but cfs_rq->curr still points to it. + * se has been already dequeued but cfs_rq->h_curr still points to it. * This means that weight will be 0 but not running for a sched_entity * but also for a cfs_rq if the latter becomes idle. As an example, * this happens during sched_balance_newidle() which calls @@ -307,7 +307,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se), - cfs_rq->curr == se)) { + cfs_rq->h_curr == se)) { ___update_load_avg(&se->avg, se_weight(se)); cfs_se_util_change(&se->avg); @@ -323,7 +323,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) if (___update_load_sum(now, &cfs_rq->avg, scale_load_down(cfs_rq->load.weight), cfs_rq->h_nr_runnable, - cfs_rq->curr != NULL)) { + cfs_rq->h_curr != NULL)) { ___update_load_avg(&cfs_rq->avg, 1); trace_pelt_cfs_tp(cfs_rq); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index d9c9d9480a45..848955f8893d 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1023,7 +1023,7 @@ void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct task_st irq = irq_time_read(cpu); delta = (s64)(irq - rq->psi_irq_time); - if (delta < 0) + if (delta <= 0) return; rq->psi_irq_time = irq; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e474c31d8fe6..e6e5f8a2caaf 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) /* * XXX If we're preempted by DL, queue a push? */ - if (p->sched_class != &rt_sched_class) + if (p->sched_class != &rt_sched_class || + donor->sched_class != &rt_sched_class) return; if (p->prio < donor->prio) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c7c2dea65edd..26ae13c86b69 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,8 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -528,21 +530,8 @@ struct task_group { }; -#ifdef CONFIG_GROUP_SCHED_WEIGHT #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD -/* - * A weight of 0 or 1 can cause arithmetics problems. - * A weight of a cfs_rq is the sum of weights of which entities - * are queued on this cfs_rq, so a weight of a entity should not be - * too large, so as the shares value of a task group. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ -#define MIN_SHARES (1UL << 1) -#define MAX_SHARES (1UL << 18) -#endif - typedef int (*tg_visitor)(struct task_group *, void *); extern int walk_tg_tree_from(struct task_group *from, @@ -571,6 +560,7 @@ extern void free_fair_sched_group(struct task_group *tg); extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); +extern void __sched_cgroup_mode_update(int mode); #else /* !CONFIG_FAIR_GROUP_SCHED: */ static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -628,6 +618,17 @@ static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false #endif /* !CONFIG_CGROUP_SCHED */ +/* + * A weight of 0 or 1 can cause arithmetics problems. + * A weight of a cfs_rq is the sum of weights of which entities + * are queued on this cfs_rq, so a weight of a entity should not be + * too large, so as the shares value of a task group. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ +#define MIN_SHARES (1UL << 1) +#define MAX_SHARES (1UL << 18) + extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); @@ -706,6 +707,7 @@ struct cfs_rq { /* * CFS load tracking */ + struct sched_entity *h_curr; struct sched_avg avg; #ifndef CONFIG_64BIT u64 last_update_time_copy; @@ -721,6 +723,7 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum; @@ -2018,7 +2021,8 @@ DEFINE_LOCK_GUARD_1(rq_lock, struct rq, rq_unlock(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock, _T) DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, @@ -2026,7 +2030,8 @@ DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, rq_unlock_irq(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irq, _T) DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, @@ -2034,9 +2039,20 @@ DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, rq_unlock_irqrestore(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, _T) +DEFINE_LOCK_GUARD_1(raw_spin_rq_lock_irqsave, struct rq, + raw_spin_rq_lock_irqsave(_T->lock, _T->flags), + raw_spin_rq_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + +DECLARE_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); +#define class_raw_spin_rq_lock_irqsave_constructor(_T) \ + WITH_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, _T) + #define this_rq_lock_irq(...) __acquire_ret(_this_rq_lock_irq(__VA_ARGS__), __rq_lockp(__ret)) static inline struct rq *_this_rq_lock_irq(struct rq_flags *rf) __acquires_ret { @@ -2571,6 +2587,7 @@ extern const u32 sched_prio_to_wmult[40]; #define ENQUEUE_MIGRATED 0x00040000 #define ENQUEUE_INITIAL 0x00080000 #define ENQUEUE_RQ_SELECTED 0x00100000 +#define ENQUEUE_QUEUED 0x00200000 #define RETRY_TASK ((void *)-1UL) @@ -4211,6 +4228,6 @@ DEFINE_CLASS(sched_change, struct sched_change_ctx *, DEFINE_CLASS_IS_UNCONDITIONAL(sched_change) -#include "ext.h" +#include "ext/ext.h" #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/signal.c b/kernel/signal.c index 9c2b32c4d755..b12878f3c578 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -130,28 +130,10 @@ static bool sig_ignored(struct task_struct *t, int sig, bool force) */ static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked) { - unsigned long ready; - long i; - - switch (_NSIG_WORDS) { - default: - for (i = _NSIG_WORDS, ready = 0; --i >= 0 ;) - ready |= signal->sig[i] &~ blocked->sig[i]; - break; - - case 4: ready = signal->sig[3] &~ blocked->sig[3]; - ready |= signal->sig[2] &~ blocked->sig[2]; - ready |= signal->sig[1] &~ blocked->sig[1]; - ready |= signal->sig[0] &~ blocked->sig[0]; - break; - - case 2: ready = signal->sig[1] &~ blocked->sig[1]; - ready |= signal->sig[0] &~ blocked->sig[0]; - break; - - case 1: ready = signal->sig[0] &~ blocked->sig[0]; - } - return ready != 0; + unsigned long ready = 0; + for (long i = 0; i < _NSIG_WORDS; i++) + ready |= signal->sig[i] & ~blocked->sig[i]; + return ready != 0; } #define PENDING(p,b) has_pending_signals(&(p)->signal, (b)) @@ -1181,6 +1163,7 @@ static inline bool has_si_pid_and_uid(struct kernel_siginfo *info) int send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type) { + struct kernel_siginfo __maybe_unused rewritten; /* Should SIGKILL or SIGSTOP be received by a pid namespace init? */ bool force = false; @@ -1192,24 +1175,34 @@ int send_signal_locked(int sig, struct kernel_siginfo *info, force = true; } else if (has_si_pid_and_uid(info)) { /* SIGKILL and SIGSTOP is special or has ids */ +#ifdef CONFIG_USER_NS struct user_namespace *t_user_ns; + kuid_t uid; rcu_read_lock(); t_user_ns = task_cred_xxx(t, user_ns); if (current_user_ns() != t_user_ns) { - kuid_t uid = make_kuid(current_user_ns(), info->si_uid); - info->si_uid = from_kuid_munged(t_user_ns, uid); + rewritten = *info; + info = &rewritten; + uid = make_kuid(current_user_ns(), info->si_uid); + rewritten.si_uid = from_kuid_munged(t_user_ns, uid); } rcu_read_unlock(); - +#endif /* A kernel generated signal? */ force = (info->si_code == SI_KERNEL); +#ifdef CONFIG_PID_NS /* From an ancestor pid namespace? */ if (!task_pid_nr_ns(current, task_active_pid_ns(t))) { - info->si_pid = 0; + if (info != &rewritten) { + rewritten = *info; + info = &rewritten; + } + rewritten.si_pid = 0; force = true; } +#endif } return __send_signal_locked(sig, info, t, type, force); } @@ -3950,11 +3943,7 @@ static void prepare_kill_siginfo(int sig, struct kernel_siginfo *info, */ SYSCALL_DEFINE2(kill, pid_t, pid, int, sig) { - struct kernel_siginfo info; - - prepare_kill_siginfo(sig, &info, PIDTYPE_TGID); - - return kill_something_info(sig, &info, pid); + return kill_something_info(sig, SEND_SIG_NOINFO, pid); } /* diff --git a/kernel/smp.c b/kernel/smp.c index a0bb56bd8dda..8a847a34f132 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -182,16 +182,22 @@ static atomic_t csd_bug_count = ATOMIC_INIT(0); static void __csd_lock_record(call_single_data_t *csd) { if (!csd) { - smp_mb(); /* NULL cur_csd after unlock. */ - __this_cpu_write(cur_csd, NULL); + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): orders any preceding CSD + * callback/unlock before a remote reader observes NULL. + */ + smp_store_release(this_cpu_ptr(&cur_csd), NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); - smp_wmb(); /* func and info before csd. */ - __this_cpu_write(cur_csd, csd); - smp_mb(); /* Update cur_csd before function call. */ - /* Or before unlock, as the case may be. */ + /* + * Pairs with smp_load_acquire() of cur_csd in + * csd_lock_wait_toolong(): publishes cur_csd_func and + * cur_csd_info before the non-NULL pointer becomes visible. + */ + smp_store_release(this_cpu_ptr(&cur_csd), csd); } static __always_inline void csd_lock_record(call_single_data_t *csd) @@ -272,7 +278,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in cpux = 0; else cpux = cpu; - cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ + /* + * Pairs with smp_store_release() of cur_csd in __csd_lock_record(): + * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info + * are the matching publication; a NULL value is ordered after any + * preceding CSD callback/unlock on the remote CPU. + */ + cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* How long since this CSD lock was stuck. */ ts_delta = ts2 - ts0; pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n", diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..a3bd54de8259 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -576,6 +576,10 @@ static inline bool lockdep_softirq_start(void) { return false; } static inline void lockdep_softirq_end(bool in_hardirq) { } #endif +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +static noinline void handle_softirqs(bool ksirqd); +#endif + static void handle_softirqs(bool ksirqd) { unsigned long end = jiffies + MAX_SOFTIRQ_TIME; diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 73bd6a6a7893..2cd0172d0516 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -210,13 +210,39 @@ static int fill_stats_for_pid(pid_t pid, struct taskstats *stats) return 0; } +static void tgid_stats_add_task(struct taskstats *stats, + struct task_struct *tsk, u64 now_ns) +{ + u64 delta, utime, stime; + + /* + * Each accounting subsystem calls its functions here to + * accumulate its per-task stats for tsk, into the per-tgid structure + * + * per-task-foo(stats, tsk); + */ + delayacct_add_tsk(stats, tsk); + + /* calculate task elapsed time in nsec */ + delta = now_ns - tsk->start_time; + /* Convert to micro seconds */ + do_div(delta, NSEC_PER_USEC); + stats->ac_etime += delta; + + task_cputime(tsk, &utime, &stime); + stats->ac_utime += div_u64(utime, NSEC_PER_USEC); + stats->ac_stime += div_u64(stime, NSEC_PER_USEC); + + stats->nvcsw += tsk->nvcsw; + stats->nivcsw += tsk->nivcsw; +} + static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) { struct task_struct *tsk, *first; unsigned long flags; int rc = -ESRCH; - u64 delta, utime, stime; - u64 start_time; + u64 now_ns; /* * Add additional stats from live tasks except zombie thread group @@ -233,30 +259,12 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) else memset(stats, 0, sizeof(*stats)); - start_time = ktime_get_ns(); + now_ns = ktime_get_ns(); for_each_thread(first, tsk) { if (tsk->exit_state) continue; - /* - * Accounting subsystem can call its functions here to - * fill in relevant parts of struct taskstsats as follows - * - * per-task-foo(stats, tsk); - */ - delayacct_add_tsk(stats, tsk); - - /* calculate task elapsed time in nsec */ - delta = start_time - tsk->start_time; - /* Convert to micro seconds */ - do_div(delta, NSEC_PER_USEC); - stats->ac_etime += delta; - task_cputime(tsk, &utime, &stime); - stats->ac_utime += div_u64(utime, NSEC_PER_USEC); - stats->ac_stime += div_u64(stime, NSEC_PER_USEC); - - stats->nvcsw += tsk->nvcsw; - stats->nivcsw += tsk->nivcsw; + tgid_stats_add_task(stats, tsk, now_ns); } unlock_task_sighand(first, &flags); @@ -275,18 +283,14 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) static void fill_tgid_exit(struct task_struct *tsk) { unsigned long flags; + u64 now_ns; spin_lock_irqsave(&tsk->sighand->siglock, flags); if (!tsk->signal->stats) goto ret; - /* - * Each accounting subsystem calls its functions here to - * accumalate its per-task stats for tsk, into the per-tgid structure - * - * per-task-foo(tsk->signal->stats, tsk); - */ - delayacct_add_tsk(tsk->signal->stats, tsk); + now_ns = ktime_get_ns(); + tgid_stats_add_task(tsk->signal->stats, tsk, now_ns); ret: spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 638ce623c342..2a278ae550a1 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -799,7 +799,7 @@ static inline void hrtimer_switch_to_hres(void) { } * * This is only invoked when: * - CONFIG_HIGH_RES_TIMERS is enabled. - * - CONFIG_NOHZ_COMMON is enabled + * - CONFIG_NO_HZ_COMMON is enabled * * For the other cases this function is empty and because the call sites * are optimized out it vanishes as well, i.e. no need for lots of @@ -2060,13 +2060,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc base->running = NULL; } -static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base) -{ - struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); - - return next ? hrtimer_from_timerqueue_node(next) : NULL; -} - static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, unsigned long flags, unsigned int active_mask) { diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 74775b94d11b..5e633d8750d1 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -41,7 +41,7 @@ void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) */ int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) { - u64 nsecs = rlim_new * NSEC_PER_SEC; + u64 nsecs = (u64)rlim_new * NSEC_PER_SEC; unsigned long irq_fl; if (!lock_task_sighand(task, &irq_fl)) diff --git a/kernel/time/time.c b/kernel/time/time.c index 771cef87ad3b..0dd63a91e7c5 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -251,7 +251,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; - if (new_ts.tv_nsec > USEC_PER_SEC || new_ts.tv_nsec < 0) + if (new_ts.tv_nsec >= USEC_PER_SEC || new_ts.tv_nsec < 0) return -EINVAL; new_ts.tv_nsec *= NSEC_PER_USEC; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 0d5b67f609bb..b1b5ec43c0f2 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2061,13 +2061,14 @@ void __init timekeeping_init(void) */ wall_to_mono = timespec64_sub(boot_offset, wall_time); + clock = clocksource_default_clock(); + if (clock->enable) + clock->enable(clock); + guard(raw_spinlock_irqsave)(&tk_core.lock); ntp_init(); - clock = clocksource_default_clock(); - if (clock->enable) - clock->enable(clock); tk_setup_internals(tks, clock); tk_set_xtime(tks, &wall_time); diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 084f34dc6c9f..0ab5916575a9 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -779,6 +779,18 @@ config PROBE_EVENTS_BTF_ARGS kernel function entry or a tracepoint. This is available only if BTF (BPF Type Format) support is enabled. +config PROBE_EVENTS_DUMP_FETCHARG + bool "Dump of dynamic probe event fetch-arguments" + depends on PROBE_EVENTS + default n + help + This shows the dump of fetch-arguments of dynamic probe events + alongside their event definitions in the dynamic_events file + as comment lines. This is useful to debug the probe events. + Since this exposes the raw values in the dynamic_events file, + it might be a security risk. Only enable it if you need to debug + probe events themselves. + config KPROBE_EVENTS depends on KPROBES depends on HAVE_REGS_AND_STACK_ACCESS_API diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 82f8feea6931..76ab51deaa6b 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2376,9 +2376,12 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 int err = -ENOMEM; unsigned int i; + if (!access_ok(usyms, cnt * sizeof(*usyms))) + return -EFAULT; + syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); if (!syms) - goto error; + return -ENOMEM; buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); if (!buf) @@ -2403,10 +2406,8 @@ static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 return 0; error: - if (err) { - kvfree(syms); - kvfree(buf); - } + kvfree(syms); + kvfree(buf); return err; } @@ -3699,6 +3700,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); + u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); + u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); + struct bpf_tracing_multi_link *tr_link; + u32 ucount = info->tracing_multi.count; + bool has_cookies, show_addrs; + int err = 0; + + if ((uids || ucookies || uaddrs) && !ucount) + return -EINVAL; + + tr_link = container_of(link, struct bpf_tracing_multi_link, link); + + info->tracing_multi.attach_type = tr_link->link.attach_type; + info->tracing_multi.count = tr_link->nodes_cnt; + info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); + + if (!uids && !ucookies && !uaddrs) + return 0; + + if (ucount < tr_link->nodes_cnt) + err = -ENOSPC; + else + ucount = tr_link->nodes_cnt; + + has_cookies = !!tr_link->cookies; + show_addrs = kallsyms_show_value(current_cred()); + + for (int i = 0; i < ucount; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u64 addr, cookie; + u32 id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); + + addr = show_addrs ? mnode->trampoline->ip : 0; + cookie = has_cookies ? tr_link->cookies[i] : 0; + + if (uids && put_user(id, uids + i)) + return -EFAULT; + if (uaddrs && put_user(addr, uaddrs + i)) + return -EFAULT; + if (ucookies && put_user(cookie, ucookies + i)) + return -EFAULT; + + cond_resched(); + } + + return err; +} + #ifdef CONFIG_PROC_FS static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) @@ -3729,6 +3784,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, + .fill_link_info = bpf_tracing_multi_link_fill_link_info, #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_tracing_multi_show_fdinfo, #endif diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f378613ad120..f215990b9061 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -613,6 +613,16 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops continue; data_size = fp->entry_data_size; + /* + * The list may have grown since it was sized, so this node + * may not fit. Skip it as missed rather than overrun the + * reservation. + */ + if (fp->exit_handler && + used + FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(data_size) > reserved_words) { + fp->nmissed++; + continue; + } if (data_size && fp->exit_handler) data = fgraph_data + used + FPROBE_HEADER_SIZE_IN_LONG; else diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 1146b83b711a..c9e182d40059 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4320,14 +4320,16 @@ static const char readme_msg[] = "\t args: =fetcharg[:type]\n" "\t fetcharg: (%|$), @
, @[+|-],\n" #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API - "\t $stack, $stack, $retval, $comm, $arg,\n" + "\t $stack, $stack, $retval, $comm, $arg, $current\n" #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS - "\t [->field[->field|.field...]],\n" + "\t [(structname[,field])][->field[->field|.field...]],\n" + "\t [(structname[,field])](fetcharg)->field[->field|.field...],\n" #endif #else - "\t $stack, $stack, $retval, $comm,\n" + "\t $stack, $stack, $retval, $comm, $current\n" #endif "\t +|-[u](), \\imm-value, \\\"imm-string\"\n" + "\t this_cpu_read(), this_cpu_ptr()\n" "\t kernel return probes support: $retval, $arg, $comm\n" "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n" "\t b@/, ustring,\n" diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index b66d6196338d..462c31145733 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -87,6 +87,8 @@ static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev) seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm); seq_putc(m, '\n'); + trace_probe_dump_args(m, &ep->tp); + return 0; } @@ -315,7 +317,7 @@ get_event_field(struct fetch_insn *code, void *rec) val = (unsigned long)addr; break; case FILTER_PTR_STRING: - val = (unsigned long)(*(char *)addr); + val = *(unsigned long *)addr; break; default: WARN_ON_ONCE(1); diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c index 9f5f08c0e7c2..536781cd4c47 100644 --- a/kernel/trace/trace_fprobe.c +++ b/kernel/trace/trace_fprobe.c @@ -764,7 +764,7 @@ static int unregister_fprobe_event(struct trace_fprobe *tf) return trace_probe_unregister_event_call(&tf->tp); } -static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf) +static int __register_tracepoint_fprobe(struct trace_fprobe *tf) { struct tracepoint_user *tuser __free(tuser_put) = NULL; struct module *mod __free(module_put) = NULL; @@ -836,7 +836,7 @@ static int __register_trace_fprobe(struct trace_fprobe *tf) tf->fp.flags &= ~FPROBE_FL_DISABLED; if (trace_fprobe_is_tracepoint(tf)) - return __regsiter_tracepoint_fprobe(tf); + return __register_tracepoint_fprobe(tf); /* TODO: handle filter, nofilter or symbol list */ return register_fprobe(&tf->fp, tf->symbol, NULL); @@ -1449,6 +1449,8 @@ static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev) seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm); seq_putc(m, '\n'); + trace_probe_dump_args(m, &tf->tp); + return 0; } diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index a8420e6abb56..cfa807d8e760 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1320,6 +1320,8 @@ static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); seq_putc(m, '\n'); + trace_probe_dump_args(m, &tk->tp); + return 0; } diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index fd1caa1f9723..18c212122344 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -342,13 +342,112 @@ static int parse_trace_event(char *arg, struct fetch_insn *code, ret = parse_trace_event_arg(arg, code, ctx); if (!ret) return 0; - if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { - code->op = FETCH_OP_COMM; - return 0; - } return -EINVAL; } +/* this_cpu_* parser */ +#define THIS_CPU_PTR_PREFIX "this_cpu_ptr(" +#define THIS_CPU_READ_PREFIX "this_cpu_read(" +#define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1) +#define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1) + +static int +parse_probe_arg(char *arg, const struct fetch_type *type, + struct fetch_insn **pcode, struct fetch_insn *end, + struct traceprobe_parse_context *ctx); + +/* handle dereference nested call */ +static inline int handle_dereference(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, struct traceprobe_parse_context *ctx, + int deref, long offset) +{ + const struct fetch_type *type = find_fetch_type(NULL, ctx->flags); + struct fetch_insn *code = *pcode; + int cur_offs = ctx->offset; + char *tmp; + int ret; + + tmp = strrchr(arg, ')'); + if (!tmp) { + trace_probe_log_err(ctx->offset + strlen(arg), + DEREF_OPEN_BRACE); + return -EINVAL; + } + + *tmp = '\0'; + ret = parse_probe_arg(arg, type, &code, end, ctx); + if (ret) + return ret; + ctx->offset = cur_offs; + if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) { + trace_probe_log_err(ctx->offset, COMM_CANT_DEREF); + return -EINVAL; + } + + /* + * this_cpu_ptr(@SYM) does not use SYM value, but use SYM address. + * So we overwrite the last FETCH_OP_DEREF with FETCH_OP_CPU_PTR. + */ + if (!(deref == FETCH_OP_CPU_PTR && *arg == '@')) { + code++; + if (code == end) { + trace_probe_log_err(ctx->offset, TOO_MANY_OPS); + return -EINVAL; + } + } + *pcode = code; + + code->op = deref; + code->offset = offset; + /* Reset the last type if used */ + ctx->last_type = NULL; + return 0; +} + +static int parse_this_cpu(char *arg, struct fetch_insn **pcode, + struct fetch_insn *end, + struct traceprobe_parse_context *ctx) +{ + struct fetch_insn *code; + bool is_ptr = false; + int ret; + + /* + * This is only for kernel probes, excluding eprobe, because per-cpu + * pointer should not be recorded by events. + */ + if (!(ctx->flags & TPARG_FL_KERNEL) || + (ctx->flags & TPARG_FL_TEVENT)) { + trace_probe_log_err(ctx->offset, NOSUP_PERCPU); + return -EINVAL; + } + if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) { + arg += THIS_CPU_PTR_LEN; + ctx->offset += THIS_CPU_PTR_LEN; + is_ptr = true; + } else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) { + arg += THIS_CPU_READ_LEN; + ctx->offset += THIS_CPU_READ_LEN; + } else + return -EINVAL; + + ret = handle_dereference(arg, pcode, end, ctx, FETCH_OP_CPU_PTR, 0); + if (ret || is_ptr) + return ret; + + /* this_cpu_read(VAR) -> +0(this_cpu_ptr(VAR)) */ + code = *pcode; + code++; + if (code == end) { + trace_probe_log_err(ctx->offset, TOO_MANY_OPS); + return -EINVAL; + } + code->op = FETCH_OP_DEREF; + code->offset = 0; + *pcode = code; + return 0; +} + #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS static u32 btf_type_int(const struct btf_type *t) @@ -360,9 +459,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type) { const struct btf_type *real_type; u32 intdata; - s32 tid; - real_type = btf_type_skip_modifiers(btf, type->type, &tid); + real_type = btf_type_skip_modifiers(btf, type->type, NULL); if (!real_type) return false; @@ -379,14 +477,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) const struct btf_type *real_type; const struct btf_array *array; u32 intdata; - s32 tid; if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY) return false; array = (const struct btf_array *)(type + 1); - real_type = btf_type_skip_modifiers(btf, array->type, &tid); + real_type = btf_type_skip_modifiers(btf, array->type, NULL); intdata = btf_type_int(real_type); return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) @@ -574,6 +671,64 @@ static int split_next_field(char *varname, char **next_field, return ret; } +/* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */ +static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype, + struct traceprobe_parse_context *ctx) +{ + const struct btf_type *type = *ptype; + const struct btf_member *field; + struct btf *btf = ctx_btf(ctx); + char *fieldname = *pfieldname; + int bitoffs = 0; + u32 anon_offs; + char *next; + int is_ptr; + + do { + next = NULL; + is_ptr = split_next_field(fieldname, &next, ctx); + if (is_ptr < 0) + return is_ptr; + + anon_offs = 0; + field = btf_find_struct_member(btf, type, fieldname, + &anon_offs); + if (IS_ERR(field)) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return PTR_ERR(field); + } + if (!field) { + trace_probe_log_err(ctx->offset, NO_BTF_FIELD); + return -ENOENT; + } + /* Add anonymous structure/union offset */ + bitoffs += anon_offs; + + /* Accumulate the bit-offsets of the dot-connected fields */ + if (btf_type_kflag(type)) { + bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); + ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); + } else { + bitoffs += field->offset; + ctx->last_bitsize = 0; + } + + type = btf_type_skip_modifiers(btf, field->type, NULL); + if (!type) { + trace_probe_log_err(ctx->offset, BAD_BTF_TID); + return -EINVAL; + } + + if (next) + ctx->offset += next - fieldname; + fieldname = next; + } while (!is_ptr && fieldname); + + *pfieldname = fieldname; + *ptype = type; + + return bitoffs; +} /* * Parse the field of data structure. The @type must be a pointer type * pointing the target data structure type. @@ -583,16 +738,13 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, struct traceprobe_parse_context *ctx) { struct fetch_insn *code = *pcode; - const struct btf_member *field; - u32 bitoffs, anon_offs; - bool is_struct = ctx->struct_btf != NULL; struct btf *btf = ctx_btf(ctx); - char *next; - int is_ptr; - s32 tid; + bool is_first_field = true; + int bitoffs; do { - if (!is_struct) { + /* For the first field of typecast, @type will be the target structure type. */ + if (!(is_first_field && ctx->struct_btf)) { /* Outer loop for solving arrow operator ('->') */ if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { trace_probe_log_err(ctx->offset, NO_PTR_STRCT); @@ -600,66 +752,31 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, } /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(btf, type->type, &tid); + type = btf_type_skip_modifiers(btf, type->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; } } - /* Only the first type can skip being a pointer */ - is_struct = false; - - bitoffs = 0; - do { - /* Inner loop for solving dot operator ('.') */ - next = NULL; - is_ptr = split_next_field(fieldname, &next, ctx); - if (is_ptr < 0) - return is_ptr; - - anon_offs = 0; - field = btf_find_struct_member(btf, type, fieldname, - &anon_offs); - if (IS_ERR(field)) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return PTR_ERR(field); - } - if (!field) { - trace_probe_log_err(ctx->offset, NO_BTF_FIELD); - return -ENOENT; - } - /* Add anonymous structure/union offset */ - bitoffs += anon_offs; - - /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(type)) { - bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); - ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); - } else { - bitoffs += field->offset; - ctx->last_bitsize = 0; - } - - type = btf_type_skip_modifiers(btf, field->type, &tid); - if (!type) { - trace_probe_log_err(ctx->offset, BAD_BTF_TID); - return -EINVAL; - } - - ctx->offset += next - fieldname; - fieldname = next; - } while (!is_ptr && fieldname); + bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx); + if (bitoffs < 0) + return bitoffs; if (++code == end) { trace_probe_log_err(ctx->offset, TOO_MANY_OPS); return -EINVAL; } code->op = FETCH_OP_DEREF; /* TODO: user deref support */ code->offset = bitoffs / 8; + if (is_first_field && ctx->struct_btf) { + /* The first field can be typecasted with field option. */ + code->offset -= ctx->prefix_byteoffs; + } *pcode = code; ctx->last_bitoffs = bitoffs % 8; ctx->last_type = type; + is_first_field = false; } while (fieldname); return 0; @@ -678,7 +795,9 @@ static int parse_btf_arg(char *varname, int i, is_ptr, ret; u32 tid; - if (WARN_ON_ONCE(!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))) + /* Note: field is not separated at this point, so check prefix. */ + if (!str_has_prefix(varname, "$current") && + !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT)) return -EINVAL; is_ptr = split_next_field(varname, &field, ctx); @@ -691,22 +810,23 @@ static int parse_btf_arg(char *varname, return -EOPNOTSUPP; } - if (ctx->flags & TPARG_FL_TEVENT) { - ret = parse_trace_event(varname, code, ctx); + if (!strcmp(varname, "$current")) { + code->op = FETCH_OP_CURRENT; + /* If no typecast is specified for $current, use task_struct by default */ + ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf); if (ret < 0) { - trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG); - return ret; + trace_probe_log_err(ctx->offset, NO_BTF_ENTRY); + return -ENOENT; } - /* TEVENT is only here via a typecast */ - if (WARN_ON_ONCE(ctx->struct_btf == NULL)) - return -EINVAL; - type = ctx->last_struct; + tid = (u32)ret; + type = ctx->last_struct = + btf_type_skip_modifiers(ctx->struct_btf, tid, NULL); goto found_type; } if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) { code->op = FETCH_OP_RETVAL; - /* Check whether the function return type is not void */ + /* Check whether the function return type is not void, even with typecast. */ if (query_btf_context(ctx) == 0) { if (ctx->proto->type == 0) { trace_probe_log_err(ctx->offset, NO_RETVAL); @@ -759,7 +879,7 @@ static int parse_btf_arg(char *varname, return -ENOENT; found: - type = btf_type_skip_modifiers(ctx->btf, tid, &tid); + type = btf_type_skip_modifiers(ctx->btf, tid, NULL); found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -829,17 +949,89 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context * return 0; } +static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx) +{ + char *field; + int ret; + + /* Field option - evaluated later. */ + field = strchr(casttype, ','); + if (field) + *field++ = '\0'; + + ret = query_btf_struct(casttype, ctx); + if (ret < 0) { + trace_probe_log_err(ctx->offset, NO_PTR_STRCT); + return -EINVAL; + } + + if (field) { + struct btf_type *type = (struct btf_type *)ctx->last_struct; + + ctx->offset += field - casttype; + ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx); + if (ret < 0) + return ret; + if (ret % 8) { + trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED); + return -EINVAL; + } + if (field != NULL) { + /* this means @field skips an arrow operator ("->"). */ + trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW); + return -EINVAL; + } + ctx->prefix_byteoffs = ret / 8; + /* Restore the original struct type (overwritten by get_bitoffset_of_field) */ + ctx->last_struct = type; + } + + return ret; +} + +/* Find the matching closing parenthesis for a given opening parenthesis. */ +static char *find_matched_close_paren(char *s) +{ + char *p = s; + int count = 0; + + while (*p) { + if (*p == '(') + count++; + else if (*p == ')') { + if (--count == 0) + return p; + } + p++; + } + return NULL; +} + static int handle_typecast(char *arg, struct fetch_insn **pcode, struct fetch_insn *end, struct traceprobe_parse_context *ctx) { + int orig_offset = ctx->offset; + char *close; char *tmp; int ret; - /* Currently this only works for eprobes */ - if (!(ctx->flags & TPARG_FL_TEVENT)) { - trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT); - return -EINVAL; + if (!(tparg_is_event_probe(ctx->flags) || + tparg_is_function_entry(ctx->flags) || + tparg_is_function_return(ctx->flags))) { + trace_probe_log_err(ctx->offset, NOSUP_BTFARG); + return -EOPNOTSUPP; + } + + /* + * Always consider the token after typecast as a nested call + * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same. + * VAR is solved in the nested call. + */ + ctx->nested_level++; + if (ctx->nested_level > TRACEPROBE_MAX_NESTED_LEVEL) { + trace_probe_log_err(ctx->offset, TOO_MANY_NESTED); + return -E2BIG; } tmp = strchr(arg, ')'); @@ -848,19 +1040,92 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode, DEREF_OPEN_BRACE); return -EINVAL; } - *tmp = '\0'; - ret = query_btf_struct(arg + 1, ctx); - *tmp = ')'; + *tmp++ = '\0'; - if (ret < 0) { - trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT); - return -EINVAL; + ctx->offset += tmp - arg; + if (*tmp == '(') { + close = find_matched_close_paren(tmp); + + if (!close) { + trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE); + return -EINVAL; + } + /* We expect a field access for typecast */ + if (close[1] != '-' || close[2] != '>') { + trace_probe_log_err(ctx->offset + close - tmp + 1, + TYPECAST_REQ_FIELD); + return -EINVAL; + } + /* Skip '(' */ + ctx->offset += 1; + tmp++; + } else if (*tmp == '+' || *tmp == '-' || + str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) || + str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) { + /* Dereference can have another field access inside it. */ + char *open = strchr(tmp + 1, '('); + + if (!open) { + trace_probe_log_err(ctx->offset, + DEREF_NEED_BRACE); + return -EINVAL; + } + close = find_matched_close_paren(open); + if (!close) { + trace_probe_log_err(ctx->offset + strlen(tmp), + DEREF_OPEN_BRACE); + return -EINVAL; + } + close++; + /* We expect a field access for typecast */ + if (close[0] != '-' || close[1] != '>') { + trace_probe_log_err(ctx->offset + close - tmp, + TYPECAST_REQ_FIELD); + return -EINVAL; + } + } else { + if (tmp[0] == '@') { + /* @sym+offset is not allowed without parenthesized */ + close = strpbrk(tmp, "+-"); + if (close && isdigit(close[1])) { + trace_probe_log_err(ctx->offset, + TYPECAST_SYM_OFFSET); + return -EINVAL; + } + } + /* Inner variable name */ + close = strchr(tmp, '-'); + if (!close || close[1] != '>') { + trace_probe_log_err(ctx->offset + strlen(tmp), + TYPECAST_REQ_FIELD); + return -EINVAL; + } } + *close = '\0'; - tmp++; + /* We need to parse the nested one */ + ret = parse_probe_arg(tmp, find_fetch_type(NULL, ctx->flags), + pcode, end, ctx); + if (ret < 0) + return ret; + ctx->nested_level--; + clear_struct_btf(ctx); - ctx->offset += tmp - arg; - ret = parse_btf_arg(tmp, pcode, end, ctx); + /* Let tmp point the field name. */ + if (close[1] == '-') + tmp = close + 3; /* Skip "->" after closing parenthesis */ + else + tmp = close + 2; /* Skip ">" after inner variable name */ + + /* resolve the typecast struct name */ + ctx->offset = orig_offset + 1; /* for the '(' */ + ret = parse_btf_casttype(arg + 1, ctx); + if (ret < 0) + return ret; + + ctx->offset = orig_offset + tmp - arg; + ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx); + ctx->prefix_byteoffs = 0; return ret; } @@ -1068,8 +1333,14 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, int len; if (ctx->flags & TPARG_FL_TEVENT) { - if (parse_trace_event(arg, code, ctx) < 0) + if (parse_trace_event(arg, code, ctx) < 0) { + /* 'comm' should be checked after field parsing. */ + if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { + code->op = FETCH_OP_COMM; + return 0; + } goto inval; + } return 0; } @@ -1116,6 +1387,24 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, return 0; } + /* $current returns the address of the current task_struct. */ + if (str_has_prefix(arg, "current")) { + /* $current is only supported by kernel probe. */ + if (!(ctx->flags & TPARG_FL_KERNEL)) { + err = TP_ERR_BAD_VAR; + goto inval; + } + arg += strlen("current"); + if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) + return parse_btf_arg(orig_arg, pcode, end, ctx); + + if (*arg != '\0') + goto inval; + + code->op = FETCH_OP_CURRENT; + return 0; + } + #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API len = str_has_prefix(arg, "arg"); if (len) { @@ -1228,7 +1517,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_IMM; code->immediate = param; } else if (arg[1] == '+') { - /* kprobes don't support file offsets */ + /* Kernel probes do not support file offsets */ if (ctx->flags & TPARG_FL_KERNEL) { trace_probe_log_err(ctx->offset, FILE_ON_KPROBE); return -EINVAL; @@ -1241,6 +1530,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type, code->op = FETCH_OP_FOFFS; code->immediate = (unsigned long)offset; // imm64? + offset = 0; } else { /* uprobes don't support symbols */ if (!(ctx->flags & TPARG_FL_KERNEL)) { @@ -1291,43 +1581,16 @@ parse_probe_arg(char *arg, const struct fetch_type *type, } ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0); arg = tmp + 1; - tmp = strrchr(arg, ')'); - if (!tmp) { - trace_probe_log_err(ctx->offset + strlen(arg), - DEREF_OPEN_BRACE); - return -EINVAL; - } else { - const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags); - int cur_offs = ctx->offset; - - *tmp = '\0'; - ret = parse_probe_arg(arg, t2, &code, end, ctx); - if (ret) - break; - ctx->offset = cur_offs; - if (code->op == FETCH_OP_COMM || - code->op == FETCH_OP_DATA) { - trace_probe_log_err(ctx->offset, COMM_CANT_DEREF); - return -EINVAL; - } - if (++code == end) { - trace_probe_log_err(ctx->offset, TOO_MANY_OPS); - return -EINVAL; - } - *pcode = code; - - code->op = deref; - code->offset = offset; - /* Reset the last type if used */ - ctx->last_type = NULL; - } + ret = handle_dereference(arg, pcode, end, ctx, deref, offset); + if (ret < 0) + return ret; break; case '\\': /* Immediate value */ if (arg[1] == '"') { /* Immediate string */ ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2); if (ret) break; - code->op = FETCH_OP_DATA; + code->op = FETCH_OP_IMMSTR; code->data = tmp; } else { ret = str_to_immediate(arg + 1, &code->immediate); @@ -1341,15 +1604,28 @@ parse_probe_arg(char *arg, const struct fetch_type *type, ret = handle_typecast(arg, pcode, end, ctx); break; default: - if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */ + if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX) || + str_has_prefix(arg, THIS_CPU_READ_PREFIX)) { + ret = parse_this_cpu(arg, pcode, end, ctx); + } else if (isalpha(arg[0]) || arg[0] == '_') { + /* BTF variable or event field*/ + if (ctx->flags & TPARG_FL_TEVENT) { + ret = parse_trace_event(arg, *pcode, ctx); + if (ret < 0) { + trace_probe_log_err(ctx->offset, + NO_EVENT_FIELD); + return -EINVAL; + } + break; + } if (!tparg_is_function_entry(ctx->flags) && !tparg_is_function_return(ctx->flags)) { trace_probe_log_err(ctx->offset, NOSUP_BTFARG); return -EINVAL; } ret = parse_btf_arg(arg, pcode, end, ctx); - break; } + break; } if (!ret && code->op == FETCH_OP_NOP) { /* Parsed, but do not find fetch method */ @@ -1482,7 +1758,7 @@ static int finalize_fetch_insn(struct fetch_insn *code, } else { if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF && code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM && - code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) { + code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) { trace_probe_log_err(ctx->offset + type_offset, BAD_STRING); return -EINVAL; @@ -1491,7 +1767,7 @@ static int finalize_fetch_insn(struct fetch_insn *code, if (!strcmp(parg->type->name, "symstr") || (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM || - code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG || + code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG || parg->count) { /* * IMM, DATA and COMM is pointing actual address, those @@ -1617,6 +1893,11 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size, ctx); if (ret < 0) goto fail; + /* nested_level must be 0 here, otherwise there is a bug. */ + if (WARN_ON_ONCE(ctx->nested_level)) { + ret = -EINVAL; + goto fail; + } /* Update storing type if BTF is available */ if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) && @@ -1667,7 +1948,7 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size, if (ret < 0) { for (code = tmp; code < tmp + FETCH_INSN_MAX; code++) if (code->op == FETCH_NOP_SYMBOL || - code->op == FETCH_OP_DATA) + code->op == FETCH_OP_IMMSTR) kfree(code->data); } kfree(tmp); @@ -1766,7 +2047,7 @@ void traceprobe_free_probe_arg(struct probe_arg *arg) while (code && code->op != FETCH_OP_END) { if (code->op == FETCH_NOP_SYMBOL || - code->op == FETCH_OP_DATA) + code->op == FETCH_OP_IMMSTR) kfree(code->data); code++; } @@ -2393,3 +2674,99 @@ int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_a } return 0; } + +#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG + +struct fetch_op_decode { + const char *name; + void (*decode)(struct seq_file *m, struct fetch_insn *insn); +}; + +static const struct fetch_op_decode fetch_op_decode[]; + +static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn) +{ + seq_puts(m, fetch_op_decode[insn->op].name); +} + +static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param); +} + +static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate); +} + +static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data); +} + +static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data); +} + +static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset); +} + +static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn) +{ + if (insn->op == FETCH_OP_ST_RAW) + seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size); + else + seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name, + insn->offset, insn->size); +} + +static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn) +{ + seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)", + fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift); +} + +static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn) +{ + struct ftrace_event_field *field = insn->data; + + seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name); +} + +#define FETCH_OP(opname, decode_fn) \ + [FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn } + +static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST; +#undef FETCH_OP + +static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg) +{ + int i; + + seq_printf(m, "# %s: ", parg->name); + for (i = 0; i < FETCH_INSN_MAX; i++) { + struct fetch_insn *insn = parg->code + i; + + if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode) + seq_printf(m, "unknown(%d)", insn->op); + else + fetch_op_decode[insn->op].decode(m, insn); + + if (insn->op == FETCH_OP_END) + break; + seq_puts(m, " -> "); + } + seq_putc(m, '\n'); +} + +void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp) +{ + int i; + + for (i = 0; i < tp->nr_args; i++) + trace_probe_dump_arg(m, &tp->args[i]); +} +#endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */ diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 15758cc11fc6..e6268a8dc378 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -83,38 +83,48 @@ static nokprobe_inline u32 update_data_loc(u32 loc, int consumed) /* Printing function type */ typedef int (*print_type_func_t)(struct trace_seq *, void *, void *); -enum fetch_op { - FETCH_OP_NOP = 0, - // Stage 1 (load) ops - FETCH_OP_REG, /* Register : .param = offset */ - FETCH_OP_STACK, /* Stack : .param = index */ - FETCH_OP_STACKP, /* Stack pointer */ - FETCH_OP_RETVAL, /* Return value */ - FETCH_OP_IMM, /* Immediate : .immediate */ - FETCH_OP_COMM, /* Current comm */ - FETCH_OP_ARG, /* Function argument : .param */ - FETCH_OP_FOFFS, /* File offset: .immediate */ - FETCH_OP_DATA, /* Allocated data: .data */ - FETCH_OP_EDATA, /* Entry data: .offset */ - // Stage 2 (dereference) op - FETCH_OP_DEREF, /* Dereference: .offset */ - FETCH_OP_UDEREF, /* User-space Dereference: .offset */ - // Stage 3 (store) ops - FETCH_OP_ST_RAW, /* Raw: .size */ - FETCH_OP_ST_MEM, /* Mem: .offset, .size */ - FETCH_OP_ST_UMEM, /* Mem: .offset, .size */ - FETCH_OP_ST_STRING, /* String: .offset, .size */ - FETCH_OP_ST_USTRING, /* User String: .offset, .size */ - FETCH_OP_ST_SYMSTR, /* Kernel Symbol String: .offset, .size */ - FETCH_OP_ST_EDATA, /* Store Entry Data: .offset */ - // Stage 4 (modify) op - FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */ - // Stage 5 (loop) op - FETCH_OP_LP_ARRAY, /* Array: .param = loop count */ - FETCH_OP_TP_ARG, /* Trace Point argument */ - FETCH_OP_END, - FETCH_NOP_SYMBOL, /* Unresolved Symbol holder */ -}; +#define FETCH_OP_LIST { \ + /* Stage 1 (load) ops */ \ + FETCH_OP(NOP, none), /* NOP */ \ + FETCH_OP(REG, param), /* Register: .param = offset */ \ + FETCH_OP(STACK, param), /* Stack: .param = index */ \ + FETCH_OP(STACKP, none), /* Stack pointer */ \ + FETCH_OP(RETVAL, none), /* Return value */ \ + FETCH_OP(IMM, imm), /* Immediate: .immediate */ \ + FETCH_OP(COMM, none), /* Current comm */ \ + FETCH_OP(CURRENT, none), /* Current task_struct address */\ + FETCH_OP(ARG, param), /* Argument: .param = index */ \ + FETCH_OP(FOFFS, imm), /* File offset: .immediate */ \ + FETCH_OP(IMMSTR, string), /* Allocated string: .data */ \ + FETCH_OP(EDATA, offset), /* Entry data: .offset */ \ + FETCH_OP(TP_ARG, tp_arg), /* Tracepoint argument: .data */\ + /* Stage 2 (dereference) ops */ \ + FETCH_OP(DEREF, offset), /* Dereference: .offset */ \ + FETCH_OP(UDEREF, offset), /* User-space dereference: .offset */\ + FETCH_OP(CPU_PTR, none), /* Per-CPU pointer: .offset */ \ + /* Stage 3 (store) ops */ \ + FETCH_OP(ST_RAW, store), /* Raw value: .size */ \ + FETCH_OP(ST_MEM, store), /* Memory: .offset, .size */ \ + FETCH_OP(ST_UMEM, store), /* User memory: .offset, .size */\ + FETCH_OP(ST_STRING, store), /* String: .offset, .size */ \ + FETCH_OP(ST_USTRING, store), /* User string: .offset, .size */\ + FETCH_OP(ST_SYMSTR, store), /* Symbol name: .offset, .size */\ + FETCH_OP(ST_EDATA, offset), /* Entry data: .offset */ \ + /* Stage 4 (modify) op */ \ + FETCH_OP(MOD_BF, bf), /* Bitfield: .basesize, .lshift, .rshift*/\ + /* Stage 5 (loop) op */ \ + FETCH_OP(LP_ARRAY, param), /* Loop array: .param = count */\ + /* End */ \ + FETCH_OP(END, none), \ + /* Unresolved Symbol holder */ \ + FETCH_OP(NOP_SYMBOL, symbol), /* Non loaded symbol: .data = symbol name */\ +} + +#define FETCH_OP(opname, decode_fn) FETCH_OP_##opname +enum fetch_op FETCH_OP_LIST; +#undef FETCH_OP + +#define FETCH_NOP_SYMBOL FETCH_OP_NOP_SYMBOL struct fetch_insn { enum fetch_op op; @@ -370,6 +380,13 @@ bool trace_probe_match_command_args(struct trace_probe *tp, int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **)); int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args, u8 *data, void *field); +#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG +void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp); +#else +static inline void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp) +{ +} +#endif #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API int traceprobe_get_entry_data_size(struct trace_probe *tp); @@ -414,6 +431,11 @@ static inline bool tparg_is_function_return(unsigned int flags) return (flags & TPARG_FL_LOC_MASK) == (TPARG_FL_KERNEL | TPARG_FL_RETURN); } +static inline bool tparg_is_event_probe(unsigned int flags) +{ + return !!(flags & TPARG_FL_TEVENT); +} + struct traceprobe_parse_context { struct trace_event_call *event; /* BTF related parameters */ @@ -430,8 +452,13 @@ struct traceprobe_parse_context { struct trace_probe *tp; unsigned int flags; int offset; + int nested_level; + int prefix_byteoffs; /* The byte offset of the prefix field of typecast */ }; +/* Each typecast consumes nested level. So the max number of typecast is 3. */ +#define TRACEPROBE_MAX_NESTED_LEVEL 3 + extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *argv, struct traceprobe_parse_context *ctx); @@ -511,12 +538,12 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NO_RETVAL, "This function returns 'void' type"), \ C(BAD_STACK_NUM, "Invalid stack number"), \ C(BAD_ARG_NUM, "Invalid argument number"), \ - C(BAD_VAR, "Invalid $-valiable specified"), \ + C(BAD_VAR, "Invalid $-variable specified"), \ C(BAD_REG_NAME, "Invalid register name"), \ C(BAD_MEM_ADDR, "Invalid memory address"), \ C(BAD_IMM, "Invalid immediate value"), \ C(IMMSTR_NO_CLOSE, "String is not closed with '\"'"), \ - C(FILE_ON_KPROBE, "File offset is not available with kprobe"), \ + C(FILE_ON_KPROBE, "File offset is not available for kernel probes"), \ C(BAD_FILE_OFFS, "Invalid file offset value"), \ C(SYM_ON_UPROBE, "Symbol is not available with uprobe"), \ C(TOO_MANY_OPS, "Dereference is too much nested"), \ @@ -559,6 +586,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(NO_PTR_STRCT, "This is not a pointer to union/structure."), \ C(NOSUP_DAT_ARG, "Non pointer structure/union argument is not supported."),\ C(BAD_HYPHEN, "Failed to parse single hyphen. Forgot '>'?"), \ + C(NO_EVENT_FIELD, "This event field is not found."), \ C(NO_BTF_FIELD, "This field is not found."), \ C(BAD_BTF_TID, "Failed to get BTF type info."),\ C(BAD_TYPE4STR, "This type does not fit for string."),\ @@ -566,7 +594,13 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(TOO_MANY_ARGS, "Too many arguments are specified"), \ C(TOO_MANY_EARGS, "Too many entry arguments specified"), \ C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \ - C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), + C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), \ + C(TYPECAST_REQ_FIELD, "Typecast requires a field access"), \ + C(TOO_MANY_NESTED, "Too many nested typecasts/dereferences"), \ + C(TYPECAST_SYM_OFFSET, "@SYM+/-OFFSET with typecast needs parentheses"), \ + C(TYPECAST_NOT_ALIGNED, "Typecast field option is not byte-aligned"), \ + C(TYPECAST_BAD_ARROW, "Typecast field option does not support -> operator"), \ + C(NOSUP_PERCPU, "Per-cpu variable access is only for kernel probes"), #undef C #define C(a, b) TP_ERR_##a diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h index f39b37fcdb3b..8db12f758fda 100644 --- a/kernel/trace/trace_probe_tmpl.h +++ b/kernel/trace/trace_probe_tmpl.h @@ -109,9 +109,12 @@ process_common_fetch_insn(struct fetch_insn *code, unsigned long *val) case FETCH_OP_COMM: *val = (unsigned long)current->comm; break; - case FETCH_OP_DATA: + case FETCH_OP_IMMSTR: *val = (unsigned long)code->data; break; + case FETCH_OP_CURRENT: + *val = (unsigned long)current; + break; default: return -EILSEQ; } @@ -126,25 +129,35 @@ process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val, struct fetch_insn *s3 = NULL; int total = 0, ret = 0, i = 0; u32 loc = 0; - unsigned long lval = val; + unsigned long lval, llval = val; stage2: /* 2nd stage: dereference memory if needed */ do { - if (code->op == FETCH_OP_DEREF) { - lval = val; + lval = val; + switch (code->op) { + case FETCH_OP_DEREF: ret = probe_mem_read(&val, (void *)val + code->offset, sizeof(val)); - } else if (code->op == FETCH_OP_UDEREF) { - lval = val; + break; + case FETCH_OP_UDEREF: ret = probe_mem_read_user(&val, (void *)val + code->offset, sizeof(val)); - } else break; + case FETCH_OP_CPU_PTR: + val = (unsigned long)this_cpu_ptr((void __percpu *)val); + ret = 0; + break; + default: + lval = llval; + goto out; + } if (ret) return ret; + llval = lval; code++; } while (1); +out: s3 = code; stage3: diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index c274346853d1..b2e264a4b96c 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -765,6 +765,9 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); seq_putc(m, '\n'); + + trace_probe_dump_args(m, &tu->tp); + return 0; } diff --git a/kernel/umh.c b/kernel/umh.c index 48117c569e1a..6e2c7bb315c6 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -71,10 +71,8 @@ static int call_usermodehelper_exec_async(void *data) spin_unlock_irq(¤t->sighand->siglock); /* - * Initial kernel threads share ther FS with init, in order to - * get the init root directory. But we've now created a new - * thread that is going to execve a user process and has its own - * 'struct fs_struct'. Reset umask to the default. + * Usermodehelper threads get a copy of userspace init's + * fs_struct. Reset umask to the default. */ current->fs->umask = 0022; diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 87dd5e0f6968..e5134ad7b663 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -359,14 +359,14 @@ static void lockup_detector_update_enable(void) if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled) watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED; if (watchdog_softlockup_user_enabled) - watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED; + watchdog_enabled |= WATCHDOG_SOFTLOCKUP_ENABLED; } #ifdef CONFIG_SOFTLOCKUP_DETECTOR /* - * Delay the soflockup report when running a known slow code. - * It does _not_ affect the timestamp of the last successdul reschedule. + * Delay the softlockup report when running a known slow code. + * It does _not_ affect the timestamp of the last successful reschedule. */ #define SOFTLOCKUP_DELAY_REPORT ULONG_MAX @@ -742,7 +742,7 @@ static int is_softlockup(unsigned long touch_ts, unsigned long period_ts, unsigned long now) { - if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) { + if ((watchdog_enabled & WATCHDOG_SOFTLOCKUP_ENABLED) && watchdog_thresh) { /* * If period_ts has not been updated during a sample_period, then * in the subsequent few sample_periods, period_ts might also not @@ -1098,11 +1098,11 @@ static void proc_watchdog_update(bool thresh_changed) * caller | table->data points to | 'which' * -------------------|----------------------------------|------------------------------- * proc_watchdog | watchdog_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED | - * | | WATCHDOG_SOFTOCKUP_ENABLED + * | | WATCHDOG_SOFTLOCKUP_ENABLED * -------------------|----------------------------------|------------------------------- * proc_nmi_watchdog | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED * -------------------|----------------------------------|------------------------------- - * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED + * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTLOCKUP_ENABLED */ static int proc_watchdog_common(int which, const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) @@ -1136,7 +1136,7 @@ static int proc_watchdog(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED | - WATCHDOG_SOFTOCKUP_ENABLED, + WATCHDOG_SOFTLOCKUP_ENABLED, table, write, buffer, lenp, ppos); } @@ -1159,7 +1159,7 @@ static int proc_nmi_watchdog(const struct ctl_table *table, int write, static int proc_soft_watchdog(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { - return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED, + return proc_watchdog_common(WATCHDOG_SOFTLOCKUP_ENABLED, table, write, buffer, lenp, ppos); } #endif diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a..784aeea768b6 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -226,6 +226,8 @@ struct worker_pool { /* L: hash of busy workers */ struct worker *manager; /* L: purely informational */ + /* L: last worker woken by kick_pool() */ + struct worker *last_woken_worker; struct list_head workers; /* A: attached workers */ struct ida worker_ida; /* worker IDs for task name */ @@ -1258,19 +1260,27 @@ static void kick_bh_pool(struct worker_pool *pool) } /** - * kick_pool - wake up an idle worker if necessary + * kick_pool_pick - select an idle worker to kick, deferring the wakeup * @pool: pool to kick + * @wakep: out-param, set to the task to wake after pool->lock is dropped * - * @pool may have pending work items. Wake up worker if necessary. Returns - * whether a worker was woken up. + * Like kick_pool() but, for a regular (non-BH) pool, returns the picked + * worker's task via @wakep instead of waking it, so the caller can issue the + * wakeup after dropping pool->lock (the wakeup takes rq->lock). Worker + * selection, wake_cpu setup and the BH kick still happen under the lock. + * Returns whether a worker was selected or kicked. + * + * Must be called with @pool->lock held. */ -static bool kick_pool(struct worker_pool *pool) +static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep) { struct worker *worker = first_idle_worker(pool); struct task_struct *p; lockdep_assert_held(&pool->lock); + *wakep = NULL; + if (!need_more_worker(pool) || !worker) return false; @@ -1310,10 +1320,30 @@ static bool kick_pool(struct worker_pool *pool) } } #endif - wake_up_process(p); + /* Track the last idle worker woken, used for stall diagnostics. */ + pool->last_woken_worker = worker; + + *wakep = p; return true; } +/** + * kick_pool - wake up an idle worker if necessary + * @pool: pool to kick + * + * @pool may have pending work items. Wake up worker if necessary. Returns + * whether a worker was woken up. + */ +static bool kick_pool(struct worker_pool *pool) +{ + struct task_struct *p; + bool kicked = kick_pool_pick(pool, &p); + + if (p) + wake_up_process(p); + return kicked; +} + #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT /* @@ -1505,7 +1535,11 @@ void wq_worker_tick(struct task_struct *task) if (!pwq) return; - pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC; + /* + * @pwq is shared across CPUs for unbound wqs and this advisory stat is + * bumped outside pool->lock, so the update is intentionally racy. + */ + data_race(pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC); if (!wq_cpu_intensive_thresh_us) return; @@ -2277,6 +2311,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, { struct pool_workqueue *pwq; struct worker_pool *last_pool, *pool; + struct task_struct *wake_task = NULL; unsigned int work_flags; unsigned int req_cpu = cpu; @@ -2399,7 +2434,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, trace_workqueue_activate_work(work); insert_work(pwq, work, &pool->worklist, work_flags); - kick_pool(pool); + kick_pool_pick(pool, &wake_task); } else { work_flags |= WORK_STRUCT_INACTIVE; insert_work(pwq, work, &pwq->inactive_works, work_flags); @@ -2407,6 +2442,8 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, out: raw_spin_unlock(&pool->lock); + if (wake_task) + wake_up_process(wake_task); rcu_read_unlock(); } @@ -2948,6 +2985,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list) pool->nr_workers--; pool->nr_idle--; + /* + * Clear last_woken_worker if it points to this worker, so that + * show_cpu_pool_busy_workers() cannot dereference a freed worker. + */ + if (pool->last_woken_worker == worker) + pool->last_woken_worker = NULL; + worker->flags |= WORKER_DIE; list_move(&worker->entry, list); @@ -3203,6 +3247,10 @@ static bool manage_workers(struct worker *worker) return true; } +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +static noinline void process_one_work(struct worker *worker, struct work_struct *work); +#endif + /** * process_one_work - process single work * @worker: self @@ -3223,6 +3271,7 @@ __acquires(&pool->lock) { struct pool_workqueue *pwq = get_work_pwq(work); struct worker_pool *pool = worker->pool; + struct task_struct *wake_task = NULL; unsigned long work_data; int lockdep_start_depth, rcu_start_depth; bool bh_draining = pool->flags & POOL_BH_DRAINING; @@ -3276,8 +3325,11 @@ __acquires(&pool->lock) * since nr_running would always be >= 1 at this point. This is used to * chain execution of the pending work items for WORKER_NOT_RUNNING * workers such as the UNBOUND and CPU_INTENSIVE ones. + * + * Select the worker under pool->lock; the wakeup is deferred until + * after the lock is dropped, guarded by the rcu_read_lock() below. */ - kick_pool(pool); + kick_pool_pick(pool, &wake_task); /* * Record the last pool and clear PENDING which should be the last @@ -3288,7 +3340,12 @@ __acquires(&pool->lock) set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool)); pwq->stats[PWQ_STAT_STARTED]++; + + rcu_read_lock(); raw_spin_unlock_irq(&pool->lock); + if (wake_task) + wake_up_process(wake_task); + rcu_read_unlock(); rcu_start_depth = rcu_preempt_depth(); lockdep_start_depth = lockdep_depth(current); @@ -7689,20 +7746,58 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); /* - * Show workers that might prevent the processing of pending work items. - * A busy worker that is not running on the CPU (e.g. sleeping in - * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as - * effectively as a CPU-bound one, so dump every in-flight worker. + * Report that a pool has no worker in running state, which is a sign that the + * pool may be stuck. Print pool info. Must be called with pool->lock held and + * inside a printk_deferred_enter/exit region. + */ +static void show_pool_no_running_worker(struct worker_pool *pool) +{ + lockdep_assert_held(&pool->lock); + + printk_deferred_enter(); + pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n", + pool->id, pool->cpu, + idle_cpu(pool->cpu) ? "idle" : "busy", + pool->nr_workers, pool->nr_idle); + pr_info("The pool might have trouble waking an idle worker.\n"); + /* + * last_woken_worker and its task are valid here: set_worker_dying() + * clears it under pool->lock before setting WORKER_DIE, so if + * last_woken_worker is non-NULL the kthread has not yet exited and + * worker->task is still alive. + */ + if (pool->last_woken_worker) { + pr_info("Backtrace of last woken worker:\n"); + sched_show_task(pool->last_woken_worker->task); + } else { + pr_info("Last woken worker empty\n"); + } + printk_deferred_exit(); +} + +/* + * Show running workers that might prevent the processing of pending work items. + * If no running worker is found, the pool may be stuck waiting for an idle + * worker to be woken, so report the pool state and the last woken worker. */ static void show_cpu_pool_busy_workers(struct worker_pool *pool) { + bool found_running = false; struct worker *worker; unsigned long irq_flags; - int bkt; + int cpu, bkt; raw_spin_lock_irqsave(&pool->lock, irq_flags); + /* Snapshot cpu inside the lock to safely use it after unlock. */ + cpu = pool->cpu; + hash_for_each(pool->busy_hash, bkt, worker, hentry) { + /* Skip workers that are not actively running on the CPU. */ + if (!task_is_running(worker->task)) + continue; + + found_running = true; /* * Defer printing to avoid deadlocks in console * drivers that queue work while holding locks @@ -7716,7 +7811,24 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool) printk_deferred_exit(); } + /* + * If no running worker was found, the pool is likely stuck. Print pool + * state and the backtrace of the last woken worker, which is the prime + * suspect for the stall. + */ + if (!found_running) + show_pool_no_running_worker(pool); + raw_spin_unlock_irqrestore(&pool->lock, irq_flags); + + /* + * Trigger a backtrace on the stalled CPU to capture what it is + * currently executing. Skip an offline CPU, whose NMI is never acked + * and would make the backtrace busy-wait until it times out. Done + * after releasing the lock to avoid issues with NMI delivery. + */ + if (!found_running && cpu_online(cpu)) + trigger_single_cpu_backtrace(cpu); } static void show_cpu_pools_busy_workers(void) diff --git a/lib/.gitignore b/lib/.gitignore index 101a4aa92fb5..42aee3ddb374 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -5,3 +5,4 @@ /gen_crc32table /gen_crc64table /oid_registry_data.c +/raid6 diff --git a/lib/Kconfig b/lib/Kconfig index a33988adfaa3..4e6b34c3346d 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -8,17 +8,6 @@ config BINARY_PRINTF menu "Library routines" -config RAID6_PQ - tristate - -config RAID6_PQ_BENCHMARK - bool "Automatically choose fastest RAID6 PQ functions" - depends on RAID6_PQ - default y - help - Benchmark all available RAID6 PQ functions on init and choose the - fastest one. - config LINEAR_RANGES tristate @@ -170,12 +159,6 @@ config AUDIT_COMPAT_GENERIC depends on AUDIT_GENERIC && AUDIT_ARCH_COMPAT_GENERIC && COMPAT default y -config RANDOM32_SELFTEST - bool "PRNG perform self test on init" - help - This option enables the 32 bit PRNG library functions to perform a - self test on initialization. - # # compression support is select'ed if needed # @@ -603,7 +586,7 @@ config OBJAGG config LWQ_TEST bool "Boot-time test for lwq queuing" help - Run boot-time test of light-weight queuing. + Run boot-time test of light-weight queuing. endmenu diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 3ca343f1a8d0..77339e68a6fb 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1048,34 +1048,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" @@ -2261,6 +2233,13 @@ config KCOV_SELFTEST On test failure, causes the kernel to panic. Recommended to be enabled, ensuring critical functionality works as intended. +config DEBUG_AID_FOR_SYZBOT + bool "Additional debug code for syzbot" + default n + select PREEMPT_RT if EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST + help + This option is intended for testing by syzbot. + menuconfig RUNTIME_TESTING_MENU bool "Runtime Testing" default y @@ -2976,6 +2955,16 @@ config BITS_TEST If unsure, say N. +config SHDI3_KUNIT_TEST + tristate "KUnit test for __ashldi3(), __ashrdi3(), and __lshrdi3()" + depends on KUNIT + depends on ARM || XTENSA || MICROBLAZE || ((RISCV || SPARC) && !64BIT) + help + This builds the unit test for __ashldi3(), __ashrdi3(), and + __lshrdi3() helper functions used to implement 64-bit arithmetic + shift left, arithmetic shift right and logical shift right, + respectively, on a 32-bit CPUs. + config SLUB_KUNIT_TEST tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS depends on SLUB_DEBUG && KUNIT @@ -3494,6 +3483,23 @@ config GCD_KUNIT_TEST If unsure, say N +config POLYNOMIAL_KUNIT_TEST + tristate "Polynomial calculation (polynomial_calc) test" if !KUNIT_ALL_TESTS + depends on KUNIT + select POLYNOMIAL + default KUNIT_ALL_TESTS + help + This option enables the KUnit test suite for the polynomial_calc() + function, which evaluates integer polynomials using factor + redistribution to avoid overflow. + + The test suite verifies correctness for constant, linear, and + quadratic polynomials, negative coefficients, per-step dividers, + divider_leftover, total_divider scaling, and a real sensor + N-to-temperature conversion polynomial. + + If unsure, say N + config PRIME_NUMBERS_KUNIT_TEST tristate "Prime number generator test" if !KUNIT_ALL_TESTS depends on KUNIT @@ -3521,6 +3527,15 @@ config GLOB_KUNIT_TEST If unsure, say N +config PRANDOM_KUNIT_TEST + tristate "KUnit test for prandom" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this option to test the prandom functions at runtime. + + If unsure, say N + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST diff --git a/lib/Makefile b/lib/Makefile index 23e07d19d01c..132147b3c547 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -167,7 +167,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ obj-$(CONFIG_XZ_DEC) += xz/ -obj-$(CONFIG_RAID6_PQ) += raid6/ lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o @@ -197,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o @@ -274,6 +272,22 @@ filechk_defbconf = cat $(or $(real-prereqs), /dev/null) $(obj)/default.bconf: $(CONFIG_BOOT_CONFIG_EMBED_FILE) FORCE $(call filechk,defbconf) +obj-$(CONFIG_CMDLINE_FROM_BOOTCONFIG) += embedded-cmdline.o +$(obj)/embedded-cmdline.o: $(obj)/embedded_cmdline.bin + +# Render the bootconfig "kernel" subtree to a flat cmdline string using +# the userspace tools/bootconfig parser (-C mode). The runtime prepend +# helper enforces COMMAND_LINE_SIZE at boot, so no build-time size +# check is performed here (COMMAND_LINE_SIZE is an arch header +# constant, not a Kconfig value). +quiet_cmd_render_cmdline = BCONF2C $@ + cmd_render_cmdline = \ + $(objtree)/tools/bootconfig/bootconfig -C $< > $@ + +targets += embedded_cmdline.bin +$(obj)/embedded_cmdline.bin: $(obj)/default.bconf $(objtree)/tools/bootconfig/bootconfig FORCE + $(call if_changed,render_cmdline) + obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o diff --git a/lib/base64.c b/lib/base64.c index 41961a444028..325c7332b049 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -122,7 +122,7 @@ EXPORT_SYMBOL_GPL(base64_encode); * @src: the string to decode. Doesn't need to be NUL-terminated. * @srclen: the length of @src in bytes * @dst: (output) the decoded binary data - * @padding: whether to append '=' padding characters + * @padding: whether the input is expected to include '=' padding characters * @variant: which base64 variant to use * * Decodes a string using the selected Base64 variant. @@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6 return -1; val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); - *bp++ = val >> 10; if (srclen == 2) { if (val & 0x800003ff) return -1; + *bp++ = val >> 10; } else { val |= base64_rev_tables[s[2]]; if (val & 0x80000003) return -1; + *bp++ = val >> 10; *bp++ = val >> 2; } return bp - dst; diff --git a/lib/bootconfig.c b/lib/bootconfig.c index f445b7703fdd..89c88e359179 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -19,9 +19,13 @@ #include #include #include +#include +#include +#include #include #include #include +#include /* COMMAND_LINE_SIZE */ #ifdef CONFIG_BOOT_CONFIG_EMBED /* embedded_bootconfig_data is defined in bootconfig-data.S */ @@ -34,7 +38,129 @@ const char * __init xbc_get_embedded_bootconfig(size_t *size) return (*size) ? embedded_bootconfig_data : NULL; } #endif -#endif + +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG +/* embedded_kernel_cmdline is defined in embedded-cmdline.S */ +extern __visible const char embedded_kernel_cmdline[]; +extern __visible const char embedded_kernel_cmdline_end[]; + +/* Set once the embedded cmdline has actually been prepended. */ +static bool xbc_cmdline_applied __initdata; + +/* + * str_prepend() - Prepend @src in front of the string in @dst, in place + * @dst: NUL-terminated destination buffer, currently @dst_len bytes long + * @dst_len: length of the current @dst string (excluding its NUL) + * @src: bytes to prepend (not NUL-terminated) + * @src_len: number of bytes from @src to prepend + * + * The caller must guarantee @dst has room for src_len + dst_len + 1 bytes. + * Moving dst_len + 1 bytes carries @dst's NUL terminator too, so an empty + * @dst needs no special case. + */ +static void __init str_prepend(char *dst, size_t dst_len, + const char *src, size_t src_len) +{ + memmove(dst + src_len, dst, dst_len + 1); + memcpy(dst, src, src_len); +} + +/** + * xbc_prepend_embedded_cmdline() - Prepend embedded bootconfig cmdline + * @dst: cmdline buffer to prepend into (must already contain a NUL byte) + * @size: total capacity of @dst in bytes + * + * Prepend the build-time-rendered "kernel" subtree of the embedded + * bootconfig to @dst. The rendered string already ends with a single + * space (the xbc_snprint_cmdline() invariant), which serves as the + * separator between the embedded keys and any existing content of @dst. + * On overflow, log an error and leave @dst untouched rather than + * silently truncating: booting without the embedded values is better + * than refusing to boot, and the error message tells the user why + * their embedded keys are missing. + * + * Intended to be called from setup_arch() before parse_early_param() so + * that early_param() handlers see the embedded values. + */ +void __init xbc_prepend_embedded_cmdline(char *dst, size_t size) +{ + size_t embed_len = embedded_kernel_cmdline_end - embedded_kernel_cmdline; + size_t dst_len; + + if (!size || embed_len <= 1) /* trailing NUL only */ + return; + embed_len--; /* exclude trailing NUL byte */ + + dst_len = strnlen(dst, size); + if (embed_len + dst_len + 1 > size) { + pr_err("embedded bootconfig cmdline (%zu bytes) does not fit in COMMAND_LINE_SIZE with %zu bytes already used; ignoring embedded values\n", + embed_len, dst_len); + return; + } + + str_prepend(dst, dst_len, embedded_kernel_cmdline, embed_len); + xbc_cmdline_applied = true; +} + +/** + * xbc_embedded_cmdline_applied() - Did the embedded cmdline get prepended? + * + * Return true if xbc_prepend_embedded_cmdline() actually prepended the + * embedded "kernel" subtree. setup_boot_config() uses this to avoid + * rendering the same keys a second time. + */ +bool __init xbc_embedded_cmdline_applied(void) +{ + return xbc_cmdline_applied; +} +#endif /* CONFIG_CMDLINE_FROM_BOOTCONFIG */ + +/* parse_args() callback: flag when the "bootconfig" parameter is present. */ +static int __init bootconfig_optin(char *param, char *val, + const char *unused, void *arg) +{ + if (!strcmp(param, "bootconfig")) + *(bool *)arg = true; + return 0; +} + +/** + * bootconfig_cmdline_requested() - Was "bootconfig" passed on the cmdline? + * @boot_cmdline: kernel command line to inspect (not modified) + * @end_offset: if non-NULL, set to the offset of the init arguments that + * follow a "--" separator, or 0 when there is none + * + * Parse a private copy of @boot_cmdline (parse_args() is destructive) and + * report whether "bootconfig" is present before the "--" separator. + * setup_arch() uses this to gate prepending the build-time embedded cmdline; + * setup_boot_config() uses it for the runtime opt-in and to locate the init + * arguments via @end_offset. Sharing one parser keeps the early and late + * paths agreeing on what counts as opt-in. CONFIG_BOOT_CONFIG_FORCE is not + * folded in here; callers apply it where they need it. + */ +bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset) +{ + static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; + bool found = false; + char *err; + + if (end_offset) + *end_offset = 0; + + strscpy(tmp_cmdline, boot_cmdline, COMMAND_LINE_SIZE); + err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, + &found, bootconfig_optin); + if (IS_ERR(err)) + return false; + + /* parse_args() stops at "--" and returns the address of the rest. */ + if (end_offset && err) + *end_offset = err - tmp_cmdline; + + return found; +} + +#endif /* __KERNEL__ */ /* * Extra Boot Config (XBC) is given as tree-structured ascii text of @@ -427,11 +553,30 @@ static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) { struct xbc_node *knode, *vnode; - char *end = buf + size; const char *val, *q; + size_t len = 0; int ret; + /* + * Track the running written length rather than advancing @buf, so we + * never form "buf + size" or "buf += ret" while @buf is NULL (the + * size-probe call passes buf=NULL, size=0). NULL pointer arithmetic + * is undefined behavior and trips host UBSan / FORTIFY_SOURCE when + * this renderer runs at kernel build time. snprintf(NULL, 0, ...) + * itself is well defined and returns the would-be length. + */ xbc_node_for_each_key_value(root, knode, val) { + /* + * An empty or value-only @root (e.g. "kernel {}" or + * "kernel = x", possibly alongside "kernel.foo = bar") + * yields @root itself here. Skip it: composing a key for it + * would fail with -EINVAL, yet any real descendant keys must + * still be rendered. An entirely empty subtree then renders + * nothing and returns 0 rather than an error. + */ + if (knode == root) + continue; + ret = xbc_node_compose_key_after(root, knode, xbc_namebuf, XBC_KEYLEN_MAX); if (ret < 0) @@ -439,10 +584,11 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) vnode = xbc_node_get_child(knode); if (!vnode) { - ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf); + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s ", xbc_namebuf); if (ret < 0) return ret; - buf += ret; + len += ret; continue; } xbc_array_for_each_value(vnode, val) { @@ -452,15 +598,15 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) * whitespace. */ q = strpbrk(val, " \t\r\n") ? "\"" : ""; - ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", - xbc_namebuf, q, val, q); + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s=%s%s%s ", xbc_namebuf, q, val, q); if (ret < 0) return ret; - buf += ret; + len += ret; } } - return buf - (end - size); + return len; } #undef rest diff --git a/lib/bug.c b/lib/bug.c index d99e369bc110..292420f45811 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -1,41 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 /* - Generic support for BUG() - - This respects the following config options: - - CONFIG_BUG - emit BUG traps. Nothing happens without this. - CONFIG_GENERIC_BUG - enable this code. - CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file - CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG - - CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable - (though they're generally always on). - - CONFIG_GENERIC_BUG is set by each architecture using this code. - - To use this, your architecture must: - - 1. Set up the config options: - - Enable CONFIG_GENERIC_BUG if CONFIG_BUG - - 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) - - Define HAVE_ARCH_BUG - - Implement BUG() to generate a faulting instruction - - NOTE: struct bug_entry does not have "file" or "line" entries - when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate - the values accordingly. - - 3. Implement the trap - - In the illegal instruction trap handler (typically), verify - that the fault was in kernel mode, and call report_bug() - - report_bug() will return whether it was a false alarm, a warning, - or an actual bug. - - You must implement the is_valid_bugaddr(bugaddr) callback which - returns true if the eip is a real kernel address, and it points - to the expected BUG trap instruction. - - Jeremy Fitzhardinge 2006 + * Generic support for BUG() + * + * This respects the following config options: + * + * CONFIG_BUG - emit BUG traps. Nothing happens without this. + * CONFIG_GENERIC_BUG - enable this code. + * CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file + * CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG + * + * CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable + * (though they're generally always on). + * + * CONFIG_GENERIC_BUG is set by each architecture using this code. + * + * To use this, your architecture must: + * + * 1. Set up the config options: + * - Enable CONFIG_GENERIC_BUG if CONFIG_BUG + * + * 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) + * - Define HAVE_ARCH_BUG + * - Implement BUG() to generate a faulting instruction + * - NOTE: struct bug_entry does not have "file" or "line" entries + * when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate + * the values accordingly. + * + * 3. Implement the trap + * - In the illegal instruction trap handler (typically), verify + * that the fault was in kernel mode, and call report_bug() + * - report_bug() will return whether it was a false alarm, a warning, + * or an actual bug. + * - You must implement the is_valid_bugaddr(bugaddr) callback which + * returns true if the eip is a real kernel address, and it points + * to the expected BUG trap instruction. + * + * Jeremy Fitzhardinge 2006 */ #define pr_fmt(fmt) fmt @@ -72,7 +72,7 @@ static struct bug_entry *module_find_bug(unsigned long bugaddr) guard(rcu)(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { - unsigned i; + unsigned int i; bug = mod->bug_table; for (i = 0; i < mod->num_bugs; ++i, ++bug) @@ -192,14 +192,14 @@ void __warn_printf(const char *fmt, struct pt_regs *regs) } #endif - printk("%s", fmt); + pr_warn("%s", fmt); } static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) { bool warning, once, done, no_cut, has_args; const char *file, *fmt; - unsigned line; + unsigned int line; if (!bug) { if (!is_valid_bugaddr(bugaddr)) @@ -247,7 +247,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga * extra debugging message it writes before triggering the handler. */ if (!no_cut) { - printk(KERN_DEFAULT CUT_HERE); + pr_info(CUT_HERE); __warn_printf(fmt, has_args ? regs : NULL); } diff --git a/lib/cmdline.c b/lib/cmdline.c index 90ed997d9570..16cce6621cec 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n) * When @pint is NULL the function can be used as a validator of * the current option in the string. * - * Return values: + * Return: * 0 - no int in string * 1 - int found, no subsequent comma * 2 - int found including a subsequent comma @@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options); * * Parses a string into a number. The number stored at @ptr is * potentially suffixed with K, M, G, T, P, E. + * + * Return: The value as recognized by simple_strtoull() multiplied + * by the value as specified by suffix, if any. */ unsigned long long memparse(const char *ptr, char **retptr) { char *endptr; /* local pointer to end of parsed string */ - unsigned long long ret = simple_strtoull(ptr, &endptr, 0); + unsigned int shl = 0; + /* Consume valid suffix even in case of overflow. */ switch (*endptr) { case 'E': case 'e': - ret <<= 10; + shl += 10; fallthrough; case 'P': case 'p': - ret <<= 10; + shl += 10; fallthrough; case 'T': case 't': - ret <<= 10; + shl += 10; fallthrough; case 'G': case 'g': - ret <<= 10; + shl += 10; fallthrough; case 'M': case 'm': - ret <<= 10; + shl += 10; fallthrough; case 'K': case 'k': - ret <<= 10; - endptr++; + shl += 10; fallthrough; default: break; } + if (shl && likely(ptr != endptr)) { + /* Have valid suffix with preceding number. */ + if (unlikely(check_shl_overflow(ret, shl, &ret))) + ret = ULLONG_MAX; + endptr++; + } + if (retptr) *retptr = endptr; @@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse); * This function parses a string containing a comma-separated list of * strings like a=b,c. * - * Return true if there's such option in the string, or return false. + * Return: True if there's such option in the string or false otherwise. */ bool parse_option_str(const char *str, const char *option) { diff --git a/lib/codetag.c b/lib/codetag.c index 304667897ad4..4001a7ea6675 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -35,12 +35,9 @@ struct codetag_module { static DEFINE_MUTEX(codetag_lock); static LIST_HEAD(codetag_types); -void codetag_lock_module_list(struct codetag_type *cttype, bool lock) +void codetag_lock_module_list(struct codetag_type *cttype) { - if (lock) - down_read(&cttype->mod_lock); - else - up_read(&cttype->mod_lock); + down_read(&cttype->mod_lock); } bool codetag_trylock_module_list(struct codetag_type *cttype) @@ -48,6 +45,11 @@ bool codetag_trylock_module_list(struct codetag_type *cttype) return down_read_trylock(&cttype->mod_lock) != 0; } +void codetag_unlock_module_list(struct codetag_type *cttype) +{ + up_read(&cttype->mod_lock); +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c index 6bf130cfbbf9..3d2b017a0525 100644 --- a/lib/crypto/md5.c +++ b/lib/crypto/md5.c @@ -298,19 +298,5 @@ void hmac_md5_usingrawkey(const u8 *raw_key, size_t raw_key_len, } EXPORT_SYMBOL_GPL(hmac_md5_usingrawkey); -#ifdef md5_mod_init_arch -static int __init md5_mod_init(void) -{ - md5_mod_init_arch(); - return 0; -} -subsys_initcall(md5_mod_init); - -static void __exit md5_mod_exit(void) -{ -} -module_exit(md5_mod_exit); -#endif - MODULE_DESCRIPTION("MD5 and HMAC-MD5 library functions"); MODULE_LICENSE("GPL"); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 6fb00e08a4e2..877f7675ba26 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -894,6 +894,14 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr) } raw_spin_unlock_irqrestore(&db->lock, flags); + + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. + */ + if (!debug_objects_enabled) + return 0; + debug_print_object(&o, "activate"); switch (o.state) { @@ -1071,6 +1079,15 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) return; } + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. Don't run the fixup + * as it might turn a valid object useless. See for example + * hrtimer_fixup_assert_init(). + */ + if (!debug_objects_enabled) + return; + /* Object is neither tracked nor static. It's not initialized. */ debug_print_object(&o, "assert_init"); debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE); diff --git a/lib/embedded-cmdline.S b/lib/embedded-cmdline.S new file mode 100644 index 000000000000..bda81b4a42be --- /dev/null +++ b/lib/embedded-cmdline.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Embed the build-time-rendered bootconfig "kernel" subtree as a flat + * cmdline string. setup_arch() prepends this to boot_command_line on + * architectures that select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao + */ + .section .init.rodata.embed_cmdline, "a", %progbits + .global embedded_kernel_cmdline +embedded_kernel_cmdline: + .incbin "lib/embedded_cmdline.bin" + .byte 0 + .global embedded_kernel_cmdline_end +embedded_kernel_cmdline_end: diff --git a/lib/error-inject.c b/lib/error-inject.c index f3d1b70be605..32f3d1ca9ea2 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -219,9 +219,9 @@ static int __init ei_debugfs_init(void) dir = debugfs_create_dir("error_injection", NULL); file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops); - if (!file) { + if (IS_ERR(file)) { debugfs_remove(dir); - return -ENOMEM; + return PTR_ERR(file); } return 0; diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index f5d5333450a0..4fc66722d00d 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -178,6 +178,37 @@ unsigned int font_data_size(font_data_t *fd) } EXPORT_SYMBOL_GPL(font_data_size); +static unsigned int font_data_num_glyphs(font_data_t *fd, unsigned int width, unsigned int height) +{ + return font_data_size(fd) / font_glyph_size(width, height); +} + +/** + * font_data_glyph_buf() - Returns the glyph for a specific character as raw bytes + * @fd: The font data + * @width: The glyph width in bits per scanline + * @vpitch: The number of scanlines per glyph + * @c: The character + * + * Glyphs start at fixed intervals within the font data. font_data_glyph_buf() + * returns the glyph shape of the specified character. If no such glyph + * exists in the font, it returns NULL. + * + * Returns: + * The character's raw glyph shape, or NULL if no glyph exists for the character. The + * provided buffer is read-only. + */ +const unsigned char *font_data_glyph_buf(font_data_t *fd, + unsigned int width, unsigned int vpitch, + unsigned int c) +{ + if (c >= font_data_num_glyphs(fd, width, vpitch)) + return NULL; + + return font_data_buf(fd) + font_glyph_size(width, vpitch) * c; +} +EXPORT_SYMBOL_GPL(font_data_glyph_buf); + /** * font_data_is_equal - Compares font data for equality * @lhs: Left-hand side font data diff --git a/lib/idr.c b/lib/idr.c index 69bee5369670..1649f41016e7 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -495,10 +495,9 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max) unsigned long index = min / IDA_BITMAP_BITS; unsigned int offset = min % IDA_BITMAP_BITS; unsigned long *addr, size, bit; - unsigned long tmp = 0; + unsigned long tmp; unsigned long flags; void *entry; - int ret; if ((int)min < 0) return -EINVAL; @@ -508,40 +507,34 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max) xa_lock_irqsave(&ida->xa, flags); entry = xa_find(&ida->xa, &index, max / IDA_BITMAP_BITS, XA_PRESENT); - if (!entry) { - ret = -ENOENT; - goto err_unlock; - } - - if (index > min / IDA_BITMAP_BITS) - offset = 0; - if (index * IDA_BITMAP_BITS + offset > max) { - ret = -ENOENT; - goto err_unlock; - } - - if (xa_is_value(entry)) { - tmp = xa_to_value(entry); - addr = &tmp; - size = BITS_PER_XA_VALUE; - } else { - addr = ((struct ida_bitmap *)entry)->bitmap; - size = IDA_BITMAP_BITS; - } - - bit = find_next_bit(addr, size, offset); + while (entry) { + if (index > min / IDA_BITMAP_BITS) + offset = 0; + if (index * IDA_BITMAP_BITS + offset > max) + break; - xa_unlock_irqrestore(&ida->xa, flags); + if (xa_is_value(entry)) { + tmp = xa_to_value(entry); + addr = &tmp; + size = BITS_PER_XA_VALUE; + } else { + addr = ((struct ida_bitmap *)entry)->bitmap; + size = IDA_BITMAP_BITS; + } - if (bit == size || - index * IDA_BITMAP_BITS + bit > max) - return -ENOENT; + bit = find_next_bit(addr, size, offset); + if (bit < size && + index * IDA_BITMAP_BITS + bit <= max) { + xa_unlock_irqrestore(&ida->xa, flags); + return index * IDA_BITMAP_BITS + bit; + } - return index * IDA_BITMAP_BITS + bit; + entry = xa_find_after(&ida->xa, &index, + max / IDA_BITMAP_BITS, XA_PRESENT); + } -err_unlock: xa_unlock_irqrestore(&ida->xa, flags); - return ret; + return -ENOENT; } EXPORT_SYMBOL(ida_find_first_range); diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index 16200feacbf3..eba2d3e28980 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -311,6 +312,27 @@ static inline int span_iteration_check(void) {return 0; } static int interval_tree_test_init(void) { + if (nnodes <= 0) { + pr_warn("nnodes must be positive\n"); + return -EINVAL; + } + if (nsearches <= 0) { + pr_warn("nsearches must be positive\n"); + return -EINVAL; + } + if (perf_loops <= 0) { + pr_warn("perf_loops must be positive\n"); + return -EINVAL; + } + if (search_loops <= 0) { + pr_warn("search_loops must be positive\n"); + return -EINVAL; + } + if (max_endpoint < 2) { + pr_warn("max_endpoint must be at least 2\n"); + return -EINVAL; + } + nodes = kmalloc_objs(struct interval_tree_node, nnodes); if (!nodes) return -ENOMEM; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 273919b16161..c2484551a4e8 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1568,6 +1568,7 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, struct folio *folio; unsigned int nr = 0, offset; loff_t pos = i->xarray_start + i->iov_offset; + bool will_alloc = !*pages; XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); offset = pos & ~PAGE_MASK; @@ -1595,6 +1596,14 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, } rcu_read_unlock(); + if (!nr) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } + return 0; + } + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); iov_iter_advance(i, maxsize); return maxsize; @@ -1628,6 +1637,8 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, bi.bi_bvec_done = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); + if (!maxpages) + return -ENOMEM; while (bi.bi_size && bi.bi_idx < i->nr_segs) { struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); @@ -1745,6 +1756,7 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, unsigned long addr; unsigned int gup_flags = 0; size_t offset; + bool will_alloc = !*pages; int res; if (i->data_source == ITER_DEST) @@ -1761,8 +1773,14 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, if (!maxpages) return -ENOMEM; res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); - if (unlikely(res <= 0)) + if (unlikely(res <= 0)) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } return res; + } + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); iov_iter_advance(i, maxsize); return maxsize; diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 97be2a39f537..bac1c057e1b0 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -39,25 +40,31 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) return s; } -/* - * Convert non-negative integer string representation in explicitly given radix - * to an integer. A maximum of max_chars characters will be converted. +/** + * _parse_integer_limit - Convert integer string representation to an integer + * @s: Integer string representation + * @base: Radix + * @p: Where to store result + * @max_chars: Maximum amount of characters to convert + * @init: Initial value of the multiply-accumulate result + * + * Convert non-negative integer string representation in explicitly given + * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX. * - * Return number of characters consumed maybe or-ed with overflow bit. - * If overflow occurs, result integer (incorrect) is still returned. + * This function is the workhorse of other string conversion functions and it + * is discouraged to use it explicitly. Consider kstrto*() family instead. * - * Don't you dare use this function. + * Return: Number of characters consumed, maybe ORed with overflow bit */ noinline unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, - size_t max_chars) + size_t max_chars, unsigned long long init) { + unsigned int rv, overflow = 0; unsigned long long res; - unsigned int rv; - res = 0; - rv = 0; - while (max_chars--) { + res = init; + for (rv = 0; rv < max_chars; rv++, s++) { unsigned int c = *s; unsigned int lc = _tolower(c); unsigned int val; @@ -76,21 +83,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon * it in the max base we support (16) */ if (unlikely(res & (~0ull << 60))) { - if (res > div_u64(ULLONG_MAX - val, base)) - rv |= KSTRTOX_OVERFLOW; + if (check_mul_overflow(res, base, &res) || + check_add_overflow(res, val, &res)) { + res = ULLONG_MAX; + overflow = KSTRTOX_OVERFLOW; + } + } else { + res = res * base + val; } - res = res * base + val; - rv++; - s++; } *p = res; - return rv; -} - -noinline -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p) -{ - return _parse_integer_limit(s, base, p, INT_MAX); + return rv | overflow; } static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) @@ -392,6 +395,109 @@ int kstrtobool(const char *s, bool *res) } EXPORT_SYMBOL(kstrtobool); +static int _kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + unsigned int rv_int, rv_frac; + u64 _res = 0; + + rv_int = _parse_integer(s, 10, &_res); + if (rv_int & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_int; + + if (*s == '.') + s++; /* skip decimal point */ + + rv_frac = _parse_integer(s, 10, &_res, scale, _res); + if (rv_frac & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_frac; + + /* + * Check input beyond rv_int and rv_frac to cover cases like ".5" with + * scale 0, which is considered a valid input, being parsed as 0. + */ + if (!rv_int && !rv_frac && !isdigit(*s)) + return -EINVAL; + + while (isdigit(*s)) /* truncate digits */ + s++; + + if (*s == '\n') + s++; + if (*s) + return -EINVAL; + + if (_res && ((scale - rv_frac) > 19 /* log10(2^64) = 19.26 */ || + check_mul_overflow(_res, int_pow(10, scale - rv_frac), &_res))) + return -ERANGE; + + *res = _res; + return 0; +} + +/** + * kstrtoudec64() - Convert a string to an unsigned 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign, but not a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 3 with input "123.45" results in 123450. Note that + * trailing zeros in the fractional part input to match the scale are not + * required. Also, digits beyond the specified scale are ignored. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + if (s[0] == '+') + s++; + return _kstrtoudec64(s, scale, res); +} +EXPORT_SYMBOL(kstrtoudec64); + +/** + * kstrtodec64() - Convert a string to a signed 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign or a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 4 with input "-3.141592" results in -31415. Note + * that digits beyond the specified scale are ignored. Also, trailing zeros in + * the fractional part input to match the scale are not required. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtodec64(const char *s, unsigned int scale, s64 *res) +{ + u64 tmp; + int rv; + + if (s[0] == '-') { + rv = _kstrtoudec64(s + 1, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)-tmp > 0) + return -ERANGE; + *res = -tmp; + } else { + rv = kstrtoudec64(s, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)tmp < 0) + return -ERANGE; + *res = tmp; + } + return 0; +} +EXPORT_SYMBOL(kstrtodec64); + /* * Since "base" would be a nonsense argument, this open-codes the * _from_user helper instead of using the helper macro below. diff --git a/lib/kstrtox.h b/lib/kstrtox.h index 158c400ca865..73dee79fd8ed 100644 --- a/lib/kstrtox.h +++ b/lib/kstrtox.h @@ -2,10 +2,23 @@ #ifndef _LIB_KSTRTOX_H #define _LIB_KSTRTOX_H +#include + #define KSTRTOX_OVERFLOW (1U << 31) const char *_parse_integer_fixup_radix(const char *s, unsigned int *base); unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *res, - size_t max_chars); -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *res); + size_t max_chars, unsigned long long init); + +#define _parse_integer0(s, base, res, ...) \ + _parse_integer_limit(s, base, res, INT_MAX, 0) + +#define _parse_integer1(s, base, res, max_chars, ...) \ + _parse_integer_limit(s, base, res, max_chars, 0) + +#define _parse_integer2(s, base, res, max_chars, init, ...) \ + _parse_integer_limit(s, base, res, max_chars, init) + +#define _parse_integer(s, base, res, ...) \ + CONCATENATE(_parse_integer, COUNT_ARGS(__VA_ARGS__))(s, base, res, __VA_ARGS__) #endif diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 9c326f1837bd..442b2ceb955b 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -76,18 +76,30 @@ static int debugfs_print_results(struct seq_file *seq, void *v) seq_puts(seq, "KTAP version 1\n"); seq_puts(seq, "1..1\n"); - /* Print suite header because it is not stored in the test logs. */ - seq_puts(seq, KUNIT_SUBTEST_INDENT "KTAP version 1\n"); - seq_printf(seq, KUNIT_SUBTEST_INDENT "# Subtest: %s\n", suite->name); - seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite)); - - kunit_suite_for_each_test_case(suite, test_case) - debugfs_print_result(seq, test_case->log); + if (suite->status != KUNIT_SKIPPED) { + /* Print suite header because it is not stored in the test logs. */ + seq_puts(seq, + KUNIT_SUBTEST_INDENT "KTAP version 1\n"); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "# Subtest: %s\n", + suite->name); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "1..%zd\n", + kunit_suite_num_test_cases(suite)); + + kunit_suite_for_each_test_case(suite, test_case) + debugfs_print_result(seq, test_case->log); + } debugfs_print_result(seq, suite->log); - seq_printf(seq, "%s %d %s\n", - kunit_status_to_ok_not_ok(success), 1, suite->name); + if (suite->status != KUNIT_SKIPPED) + seq_printf(seq, "%s %d %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name); + else + seq_printf(seq, "%s %d %s # SKIP %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name, + suite->status_comment); return 0; } diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index 0bae7b7ca0b0..b8ded54fa46d 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -591,5 +591,34 @@ static struct kunit_suite example_init_test_suite = { */ kunit_test_init_section_suites(&example_init_test_suite); +/* + * This test should always be skipped. + */ +static void example_skip_suite_test(struct kunit *test) +{ + /* This line should never be seen */ + KUNIT_FAIL(test, "You should not see a this."); +} + +static struct kunit_case example_skip_suite_test_cases[] = { + KUNIT_CASE(example_skip_suite_test), + {} +}; + +static int example_skip_suite_init(struct kunit_suite *suite) +{ + kunit_mark_skipped(suite, "Test suite expected to be skipped"); + return 0; +} + +static struct kunit_suite example_test_skip_suite = { + .name = "example_skip_suite", + .suite_init = example_skip_suite_init, + .test_cases = example_skip_suite_test_cases, +}; + +/* This registers a test suite that will be skipped */ +kunit_test_suite(example_test_skip_suite); + MODULE_DESCRIPTION("Example KUnit test suite"); MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 99773e000e1b..09e3dabfac0c 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -214,12 +214,18 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) const struct kunit_case *test_case; enum kunit_status status = KUNIT_SKIPPED; + if (suite->status == KUNIT_SKIPPED) + return KUNIT_SKIPPED; + if (suite->suite_init_err) return KUNIT_FAILURE; kunit_suite_for_each_test_case(suite, test_case) { - if (test_case->status == KUNIT_FAILURE) + if (test_case->status == KUNIT_FAILURE) { + /* Update the kunit_suite status also */ + suite->status = KUNIT_FAILURE; return KUNIT_FAILURE; + } else if (test_case->status == KUNIT_SUCCESS) status = KUNIT_SUCCESS; } @@ -795,12 +801,20 @@ int kunit_run_tests(struct kunit_suite *suite) /* Taint the kernel so we know we've run tests. */ add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + if (suite->status == KUNIT_SKIPPED) + goto suite_end; + if (suite->suite_init) { suite->suite_init_err = suite->suite_init(suite); if (suite->suite_init_err) { + suite->status = KUNIT_FAILURE; kunit_err(suite, KUNIT_SUBTEST_INDENT "# failed to initialize (%d)", suite->suite_init_err); goto suite_end; + + } else if (suite->status == KUNIT_SKIPPED) { + /* Skip this kunit suite */ + goto suite_end; } } @@ -825,6 +839,7 @@ static void kunit_init_suite(struct kunit_suite *suite) kunit_debugfs_create_suite(suite); suite->status_comment[0] = '\0'; suite->suite_init_err = 0; + suite->status = KUNIT_SUCCESS; if (suite->log) string_stream_clear(suite->log); diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77..06cc05b79fbd 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) return mas->status == ma_underflow; } +static inline void mas_make_walkable(struct ma_state *mas) +{ + if (!mas_is_active(mas) && !mas_is_start(mas)) + mas->status = ma_start; +} + static __always_inline struct maple_node *mte_to_node( const struct maple_enode *entry) { @@ -455,46 +461,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) return 0; } -/* - * mas_set_parent() - Set the parent node and encode the slot - * @mas: The maple state - * @enode: The encoded maple node. - * @parent: The encoded maple node that is the parent of @enode. - * @slot: The slot that @enode resides in @parent. - * - * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the - * parent type. - */ -static inline -void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, - const struct maple_enode *parent, unsigned char slot) -{ - unsigned long val = (unsigned long)parent; - unsigned long shift; - unsigned long type; - enum maple_type p_type = mte_node_type(parent); - - MAS_BUG_ON(mas, p_type == maple_dense); - MAS_BUG_ON(mas, p_type == maple_leaf_64); - - switch (p_type) { - case maple_range_64: - case maple_arange_64: - shift = MAPLE_PARENT_SLOT_SHIFT; - type = MAPLE_PARENT_RANGE64; - break; - default: - case maple_dense: - case maple_leaf_64: - shift = type = 0; - break; - } - - val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ - val |= (slot << shift) | type; - mte_to_node(enode)->parent = ma_parent_ptr(val); -} - /* * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. @@ -876,6 +842,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, meta->gap = offset; } +/* + * mas_set_parent_slots() - Bulk operation to set many slot parent pointers + * @mas: The maple state + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that of the @enode. + * @start_slot: The offset into @slot + * @count: The number of slots to set (eg: exclusive) + */ +static inline +void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent, + void __rcu **slots, unsigned char start_slot, unsigned char count) +{ + unsigned long val; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + unsigned char i; + + MAS_BUG_ON(mas, p_type != maple_range_64 && + p_type != maple_arange_64); + + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + + val = (unsigned long)parent; + val &= ~MAPLE_NODE_MASK; + + for (i = 0; i < count; i++) { + unsigned long pval = val | ((start_slot + i) << shift) | type; + struct maple_enode *child; + + child = mt_slot_locked(mas->tree, slots, i); + mte_to_node(child)->parent = ma_parent_ptr(pval); + } +} + /* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. * @mat: the ma_topiary, a linked list of dead nodes. @@ -1153,6 +1155,73 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ma_free_rcu(mte_to_node(used)); } + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map *mas_lockdep_map(struct ma_state *mas) +{ + struct maple_tree *mt = mas->tree; + + if (mt_external_lock(mt)) + return mt->ma_external_lock; + + return &(mt->ma_lock).dep_map; +} + +#endif + +static void mas_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; + if (!mas_is_active(mas)) + return; + +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp)); + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map && lock_is_held(map)) + WARN_ON_ONCE(mas->ld_seq != lock_sequence(map)); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + mas->rcu_gp = get_state_synchronize_rcu(); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map && lock_is_held(map)) + mas->ld_seq = lock_sequence(mas_lockdep_map(mas)); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_may_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (mas_is_start(mas) || mas_is_paused(mas)) { + mas_init_lock_check(mas); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + mas_lock_check(mas); +#endif /* CONFIG_LOCKDEP */ +} + /* * mas_start() - Sets up maple state for operations. * @mas: The maple state. @@ -1171,6 +1240,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) if (likely(mas_is_start(mas))) { struct maple_enode *root; + mas_init_lock_check(mas); mas->min = 0; mas->max = ULONG_MAX; @@ -1501,14 +1571,26 @@ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, goto ascend; } +static __always_inline void mas_update_gap_known(struct ma_state *mas, + unsigned long gap) +{ + unsigned char pslot; + unsigned long p_gap; + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != gap) + mas_parent_gap(mas, pslot, gap); +} + /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. * @mas: the maple state. */ static inline void mas_update_gap(struct ma_state *mas) { - unsigned char pslot; - unsigned long p_gap; unsigned long max_gap; if (!mt_is_alloc(mas->tree)) @@ -1518,13 +1600,7 @@ static inline void mas_update_gap(struct ma_state *mas) return; max_gap = mas_max_gap(mas); - - pslot = mte_parent_slot(mas->node); - p_gap = ma_gaps(mte_parent(mas->node), - mas_parent_type(mas, mas->node))[pslot]; - - if (p_gap != max_gap) - mas_parent_gap(mas, pslot, max_gap); + mas_update_gap_known(mas, max_gap); } /* @@ -1540,14 +1616,10 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); - struct maple_enode *child; - unsigned char offset; + unsigned char end; - offset = ma_data_end(node, type, pivots, mas->max); - do { - child = mas_slot_locked(mas, slots, offset); - mas_set_parent(mas, child, parent, offset); - } while (offset--); + end = ma_data_end(node, type, pivots, mas->max); + mas_set_parent_slots(mas, parent, slots, 0, end + 1); } /* @@ -1929,15 +2001,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src, s_slots = ma_slots(src, s_mt) + start; s_pivots = ma_pivots(src, s_mt) + start; memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); - if (!ma_is_leaf(d_mt) && s_mt == maple_copy) { - struct maple_enode *edst = mt_mk_node(dst, d_mt); - - for (int i = 0; i < size; i++) - mas_set_parent(mas, - mt_slot_locked(mas->tree, d_slots, i), - edst, d_start + i); - } + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) + mas_set_parent_slots(mas, mt_mk_node(dst, d_mt), + d_slots, d_start, size); d_gaps = ma_gaps(dst, d_mt); if (d_gaps) { @@ -2081,8 +2148,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp) mas->node = mt_slot_locked(mas->tree, cp->slot, 0); /* Insert the new data in the tree */ mas_topiary_replace(mas, old_enode, cp->height); - if (!mte_is_leaf(mas->node)) - mas_update_gap(mas); + if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node)) + mas_update_gap_known(mas, cp->gap[0]); mtree_range_walk(mas); } @@ -2928,13 +2995,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, if (r_mas->last < r_wr_mas->r_max) r_mas->last = r_wr_mas->r_max; r_mas->offset++; - } else if ((r_mas->last == r_wr_mas->r_max) && - (r_mas->last < r_mas->max) && - !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { - r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, - r_wr_mas->type, r_mas->offset + 1); - r_mas->offset++; - r_wr_mas->r_max = r_mas->last; } } @@ -3125,7 +3185,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) { unsigned char dst_offset, offset_end; - unsigned char copy_size, node_pivots; + unsigned char copy_size, node_pivots, node_slots; struct maple_node reuse, *newnode; unsigned long *dst_pivots; void __rcu **dst_slots; @@ -3138,6 +3198,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) in_rcu = mt_in_rcu(mas->tree); offset_end = wr_mas->offset_end; node_pivots = mt_pivots[wr_mas->type]; + node_slots = mt_slots[wr_mas->type]; /* Assume last adds an entry */ new_end = mas->end + 1 - offset_end + mas->offset; if (mas->last == wr_mas->end_piv) { @@ -3149,7 +3210,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) if (in_rcu) { newnode = mas_pop_node(mas); } else { - memset(&reuse, 0, sizeof(struct maple_node)); newnode = &reuse; } @@ -3193,7 +3253,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) dst_pivots[new_end] = mas->max; done: - mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (!in_rcu && new_end + 2 < node_slots) { + unsigned char clear_from = new_end + 1; + + /* + * Note that the last slot is never cleared, since the metadata + * will be stored there or it has a value. + */ + memset(dst_slots + clear_from, 0, + sizeof(void __rcu *) * (node_slots - clear_from)); + if (clear_from < node_pivots) + memset(dst_pivots + clear_from, 0, + sizeof(unsigned long) * (node_pivots - clear_from)); + } + + mas_leaf_set_meta(newnode, wr_mas->type, new_end); if (in_rcu) { struct maple_enode *old_enode = mas->node; @@ -3218,7 +3292,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) void __rcu **slots = wr_mas->slots; bool gap = false; - gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !wr_mas->content; gap |= !mt_slot_locked(mas->tree, slots, offset + 1); if (wr_mas->offset_end - offset == 1) { @@ -3661,6 +3735,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char new_end; + bool appending; + bool one_slot; + bool in_rcu; if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) return wr_store_root; @@ -3680,21 +3757,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) return wr_new_root; new_end = mas_wr_new_end(wr_mas); + in_rcu = mt_in_rcu(mas->tree); + appending = mas->offset == mas->end; + one_slot = wr_mas->offset_end - mas->offset == 1; + /* Potential spanning rebalance collapsing a node */ if (new_end < mt_min_slots[wr_mas->type]) { if (!mte_is_root(mas->node)) return wr_rebalance; + if (!in_rcu) { + if (appending) + return wr_append; + else if (mas->end == new_end && one_slot) + return wr_slot_store; + } return wr_node_store; } if (new_end >= mt_slots[wr_mas->type]) return wr_split_store; - if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end)) + if (!in_rcu && appending) return wr_append; - if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) || - (wr_mas->offset_end - mas->offset == 1))) + if (new_end == mas->end && (!in_rcu || one_slot)) return wr_slot_store; return wr_node_store; @@ -3793,35 +3879,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } @@ -4360,8 +4451,8 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (!mas_is_active(mas) && !mas_is_start(mas)) - mas->status = ma_start; + mas_may_init_lock_check(mas); + mas_make_walkable(mas); retry: entry = mas_state_walk(mas); if (mas_is_start(mas)) { @@ -4831,6 +4922,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) @@ -4889,6 +4981,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); int ret = 0; + mas_may_init_lock_check(mas); retry: mas_wr_preallocate(&wr_mas, entry); if (unlikely(mas_nomem(mas, gfp))) { @@ -4919,6 +5012,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_lock_check(mas); if (mas->store_type == wr_store_root) { mas_wr_prealloc_setup(&wr_mas); goto store; @@ -4951,6 +5045,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); mas_wr_prealloc_setup(&wr_mas); mas->store_type = mas_wr_store_type(&wr_mas); mas_prealloc_calc(&wr_mas, entry); @@ -4997,6 +5092,7 @@ static void mas_may_activate(struct ma_state *mas) mas->status = ma_start; } else { mas->status = ma_active; + mas_lock_check(mas); } } @@ -5074,6 +5170,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5097,6 +5194,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5205,6 +5303,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5228,6 +5327,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5274,6 +5374,7 @@ EXPORT_SYMBOL_GPL(mt_prev); */ void mas_pause(struct ma_state *mas) { + mas_lock_check(mas); mas->status = ma_pause; mas->node = NULL; } @@ -5382,6 +5483,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5409,6 +5511,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5428,7 +5531,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - switch (mas->status) { case ma_active: goto active; @@ -5521,6 +5623,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5547,6 +5650,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5564,6 +5668,10 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); * Searches for @mas->index, sets @mas->index and @mas->last to the range and * erases that range. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. */ void *mas_erase(struct ma_state *mas) @@ -5572,13 +5680,19 @@ void *mas_erase(struct ma_state *mas) unsigned long index = mas->index; MA_WR_STATE(wr_mas, mas, NULL); - if (!mas_is_active(mas) || !mas_is_start(mas)) - mas->status = ma_start; + /* + * In low memory situations, the allocation is retried with the gfp flag + * GFP_KERNEL. The internal spinlock is dropped in mas_nomem(), however + * the external lock is not dropped. + */ + if (mt_external_lock(mas->tree)) + might_alloc(GFP_KERNEL); + mas_make_walkable(mas); write_retry: entry = mas_state_walk(mas); if (!entry) - return NULL; + goto out; /* Must reset to ensure spanning writes of last slot are detected */ mas_reset(mas); @@ -5589,8 +5703,10 @@ void *mas_erase(struct ma_state *mas) goto write_retry; } - if (mas_is_err(mas)) + if (mas_is_err(mas)) { + entry = NULL; goto out; + } mas_wr_store_entry(&wr_mas); out: @@ -5601,10 +5717,11 @@ EXPORT_SYMBOL_GPL(mas_erase); /** * mas_nomem() - Check if there was an error allocating and do the allocation - * if necessary If there are allocations, then free them. + * if necessary. + * * @mas: The maple state * @gfp: The GFP_FLAGS to use for allocations - * Return: true on allocation, false otherwise. + * Return: False on no memory. True otherwise (partial success as well) */ bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) @@ -5612,6 +5729,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) if (likely(mas->node != MA_ERROR(-ENOMEM))) return false; + /* Allocations can fail, don't do this. */ + WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) && + mt_external_lock(mas->tree)); + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree); mas_alloc_nodes(mas, gfp); @@ -5620,10 +5741,17 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) mas_alloc_nodes(mas, gfp); } - if (!mas->sheaf && !mas->alloc) + /* + * Return false on zero forward progress. Partial allocations are kept + * so the retry path will attempt to get the rest. The failure should + * not happen as we try our best to reclaim. The user would need an + * external lock with a non-blocking gfp in a low memory situation - + * which would have triggered the first warning in this function. + */ + if (WARN_ON_ONCE(!mas->sheaf && !mas->alloc)) return false; - mas->status = ma_start; + mas_reset(mas); return true; } @@ -5913,6 +6041,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); * Erasing is the same as a walk to an entry then a store of a NULL to that * ENTIRE range. In fact, it is implemented as such using the advanced API. * + * Note that erase requires allocations and will use GFP_KERNEL to do so if + * necessary. If the allocation fails, the internal lock will be dropped to + * retry. + * * Return: The entry stored at the @index or %NULL */ void *mtree_erase(struct maple_tree *mt, unsigned long index) @@ -5922,6 +6054,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); trace_ma_op(TP_FCT, &mas); + might_alloc(GFP_KERNEL); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); diff --git a/lib/math/div64.c b/lib/math/div64.c index d1e92ea24fce..0b10ded09a9b 100644 --- a/lib/math/div64.c +++ b/lib/math/div64.c @@ -158,6 +158,21 @@ u64 div64_u64(u64 dividend, u64 divisor) EXPORT_SYMBOL(div64_u64); #endif +#ifndef div64_s64_rem +s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder) +{ + s64 quot, t, rem; + + quot = div64_u64_rem(abs(dividend), abs(divisor), (u64 *)&rem); + t = dividend >> 63; + *remainder = (rem ^ t) - t; + t = (dividend ^ divisor) >> 63; + + return (quot ^ t) - t; +} +EXPORT_SYMBOL(div64_s64_rem); +#endif + #ifndef div64_s64 s64 div64_s64(s64 dividend, s64 divisor) { diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c index 338d014f0c73..d12dc05938fb 100644 --- a/lib/math/test_mul_u64_u64_div_u64.c +++ b/lib/math/test_mul_u64_u64_div_u64.c @@ -157,6 +157,7 @@ static void __exit test_exit(void) #define __div64_32 __div64_32 #define div_s64_rem div_s64_rem #define div64_u64_rem div64_u64_rem +#define div64_s64_rem div64_s64_rem #define div64_u64 div64_u64 #define div64_s64 div64_s64 #define iter_div_u64_rem iter_div_u64_rem diff --git a/lib/math/tests/Makefile b/lib/math/tests/Makefile index 13dc96e48408..85e1ad59f29d 100644 --- a/lib/math/tests/Makefile +++ b/lib/math/tests/Makefile @@ -4,5 +4,6 @@ obj-$(CONFIG_GCD_KUNIT_TEST) += gcd_kunit.o obj-$(CONFIG_INT_LOG_KUNIT_TEST) += int_log_kunit.o obj-$(CONFIG_INT_POW_KUNIT_TEST) += int_pow_kunit.o obj-$(CONFIG_INT_SQRT_KUNIT_TEST) += int_sqrt_kunit.o +obj-$(CONFIG_POLYNOMIAL_KUNIT_TEST) += polynomial_kunit.o obj-$(CONFIG_PRIME_NUMBERS_KUNIT_TEST) += prime_numbers_kunit.o obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational_kunit.o diff --git a/lib/math/tests/polynomial_kunit.c b/lib/math/tests/polynomial_kunit.c new file mode 100644 index 000000000000..ef443b57fc12 --- /dev/null +++ b/lib/math/tests/polynomial_kunit.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +struct polynomial_test_param { + const struct polynomial *poly; + long data; + long expected; + const char *name; +}; + +/* f(x) = 5 */ +static const struct polynomial poly_constant = { + .total_divider = 1, + .terms = { + {0, 5, 1, 1}, + } +}; + +/* f(x) = 2x^2 + 3x + 5 */ +static const struct polynomial poly_simple = { + .total_divider = 1, + .terms = { + {2, 2, 1, 1}, + {1, 3, 1, 1}, + {0, 5, 1, 1}, + } +}; + +/* f(x) = -5x + 100 */ +static const struct polynomial poly_negative_coef = { + .total_divider = 1, + .terms = { + {1, -5, 1, 1}, + {0, 100, 1, 1}, + } +}; + +/* f(x) = (150x + 50) / 10 */ +static const struct polynomial poly_total_divider = { + .total_divider = 10, + .terms = { + {1, 150, 1, 1}, + {0, 50, 1, 1}, + } +}; + +/* + * f(x) = x / 2 + * divider=2 applied once per multiply: mult_frac(coef, data, 2) = coef*data/2 + */ +static const struct polynomial poly_step_divider = { + .total_divider = 1, + .terms = { + {1, 1, 2, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = (100/500) * x^2 = 0.2 * x^2 + * Encoded as coef=100, divider=10, divider_leftover=5: + * denom = 10^2 * 5 = 500 + */ +static const struct polynomial poly_leftover = { + .total_divider = 1, + .terms = { + {2, 100, 10, 5}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 2x^3 (single high-degree term, no constant) + * Used to exercise the power loop alone. + */ +static const struct polynomial poly_cubic = { + .total_divider = 1, + .terms = { + {3, 2, 1, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 4x + 1 with a zero-coefficient quadratic term. + * The deg-2 term contributes nothing regardless of input. + */ +static const struct polynomial poly_zero_coef = { + .total_divider = 1, + .terms = { + {2, 0, 1, 1}, + {1, 4, 1, 1}, + {0, 1, 1, 1}, + } +}; + +/* + * f(x) = 9 with total_divider = 0. + * The implementation treats 0 as 1 via `total_divider ?: 1`, so the + * result must equal the constant term unchanged. + */ +static const struct polynomial poly_zero_total_divider = { + .total_divider = 0, + .terms = { + {0, 9, 1, 1}, + } +}; + + +static const struct polynomial_test_param test_params[] = { + { + .poly = &poly_constant, + .data = 0, + .expected = 5, + .name = "Constant polynomial at x=0", + }, + { + .poly = &poly_constant, + .data = 42, + .expected = 5, + .name = "Constant polynomial is independent of input", + }, + { + .poly = &poly_simple, + .data = 0, + .expected = 5, /* zero input collapses all power terms */ + .name = "Zero input yields constant term only", + }, + { + .poly = &poly_simple, + .data = 10, + .expected = 235, /* 2*100 + 3*10 + 5 */ + .name = "Simple quadratic at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 10, + .expected = 50, /* -5*10 + 100 */ + .name = "Negative coefficient at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 20, + .expected = 0, /* -5*20 + 100 = 0 */ + .name = "Negative coefficient result is zero", + }, + { + .poly = &poly_total_divider, + .data = 3, + .expected = 50, /* (150*3 + 50) / 10 = 500/10 */ + .name = "total_divider scales the final sum", + }, + { + .poly = &poly_step_divider, + .data = 100, + .expected = 50, /* 1*100/2 */ + .name = "Per-step divider halves input", + }, + { + .poly = &poly_leftover, + .data = 30, + .expected = 180, /* 100*30^2 / (10^2 * 5) = 90000/500 */ + .name = "divider_leftover with quadratic term", + }, + /* Boundary: unit and negative-unit input */ + { + /* + * data=1: each mult_frac(tmp, 1, divider) strips one factor of + * divider from coef per degree, so coef is left-shifted right + * until intermediate precision is exhausted. + * 2*1 + 3*1 + 5 = 10 + */ + .poly = &poly_simple, + .data = 1, + .expected = 10, + .name = "Boundary: data=1 (unit input)", + }, + { + /* + * data=-1: even degrees produce positive contributions, + * odd degrees produce negative ones. + * 2*(-1)^2 + 3*(-1) + 5 = 2 - 3 + 5 = 4 + */ + .poly = &poly_simple, + .data = -1, + .expected = 4, + .name = "Boundary: data=-1 (negative unit input)", + }, + + /* Boundary: negative non-trivial input */ + { + /* + * 2*(-3)^2 + 3*(-3) + 5 = 18 - 9 + 5 = 14 + * Verifies sign handling for negative data across all degrees. + */ + .poly = &poly_simple, + .data = -3, + .expected = 14, + .name = "Boundary: negative data with quadratic", + }, + + /* Boundary: total_divider = 0 is treated as 1 */ + { + .poly = &poly_zero_total_divider, + .data = 42, + .expected = 9, + .name = "Boundary: total_divider=0 defaults to 1", + }, + + /* Boundary: zero-coefficient high-degree term */ + { + /* + * The deg-2 term has coef=0, so it contributes 0 regardless + * of data. Result: 0 + 4*10 + 1 = 41 + */ + .poly = &poly_zero_coef, + .data = 10, + .expected = 41, + .name = "Boundary: zero-coefficient term is inert", + }, + + /* Boundary: single high-degree term, no constant */ + { + /* 2 * 5^3 = 250; also verifies the loop terminates on deg-0 */ + .poly = &poly_cubic, + .data = 5, + .expected = 250, + .name = "Boundary: single cubic term", + }, + { + /* 2 * (-2)^3 = -16; odd power preserves sign of negative data */ + .poly = &poly_cubic, + .data = -2, + .expected = -16, + .name = "Boundary: single cubic term, negative data", + }, + +}; + +static void get_desc(const struct polynomial_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(polynomial, test_params, get_desc); + +static void polynomial_calc_test(struct kunit *test) +{ + const struct polynomial_test_param *param = test->param_value; + + KUNIT_EXPECT_EQ(test, polynomial_calc(param->poly, param->data), + param->expected); +} + +static struct kunit_case polynomial_test_cases[] = { + KUNIT_CASE_PARAM(polynomial_calc_test, polynomial_gen_params), + {} +}; + +static struct kunit_suite polynomial_test_suite = { + .name = "math-polynomial", + .test_cases = polynomial_test_cases, +}; + +kunit_test_suites(&polynomial_test_suite); + +MODULE_DESCRIPTION("math.polynomial_calc KUnit test suite"); +MODULE_LICENSE("GPL"); diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 33c154264bfe..a3bfa9360b23 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,8 @@ static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; /* "in progress" flag of arch_trigger_cpumask_backtrace */ static unsigned long backtrace_flag; +#define NMI_BT_TIMEOUT_SEC 10 + /* * When raise() is called it will be passed a pointer to the * backtrace_mask. Architectures that call nmi_cpu_backtrace() @@ -68,14 +71,20 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, raise(to_cpumask(backtrace_mask)); } - /* Wait for up to 10 seconds for all CPUs to do the backtrace */ - for (i = 0; i < 10 * 1000; i++) { + /* Wait for up to NMI_BT_TIMEOUT_SEC seconds for all CPUs to do the backtrace */ + for (i = 0; i < NMI_BT_TIMEOUT_SEC * 1000; i++) { if (cpumask_empty(to_cpumask(backtrace_mask))) break; mdelay(1); touch_softlockup_watchdog(); } - nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + + if (!cpumask_empty(to_cpumask(backtrace_mask))) { + pr_warn("After " __stringify(NMI_BT_TIMEOUT_SEC) " seconds, these CPUS still haven't responded to the NMI: %*pbl\n", + cpumask_pr_args(to_cpumask(backtrace_mask))); + + nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + } /* * Force flush any remote buffers that might be stuck in IRQ context diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 97772e42b9b2..f8d90689af9f 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. - * Use percpu_ref_kill[_and_confirm](). + * Use percpu_ref_kill() or percpu_ref_kill_and_confirm(). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig index 5ab2b0a7be4c..978cd6ba08ac 100644 --- a/lib/raid/Kconfig +++ b/lib/raid/Kconfig @@ -28,3 +28,36 @@ config XOR_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. + +config RAID6_PQ + tristate + +# selected by architectures that provide an optimized PQ implementation +config RAID6_PQ_ARCH + depends on RAID6_PQ + default y if KERNEL_MODE_NEON # arm32/arm64 + default y if LOONGARCH + default y if ALTIVEC # powerpc + default y if RISCV_ISA_V + default y if S390 + default y if X86 + bool + +config RAID6_PQ_KUNIT_TEST + tristate "KUnit tests for RAID6 PQ functions" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RAID6_PQ + default KUNIT_ALL_TESTS + help + Unit tests for the RAID6 PQ library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. diff --git a/lib/raid/Makefile b/lib/raid/Makefile index 3540fe846dc4..6fc5eeb53df0 100644 --- a/lib/raid/Makefile +++ b/lib/raid/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += xor/ +obj-y += xor/ raid6/ diff --git a/lib/raid6/.gitignore b/lib/raid/raid6/.gitignore similarity index 100% rename from lib/raid6/.gitignore rename to lib/raid/raid6/.gitignore diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile new file mode 100644 index 000000000000..038d6c74d1ba --- /dev/null +++ b/lib/raid/raid6/Makefile @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0 + +ccflags-y += -I $(src) + +ifeq ($(CONFIG_RAID6_PQ_ARCH),y) +CFLAGS_algos.o += -I$(src)/$(SRCARCH) +endif + +obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/ + +raid6_pq-y += algos.o tables.o + +# generic integer generation and recovery implementation +raid6_pq-y += int1.o int2.o int4.o int8.o +raid6_pq-y += recov.o + +# architecture-specific generation and recovery implementations: +raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \ + arm/neon1.o \ + arm/neon2.o \ + arm/neon4.o \ + arm/neon8.o \ + arm/recov_neon.o \ + arm/recov_neon_inner.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch/loongarch_simd.o \ + loongarch/recov_loongarch_simd.o +raid6_pq-$(CONFIG_ALTIVEC) += powerpc/altivec1.o \ + powerpc/altivec2.o \ + powerpc/altivec4.o \ + powerpc/altivec8.o \ + powerpc/vpermxor1.o \ + powerpc/vpermxor2.o \ + powerpc/vpermxor4.o \ + powerpc/vpermxor8.o +raid6_pq-$(CONFIG_RISCV_ISA_V) += riscv/rvv.o \ + riscv/recov_rvv.o +raid6_pq-$(CONFIG_S390) += s390/s390vx8.o \ + s390/recov_s390xc.o +ifeq ($(CONFIG_X86),y) +raid6_pq-$(CONFIG_X86_32) += x86/mmx.o \ + x86/sse1.o +endif +raid6_pq-$(CONFIG_X86) += x86/sse2.o \ + x86/avx2.o \ + x86/avx512.o \ + x86/recov_ssse3.o \ + x86/recov_avx2.o \ + x86/recov_avx512.o + +hostprogs += mktables + +CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU) +CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU) + +ifeq ($(CONFIG_ALTIVEC),y) +altivec_flags := -maltivec $(call cc-option,-mabi=altivec) +# Enable +altivec_flags += -isystem $(shell $(CC) -print-file-name=include) + +CFLAGS_powerpc/altivec1.o += $(altivec_flags) +CFLAGS_powerpc/altivec2.o += $(altivec_flags) +CFLAGS_powerpc/altivec4.o += $(altivec_flags) +CFLAGS_powerpc/altivec8.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor1.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor2.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor4.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor8.o += $(altivec_flags) + +ifdef CONFIG_CC_IS_CLANG +# clang ppc port does not yet support -maltivec when -msoft-float is +# enabled. A future release of clang will resolve this +# https://llvm.org/pr31177 +CFLAGS_REMOVE_powerpc/altivec1.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec2.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec4.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec8.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor1.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor2.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor4.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor8.o += -msoft-float +endif # CONFIG_CC_IS_CLANG +endif # CONFIG_ALTIVEC + +quiet_cmd_mktable = TABLE $@ + cmd_mktable = $(obj)/mktables > $@ + +targets += tables.c +$(obj)/tables.c: $(obj)/mktables FORCE + $(call if_changed,mktable) + +quiet_cmd_unroll = UNROLL $@ + cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ + +targets += int1.c int2.c int4.c int8.c +$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c +$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/altivec1.c \ + powerpc/altivec2.c \ + powerpc/altivec4.c \ + powerpc/altivec8.c +$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/vpermxor1.c \ + powerpc/vpermxor2.c \ + powerpc/vpermxor4.c \ + powerpc/vpermxor8.c +$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += s390/s390vx8.c +$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c new file mode 100644 index 000000000000..6f5c89ab2b17 --- /dev/null +++ b/lib/raid/raid6/algos.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002 H. Peter Anvin - All Rights Reserved + * + * Algorithm list and algorithm selection for RAID-6 + */ + +#include +#include +#include +#include +#include +#include +#include "algos.h" + +#define RAID6_MAX_ALGOS 16 +static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; +static unsigned int raid6_nr_algos; +static const struct raid6_recov_calls *raid6_recov_algo; + +/* Selected algorithm */ +DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); +DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); + +/** + * raid6_gen_syndrome - generate RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to + * @ptrs[@disks - 3]. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. + * + * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying + * algorithm. + */ +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + + static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_gen_syndrome); + +/** + * raid6_xor_syndrome - update RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @start: first index into @disk to update + * @stop: last index into @disk to update + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively for the memory pointed to by + * @ptrs[@start..@stop]. + * + * This is used to update parity in place using the following sequence: + * + * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. + * 2) update the the data in @ptrs[@start..@stop]. + * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. + * + * Data between @start and @stop that is not changed should be filled + * with a pointer to the kernel zero page. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. @stop must be larger or equal to @start. + */ +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + WARN_ON_ONCE(stop < start); + + static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_xor_syndrome); + +/* + * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used + * + * Returns %true if raid6_can_xor_syndrome() can be used, else %false. + */ +bool raid6_can_xor_syndrome(void) +{ + return !!static_call_query(raid6_xor_syndrome_impl); +} +EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); + +/** + * raid6_recov_2data - recover two missing data disks + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: first failed data disk index + * @failb: second failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the + * data in the remaining disks and the two parities pointed to by the other + * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks + * and the two parities. @faila must be smaller than @failb. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + WARN_ON_ONCE(failb <= faila); + + static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_2data); + +/** + * raid6_recov_datap - recover a missing data disk and missing P-parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in + * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity + * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks + * includes the data disks and the two parities. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + + static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_datap); + +#define RAID6_TIME_JIFFIES_LG2 4 +#define RAID6_TEST_DISKS 8 + +static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], + const int disks) +{ + /* work on the second half of the disks */ + int start = (disks >> 1) - 1, stop = disks - 3; + const struct raid6_calls *best = NULL; + unsigned long bestgenperf = 0; + unsigned int i; + + for (i = 0; i < raid6_nr_algos; i++) { + const struct raid6_calls *algo = raid6_algos[i]; + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1<gen_syndrome(disks, PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + if (perf > bestgenperf) { + bestgenperf = perf; + best = algo; + } + pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + } + + if (!best) { + pr_err("raid6: Yikes! No algorithm found!\n"); + return -EINVAL; + } + + static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); + + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, + (bestgenperf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + + if (best->xor_syndrome) { + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { + best->xor_syndrome(disks, start, stop, + PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", + (perf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + } + + return 0; +} + + +/* Try to pick the best algorithm */ +/* This code uses the gfmul table as convenient data set to abuse */ + +static int __init raid6_select_algo(void) +{ + const int disks = RAID6_TEST_DISKS; + char *disk_ptr, *p; + void *dptrs[RAID6_TEST_DISKS]; + int i, cycle; + int error; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { + pr_info("raid6: skipped pq benchmark and selected %s\n", + raid6_algos[raid6_nr_algos - 1]->name); + static_call_update(raid6_gen_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->xor_syndrome); + return 0; + } + + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = kmalloc(PAGE_SIZE * RAID6_TEST_DISKS, GFP_KERNEL); + if (!disk_ptr) { + pr_err("raid6: Yikes! No memory available.\n"); + return -ENOMEM; + } + + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + PAGE_SIZE * i; + + cycle = ((disks - 2) * PAGE_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * PAGE_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); + + /* select raid gen_syndrome function */ + error = raid6_choose_gen(&dptrs, disks); + + kfree(disk_ptr); + + return error; +} + +/* + * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled + * implementation should be registered last so it will be selected when the + * boot-time benchmark is disabled. + */ +void __init raid6_algo_add(const struct raid6_calls *algo) +{ + if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) + return; + raid6_algos[raid6_nr_algos++] = algo; +} + +void __init raid6_algo_add_default(void) +{ + raid6_algo_add(&raid6_intx1); + raid6_algo_add(&raid6_intx2); + raid6_algo_add(&raid6_intx4); + raid6_algo_add(&raid6_intx8); +} + +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) +{ + if (WARN_ON_ONCE(raid6_recov_algo)) + return; + raid6_recov_algo = algo; +} + +#ifdef CONFIG_RAID6_PQ_ARCH +#include "pq_arch.h" +#else +static inline void arch_raid6_init(void) +{ + raid6_algo_add_default(); +} +#endif /* CONFIG_RAID6_PQ_ARCH */ + +static int __init raid6_init(void) +{ + /* + * Architectures providing arch_raid6_init must add all PQ generation + * algorithms they want to consider in arch_raid6_init(), including + * the generic ones using raid6_algo_add_default() if wanted. + */ + arch_raid6_init(); + + /* + * Architectures don't have to set a recovery algorithm, we'll just pick + * the generic integer one if none was set. + */ + if (!raid6_recov_algo) + raid6_recov_algo = &raid6_recov_intx1; + static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); + static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); + pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); + + return raid6_select_algo(); +} + +static void __exit raid6_exit(void) +{ +} + +subsys_initcall(raid6_init); +module_exit(raid6_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); + +#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) +const struct raid6_calls *raid6_algo_find(unsigned int idx) +{ + if (idx >= raid6_nr_algos) { + /* + * Always include the simplest generic integer implementation in + * the unit tests as a baseline. + */ + if (idx == raid6_nr_algos && + raid6_algos[0] != &raid6_intx1) + return &raid6_intx1; + return NULL; + } + return raid6_algos[idx]; +} +EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); + +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) +{ + switch (idx) { + case 0: + /* always test the generic integer implementation */ + return &raid6_recov_intx1; + case 1: + /* test the optimized implementation if there is one */ + if (raid6_recov_algo != &raid6_recov_intx1) + return raid6_recov_algo; + return NULL; + default: + return NULL; + } +} +EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); +#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h new file mode 100644 index 000000000000..43f636be183f --- /dev/null +++ b/lib/raid/raid6/algos.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + */ +#ifndef _PQ_IMPL_H +#define _PQ_IMPL_H + +#include +#include + +/* Routine choices */ +struct raid6_calls { + const char *name; + void (*gen_syndrome)(int disks, size_t bytes, void **ptrs); + void (*xor_syndrome)(int disks, int start, int stop, size_t bytes, + void **ptrs); +}; + +struct raid6_recov_calls { + const char *name; + void (*data2)(int disks, size_t bytes, int faila, int failb, + void **ptrs); + void (*datap)(int disks, size_t bytes, int faila, void **ptrs); +}; + +void __init raid6_algo_add(const struct raid6_calls *algo); +void __init raid6_algo_add_default(void); +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo); + +/* for the kunit test */ +const struct raid6_calls *raid6_algo_find(unsigned int idx); +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx); + +/* generic implementations */ +extern const struct raid6_calls raid6_intx1; +extern const struct raid6_calls raid6_intx2; +extern const struct raid6_calls raid6_intx4; +extern const struct raid6_calls raid6_intx8; +extern const struct raid6_recov_calls raid6_recov_intx1; + +#endif /* _PQ_IMPL_H */ diff --git a/lib/raid6/neon.c b/lib/raid/raid6/arm/neon.c similarity index 79% rename from lib/raid6/neon.c rename to lib/raid/raid6/arm/neon.c index 6d9474ce6da9..af90869aaffc 100644 --- a/lib/raid6/neon.c +++ b/lib/raid/raid6/arm/neon.c @@ -1,18 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics + * RAID6 syndrome calculation using ARM NEON intrinsics * * Copyright (C) 2013 Linaro Ltd */ -#include - -#ifdef __KERNEL__ #include -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif +#include "algos.h" /* * There are 2 reasons these wrappers are kept in a separate compilation unit @@ -46,18 +40,11 @@ start, stop, (unsigned long)bytes, ptrs);\ } \ struct raid6_calls const raid6_neonx ## _n = { \ - raid6_neon ## _n ## _gen_syndrome, \ - raid6_neon ## _n ## _xor_syndrome, \ - raid6_have_neon, \ - "neonx" #_n, \ - 0 \ + .gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \ + .name = "neonx" #_n, \ } -static int raid6_have_neon(void) -{ - return cpu_has_neon(); -} - RAID6_NEON_WRAPPER(1); RAID6_NEON_WRAPPER(2); RAID6_NEON_WRAPPER(4); diff --git a/lib/raid6/neon.h b/lib/raid/raid6/arm/neon.h similarity index 100% rename from lib/raid6/neon.h rename to lib/raid/raid6/arm/neon.h diff --git a/lib/raid6/neon.uc b/lib/raid/raid6/arm/neon.uc similarity index 99% rename from lib/raid6/neon.uc rename to lib/raid/raid6/arm/neon.uc index 355270af0cd6..14a9fc2c60fa 100644 --- a/lib/raid6/neon.uc +++ b/lib/raid/raid6/arm/neon.uc @@ -25,7 +25,7 @@ */ #include -#include "neon.h" +#include "arm/neon.h" typedef uint8x16_t unative_t; diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h new file mode 100644 index 000000000000..3f876ea6749c --- /dev/null +++ b/lib/raid/raid6/arm/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_neonx1; +extern const struct raid6_calls raid6_neonx2; +extern const struct raid6_calls raid6_neonx4; +extern const struct raid6_calls raid6_neonx8; +extern const struct raid6_recov_calls raid6_recov_neon; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (cpu_has_neon()) { + raid6_algo_add(&raid6_neonx1); + raid6_algo_add(&raid6_neonx2); + raid6_algo_add(&raid6_neonx4); + raid6_algo_add(&raid6_neonx8); + raid6_recov_algo_add(&raid6_recov_neon); + } +} diff --git a/lib/raid6/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c similarity index 81% rename from lib/raid6/recov_neon.c rename to lib/raid/raid6/arm/recov_neon.c index 9d99aeabd31a..1524050d09b7 100644 --- a/lib/raid6/recov_neon.c +++ b/lib/raid/raid6/arm/recov_neon.c @@ -4,20 +4,11 @@ * Copyright (C) 2017 Linaro Ltd. */ +#include #include - -#ifdef __KERNEL__ #include -#include "neon.h" -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif - -static int raid6_has_neon(void) -{ - return cpu_has_neon(); -} +#include "algos.h" +#include "arm/neon.h" static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -35,13 +26,13 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +63,10 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -91,7 +82,5 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_neon = { .data2 = raid6_2data_recov_neon, .datap = raid6_datap_recov_neon, - .valid = raid6_has_neon, .name = "neon", - .priority = 10, }; diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c similarity index 99% rename from lib/raid6/recov_neon_inner.c rename to lib/raid/raid6/arm/recov_neon_inner.c index f9e7e8f5a151..53c355efa7ff 100644 --- a/lib/raid6/recov_neon_inner.c +++ b/lib/raid/raid6/arm/recov_neon_inner.c @@ -5,7 +5,7 @@ */ #include -#include "neon.h" +#include "arm/neon.h" #ifdef CONFIG_ARM /* diff --git a/lib/raid/raid6/arm64/pq_arch.h b/lib/raid/raid6/arm64/pq_arch.h new file mode 100644 index 000000000000..27ff564d7594 --- /dev/null +++ b/lib/raid/raid6/arm64/pq_arch.h @@ -0,0 +1 @@ +#include "arm/pq_arch.h" diff --git a/lib/raid6/int.uc b/lib/raid/raid6/int.uc similarity index 96% rename from lib/raid6/int.uc rename to lib/raid/raid6/int.uc index 1ba56c3fa482..e63bd5a9c2ed 100644 --- a/lib/raid6/int.uc +++ b/lib/raid/raid6/int.uc @@ -18,7 +18,7 @@ * This file is postprocessed using unroll.awk */ -#include +#include "algos.h" /* * This is the C data type to use @@ -139,9 +139,7 @@ static void raid6_int$#_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_intx$# = { - raid6_int$#_gen_syndrome, - raid6_int$#_xor_syndrome, - NULL, /* always valid */ - "int" NSTRING "x$#", - 0 + .gen_syndrome = raid6_int$#_gen_syndrome, + .xor_syndrome = raid6_int$#_xor_syndrome, + .name = "int" NSTRING "x$#", }; diff --git a/lib/raid6/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c similarity index 96% rename from lib/raid6/loongarch_simd.c rename to lib/raid/raid6/loongarch/loongarch_simd.c index aa5d9f924ca3..c1eb53fafd27 100644 --- a/lib/raid6/loongarch_simd.c +++ b/lib/raid/raid6/loongarch/loongarch_simd.c @@ -9,8 +9,9 @@ * Copyright 2002-2004 H. Peter Anvin */ -#include -#include "loongarch.h" +#include +#include +#include "algos.h" /* * The vector algorithms are currently priority 0, which means the generic @@ -25,11 +26,6 @@ #ifdef CONFIG_CPU_HAS_LSX #define NSIZE 16 -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -243,11 +239,9 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lsx = { - raid6_lsx_gen_syndrome, - raid6_lsx_xor_syndrome, - raid6_has_lsx, - "lsx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lsx_gen_syndrome, + .xor_syndrome = raid6_lsx_xor_syndrome, + .name = "lsx", }; #undef NSIZE @@ -256,11 +250,6 @@ const struct raid6_calls raid6_lsx = { #ifdef CONFIG_CPU_HAS_LASX #define NSIZE 32 -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -412,11 +401,9 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lasx = { - raid6_lasx_gen_syndrome, - raid6_lasx_xor_syndrome, - raid6_has_lasx, - "lasx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lasx_gen_syndrome, + .xor_syndrome = raid6_lasx_xor_syndrome, + .name = "lasx", }; #undef NSIZE #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h new file mode 100644 index 000000000000..ae443a4d7b69 --- /dev/null +++ b/lib/raid/raid6/loongarch/pq_arch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_lsx; +extern const struct raid6_calls raid6_lasx; + +extern const struct raid6_recov_calls raid6_recov_lsx; +extern const struct raid6_recov_calls raid6_recov_lasx; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_algo_add(&raid6_lsx); + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_algo_add(&raid6_lasx); + + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_recov_algo_add(&raid6_recov_lasx); + else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_recov_algo_add(&raid6_recov_lsx); +} diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c similarity index 96% rename from lib/raid6/recov_loongarch_simd.c rename to lib/raid/raid6/loongarch/recov_loongarch_simd.c index 93dc515997a1..87a2313bbb4f 100644 --- a/lib/raid6/recov_loongarch_simd.c +++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c @@ -10,8 +10,11 @@ * Author: Jim Kukunas */ +#include #include -#include "loongarch.h" +#include +#include +#include "algos.h" /* * Unlike with the syndrome calculation algorithms, there's no boot-time @@ -21,11 +24,6 @@ */ #ifdef CONFIG_CPU_HAS_LSX -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -42,13 +40,13 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -197,10 +195,10 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -288,18 +286,11 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lsx = { .data2 = raid6_2data_recov_lsx, .datap = raid6_datap_recov_lsx, - .valid = raid6_has_lsx, .name = "lsx", - .priority = 1, }; #endif /* CONFIG_CPU_HAS_LSX */ #ifdef CONFIG_CPU_HAS_LASX -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -316,13 +307,13 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -436,10 +427,10 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -506,8 +497,6 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lasx = { .data2 = raid6_2data_recov_lasx, .datap = raid6_datap_recov_lasx, - .valid = raid6_has_lasx, .name = "lasx", - .priority = 2, }; #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/mktables.c b/lib/raid/raid6/mktables.c similarity index 83% rename from lib/raid6/mktables.c rename to lib/raid/raid6/mktables.c index 3be03793237c..b6327b562fdb 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid/raid6/mktables.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * mktables.c + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved * - * Make RAID-6 tables. This is a host user space program to be run at - * compile time. + * Make RAID-6 tables. This is a host user space program to be run at compile + * time. */ #include @@ -56,10 +50,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#ifdef __KERNEL__\n"); printf("#include \n"); - printf("#endif\n"); - printf("#include \n"); + printf("#include \"algos.h\"\n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -76,9 +68,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfmul);\n"); - printf("#endif\n"); /* Compute vector multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -101,9 +91,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_vgfmul);\n"); - printf("#endif\n"); /* Compute power-of-2 table (exponent) */ v = 1; @@ -120,9 +108,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexp);\n"); - printf("#endif\n"); /* Compute log-of-2 table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -140,9 +126,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gflog);\n"); - printf("#endif\n"); /* Compute inverse table x^-1 == x^254 */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -155,9 +139,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfinv);\n"); - printf("#endif\n"); /* Compute inv(2^x + 1) (exponent-xor-inverse) table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -169,9 +151,7 @@ int main(int argc, char *argv[]) (j == 7) ? '\n' : ' '); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexi);\n"); - printf("#endif\n"); return 0; } diff --git a/lib/raid6/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc similarity index 84% rename from lib/raid6/altivec.uc rename to lib/raid/raid6/powerpc/altivec.uc index d20ed0d11411..c5429fb71dd6 100644 --- a/lib/raid6/altivec.uc +++ b/lib/raid/raid6/powerpc/altivec.uc @@ -22,15 +22,11 @@ * bracked this with preempt_disable/enable or in a lock) */ -#include - -#ifdef CONFIG_ALTIVEC +#include "algos.h" #include -#ifdef __KERNEL__ -# include -# include -#endif /* __KERNEL__ */ +#include +#include /* * This is the C data type to use. We use a vector of @@ -108,25 +104,7 @@ static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec(void); -#if $# == 1 -int raid6_have_altivec(void) -{ - /* This assumes either all CPUs have Altivec or none does */ -# ifdef __KERNEL__ - return cpu_has_feature(CPU_FTR_ALTIVEC); -# else - return 1; -# endif -} -#endif - const struct raid6_calls raid6_altivec$# = { - raid6_altivec$#_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_altivec, - "altivecx$#", - 0 + .gen_syndrome = raid6_altivec$#_gen_syndrome, + .name = "altivecx$#", }; - -#endif /* CONFIG_ALTIVEC */ diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h new file mode 100644 index 000000000000..ea1878777ff2 --- /dev/null +++ b/lib/raid/raid6/powerpc/pq_arch.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_altivec1; +extern const struct raid6_calls raid6_altivec2; +extern const struct raid6_calls raid6_altivec4; +extern const struct raid6_calls raid6_altivec8; +extern const struct raid6_calls raid6_vpermxor1; +extern const struct raid6_calls raid6_vpermxor2; +extern const struct raid6_calls raid6_vpermxor4; +extern const struct raid6_calls raid6_vpermxor8; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + + /* This assumes either all CPUs have Altivec or none does */ + if (cpu_has_feature(CPU_FTR_ALTIVEC)) { + raid6_algo_add(&raid6_altivec1); + raid6_algo_add(&raid6_altivec2); + raid6_algo_add(&raid6_altivec4); + raid6_algo_add(&raid6_altivec8); + } + if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && + cpu_has_feature(CPU_FTR_ARCH_207S)) { + raid6_algo_add(&raid6_vpermxor1); + raid6_algo_add(&raid6_vpermxor2); + raid6_algo_add(&raid6_vpermxor4); + raid6_algo_add(&raid6_vpermxor8); + } +} diff --git a/lib/raid6/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc similarity index 82% rename from lib/raid6/vpermxor.uc rename to lib/raid/raid6/powerpc/vpermxor.uc index 1bfb127fbfe8..e8964361aaef 100644 --- a/lib/raid6/vpermxor.uc +++ b/lib/raid/raid6/powerpc/vpermxor.uc @@ -20,15 +20,11 @@ * This instruction was introduced in POWER8 - ISA v2.07. */ -#include -#ifdef CONFIG_ALTIVEC - #include #include -#ifdef __KERNEL__ #include #include -#endif +#include "algos.h" typedef vector unsigned char unative_t; #define NSIZE sizeof(unative_t) @@ -80,26 +76,7 @@ static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec_vpermxor(void); -#if $# == 1 -int raid6_have_altivec_vpermxor(void) -{ - /* Check if arch has both altivec and the vpermxor instructions */ -# ifdef __KERNEL__ - return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && - cpu_has_feature(CPU_FTR_ARCH_207S)); -# else - return 1; -#endif - -} -#endif - const struct raid6_calls raid6_vpermxor$# = { - raid6_vpermxor$#_gen_syndrome, - NULL, - raid6_have_altivec_vpermxor, - "vpermxor$#", - 0 + .gen_syndrome = raid6_vpermxor$#_gen_syndrome, + .name = "vpermxor$#", }; -#endif diff --git a/lib/raid6/recov.c b/lib/raid/raid6/recov.c similarity index 60% rename from lib/raid6/recov.c rename to lib/raid/raid6/recov.c index b5e47c008b41..3fa53bc3fde4 100644 --- a/lib/raid6/recov.c +++ b/lib/raid/raid6/recov.c @@ -1,19 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/recov.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * RAID-6 data recovery in dual failure mode. In single failure mode, - * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct - * the syndrome.) + * RAID-6 data recovery in dual failure mode. In single failure mode, use the + * RAID-5 algorithm (or, in the case of Q failure, just reconstruct the + * syndrome.) */ +#include #include +#include "algos.h" /* Recover two failed data blocks. */ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, @@ -31,13 +27,13 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +68,10 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -95,41 +91,5 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_intx1 = { .data2 = raid6_2data_recov_intx1, .datap = raid6_datap_recov_intx1, - .valid = NULL, .name = "intx1", - .priority = 0, }; - -#ifndef __KERNEL__ -/* Testing only */ - -/* Recover two failed blocks. */ -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs) -{ - if ( faila > failb ) { - int tmp = faila; - faila = failb; - failb = tmp; - } - - if ( failb == disks-1 ) { - if ( faila == disks-2 ) { - /* P+Q failure. Just rebuild the syndrome. */ - raid6_call.gen_syndrome(disks, bytes, ptrs); - } else { - /* data+Q failure. Reconstruct data from P, - then rebuild syndrome. */ - /* NOT IMPLEMENTED - equivalent to RAID-5 */ - } - } else { - if ( failb == disks-2 ) { - /* data+P failure. */ - raid6_datap_recov(disks, bytes, faila, ptrs); - } else { - /* data+data failure. */ - raid6_2data_recov(disks, bytes, faila, failb, ptrs); - } - } -} - -#endif diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h new file mode 100644 index 000000000000..82f1a188f8c4 --- /dev/null +++ b/lib/raid/raid6/riscv/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_rvvx1; +extern const struct raid6_calls raid6_rvvx2; +extern const struct raid6_calls raid6_rvvx4; +extern const struct raid6_calls raid6_rvvx8; +extern const struct raid6_recov_calls raid6_recov_rvv; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (has_vector()) { + raid6_algo_add(&raid6_rvvx1); + raid6_algo_add(&raid6_rvvx2); + raid6_algo_add(&raid6_rvvx4); + raid6_algo_add(&raid6_rvvx8); + raid6_recov_algo_add(&raid6_recov_rvv); + } +} diff --git a/lib/raid6/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c similarity index 94% rename from lib/raid6/recov_rvv.c rename to lib/raid/raid6/riscv/recov_rvv.c index 40c393206b6a..78e158a3e332 100644 --- a/lib/raid6/recov_rvv.c +++ b/lib/raid/raid6/riscv/recov_rvv.c @@ -4,8 +4,11 @@ * Author: Chunyan Zhang */ +#include #include +#include "algos.h" #include "rvv.h" +#include "pq_arch.h" static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, u8 *dq, const u8 *pbmul, @@ -158,13 +161,13 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +199,10 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -216,7 +219,5 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_rvv = { .data2 = raid6_2data_recov_rvv, .datap = raid6_datap_recov_rvv, - .valid = rvv_has_vector, .name = "rvv", - .priority = 1, }; diff --git a/lib/raid6/rvv.c b/lib/raid/raid6/riscv/rvv.c similarity index 99% rename from lib/raid6/rvv.c rename to lib/raid/raid6/riscv/rvv.c index 75c9dafedb28..4ac50606f3dc 100644 --- a/lib/raid6/rvv.c +++ b/lib/raid/raid6/riscv/rvv.c @@ -10,6 +10,7 @@ */ #include "rvv.h" +#include "pq_arch.h" #ifdef __riscv_vector #error "This code must be built without compiler support for vector" diff --git a/lib/raid6/rvv.h b/lib/raid/raid6/riscv/rvv.h similarity index 70% rename from lib/raid6/rvv.h rename to lib/raid/raid6/riscv/rvv.h index 6d0708a2c8a4..df0e3637cae8 100644 --- a/lib/raid6/rvv.h +++ b/lib/raid/raid6/riscv/rvv.h @@ -2,27 +2,11 @@ /* * Copyright 2024 Institute of Software, CAS. * - * raid6/rvv.h - * * Definitions for RISC-V RAID-6 code */ -#ifdef __KERNEL__ #include -#else -#define kernel_vector_begin() -#define kernel_vector_end() -#include -#include -#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V) -#endif - -#include - -static int rvv_has_vector(void) -{ - return has_vector(); -} +#include "algos.h" #define RAID6_RVV_WRAPPER(_n) \ static void raid6_rvv ## _n ## _gen_syndrome(int disks, \ @@ -48,9 +32,7 @@ static int rvv_has_vector(void) kernel_vector_end(); \ } \ struct raid6_calls const raid6_rvvx ## _n = { \ - raid6_rvv ## _n ## _gen_syndrome, \ - raid6_rvv ## _n ## _xor_syndrome, \ - rvv_has_vector, \ - "rvvx" #_n, \ - 0 \ + .gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \ + .name = "rvvx" #_n, \ } diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h new file mode 100644 index 000000000000..95d14c342306 --- /dev/null +++ b/lib/raid/raid6/s390/pq_arch.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_s390vx8; +extern const struct raid6_recov_calls raid6_recov_s390xc; + +static __always_inline void __init arch_raid6_init(void) +{ + if (cpu_has_vx()) + raid6_algo_add(&raid6_s390vx8); + else + raid6_algo_add_default(); + raid6_recov_algo_add(&raid6_recov_s390xc); +} diff --git a/lib/raid6/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c similarity index 90% rename from lib/raid6/recov_s390xc.c rename to lib/raid/raid6/s390/recov_s390xc.c index 487018f81192..08d56896e5ea 100644 --- a/lib/raid6/recov_s390xc.c +++ b/lib/raid/raid6/s390/recov_s390xc.c @@ -6,7 +6,9 @@ * Author(s): Martin Schwidefsky */ +#include #include +#include "algos.h" static inline void xor_block(u8 *p1, u8 *p2) { @@ -34,13 +36,13 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -81,10 +83,10 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -110,7 +112,5 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_s390xc = { .data2 = raid6_2data_recov_s390xc, .datap = raid6_datap_recov_s390xc, - .valid = NULL, .name = "s390xc", - .priority = 1, }; diff --git a/lib/raid6/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc similarity index 93% rename from lib/raid6/s390vx.uc rename to lib/raid/raid6/s390/s390vx.uc index 8aa53eb2f395..e5cf9054be2a 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid/raid6/s390/s390vx.uc @@ -12,8 +12,8 @@ */ #include -#include #include +#include "algos.h" #define NSIZE 16 @@ -121,15 +121,8 @@ static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(&vxstate, KERNEL_VXR); } -static int raid6_s390vx$#_valid(void) -{ - return cpu_has_vx(); -} - const struct raid6_calls raid6_s390vx$# = { - raid6_s390vx$#_gen_syndrome, - raid6_s390vx$#_xor_syndrome, - raid6_s390vx$#_valid, - "vx128x$#", - 1 + .gen_syndrome = raid6_s390vx$#_gen_syndrome, + .xor_syndrome = raid6_s390vx$#_xor_syndrome, + .name = "vx128x$#", }; diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile new file mode 100644 index 000000000000..87a001b22847 --- /dev/null +++ b/lib/raid/raid6/tests/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += raid6_kunit.o diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c new file mode 100644 index 000000000000..9f3e671a1224 --- /dev/null +++ b/lib/raid/raid6/tests/raid6_kunit.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved + * + * Test RAID-6 recovery algorithms. + */ + +#include +#include +#include +#include +#include "../algos.h" + +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + +#define RAID6_KUNIT_SEED 42 +#define RAID6_KUNIT_NUM_TEST_ITERS 10 +#define RAID6_KUNIT_MAX_BUFFERS 64 /* Including P and Q */ +#define RAID6_KUNIT_MAX_FAILURES 2 +#define RAID6_KUNIT_MAX_BYTES PAGE_SIZE + +static struct rnd_state rng; +static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *aligned_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *test_recov_buffers[RAID6_KUNIT_MAX_FAILURES]; +static size_t test_buflen; + +struct test_args { + unsigned int recov_idx; + const struct raid6_recov_calls *recov; + unsigned int gen_idx; + const struct raid6_calls *gen; +}; + +static struct test_args args; + +static u32 rand32(void) +{ + return prandom_u32_state(&rng); +} + +/* Generate a random length that is a multiple of 512. */ +static unsigned int random_length(unsigned int max_length) +{ + return round_up((rand32() % max_length) + 1, 512); +} + +static unsigned int random_nr_buffers(void) +{ + return (rand32() % (RAID6_KUNIT_MAX_BUFFERS - (RAID6_MIN_DISKS - 1))) + + RAID6_MIN_DISKS; +} + +/* Generate a random alignment that is a multiple of 64. */ +static unsigned int random_alignment(unsigned int max_alignment) +{ + if (max_alignment == 0) + return 0; + return (rand32() % (max_alignment + 1)) & ~63; +} + +static void makedata(int start, int stop) +{ + int i; + + for (i = start; i <= stop; i++) + prandom_bytes_state(&rng, test_buffers[i], test_buflen); +} + +static char member_type(unsigned int nr_buffers, int d) +{ + if (d == nr_buffers - 2) + return 'P'; + if (d == nr_buffers - 1) + return 'Q'; + return 'D'; +} + +static void test_recover_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int faila, int failb) +{ + const struct test_args *ta = test->param_value; + void *dataptrs[RAID6_KUNIT_MAX_BUFFERS]; + int i; + + if (faila > failb) + swap(faila, failb); + + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + memset(test_recov_buffers[i], 0xf0, test_buflen); + + memcpy(dataptrs, aligned_buffers, sizeof(dataptrs)); + dataptrs[faila] = test_recov_buffers[0]; + dataptrs[failb] = test_recov_buffers[1]; + + if (failb == nr_buffers - 1) { + /* + * We don't implement the data+Q failure scenario, since it + * is equivalent to a RAID-5 failure (XOR, then recompute Q). + */ + if (WARN_ON_ONCE(faila != nr_buffers - 2)) + return; + + /* P+Q failure. Just rebuild the syndrome. */ + ta->gen->gen_syndrome(nr_buffers, len, dataptrs); + } else if (failb == nr_buffers - 2) { + /* data+P failure. */ + ta->recov->datap(nr_buffers, len, faila, dataptrs); + } else { + /* data+data failure. */ + ta->recov->data2(nr_buffers, len, faila, failb, dataptrs); + } + + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[faila], dataptrs[faila], + len, + "faila miscompared: %3d[%c] buffers %u len %u (failb=%3d[%c])\n", + faila, member_type(nr_buffers, faila), + nr_buffers, len, + failb, member_type(nr_buffers, failb)); + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[failb], dataptrs[failb], + len, + "failb miscompared: %3d[%c] buffers %u len %u (faila=%3d[%c])\n", + failb, member_type(nr_buffers, failb), + nr_buffers, len, + faila, member_type(nr_buffers, faila)); +} + +static void test_recover(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + unsigned int nr_data = nr_buffers - 2; + int iterations, i; + + /* Test P+Q recovery */ + test_recover_one(test, nr_buffers, len, nr_data, nr_buffers - 1); + + /* Test data+P recovery */ + for (i = 0; i < nr_buffers - 2; i++) + test_recover_one(test, nr_buffers, len, i, nr_data); + + /* Double data failure is impossible with a single data disk */ + if (nr_data == 1) + return; + + /* Test data+data recovery using random sampling */ + iterations = nr_buffers * 2; /* should provide good enough coverage */ + for (i = 0; i < iterations; i++) { + int faila = rand32() % nr_data, failb; + + do { + failb = rand32() % nr_data; + } while (failb == faila); + + test_recover_one(test, nr_buffers, len, faila, failb); + } +} + +/* Simulate rmw run */ +static void test_rmw_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int p1, int p2) +{ + const struct test_args *ta = test->param_value; + + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + makedata(p1, p2); + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + test_recover(test, nr_buffers, len); +} + +static void test_rmw(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + int iterations = nr_buffers / 2, i; + + for (i = 0; i < iterations; i++) { + int p1 = rand32() % (nr_buffers - 2); + int p2 = rand32() % (nr_buffers - 2); + + if (p2 < p1) + swap(p1, p2); + test_rmw_one(test, nr_buffers, len, p1, p2); + } +} + +static void raid6_test_one(struct kunit *test) +{ + const struct test_args *ta = test->param_value; + unsigned int nr_buffers = random_nr_buffers(); + unsigned int len = random_length(RAID6_KUNIT_MAX_BYTES); + unsigned int max_alignment; + int i; + + /* Nuke syndromes */ + memset(test_buffers[nr_buffers - 2], 0xee, test_buflen); + memset(test_buffers[nr_buffers - 1], 0xee, test_buflen); + + /* + * If we're not using the entire buffer size, inject randomize alignment + * into the buffer. + */ + max_alignment = RAID6_KUNIT_MAX_BYTES - len; + if (rand32() % 2 == 0) { + /* Use random alignments mod 64 */ + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + + random_alignment(max_alignment); + } else { + /* Go up to the guard page, to catch buffer overreads */ + unsigned int align = test_buflen - len; + + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + align; + } + + /* Generate assumed good syndrome */ + ta->gen->gen_syndrome(nr_buffers, len, aligned_buffers); + + test_recover(test, nr_buffers, len); + + if (ta->gen->xor_syndrome) + test_rmw(test, nr_buffers, len); +} + +static void raid6_test(struct kunit *test) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_NUM_TEST_ITERS; i++) + raid6_test_one(test); +} + +static const void *raid6_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + if (!prev) { + memset(&args, 0, sizeof(args)); +next_algo: + args.recov_idx = 0; + args.gen = raid6_algo_find(args.gen_idx); + if (!args.gen) + return NULL; + } + + if (args.recov) + args.recov_idx++; + args.recov = raid6_recov_algo_find(args.recov_idx); + if (!args.recov) { + args.gen_idx++; + goto next_algo; + } + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "gen=%s recov=%s", + args.gen->name, args.recov->name); + return &args; +} + +static struct kunit_case raid6_test_cases[] = { + KUNIT_CASE_PARAM(raid6_test, raid6_gen_params), + {}, +}; + +static int raid6_suite_init(struct kunit_suite *suite) +{ + int i; + + prandom_seed_state(&rng, RAID6_KUNIT_SEED); + + /* + * Allocate the test buffer using vmalloc() with a page-aligned length + * so that it is immediately followed by a guard page. This allows + * buffer overreads to be detected, even in assembly code. + */ + test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) { + test_recov_buffers[i] = vmalloc(test_buflen); + if (!test_recov_buffers[i]) + goto out_free_recov_buffers; + } + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) { + test_buffers[i] = vmalloc(test_buflen); + if (!test_buffers[i]) + goto out_free_buffers; + } + + makedata(0, RAID6_KUNIT_MAX_BUFFERS - 1); + + return 0; + +out_free_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); +out_free_recov_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); + return -ENOMEM; +} + +static void raid6_suite_exit(struct kunit_suite *suite) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); +} + +static struct kunit_suite raid6_test_suite = { + .name = "raid6", + .test_cases = raid6_test_cases, + .suite_init = raid6_suite_init, + .suite_exit = raid6_suite_exit, +}; +kunit_test_suite(raid6_test_suite); + +MODULE_DESCRIPTION("Unit test for the RAID P/Q library functions"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid6/unroll.awk b/lib/raid/raid6/unroll.awk similarity index 100% rename from lib/raid6/unroll.awk rename to lib/raid/raid6/unroll.awk diff --git a/lib/raid6/avx2.c b/lib/raid/raid6/x86/avx2.c similarity index 94% rename from lib/raid6/avx2.c rename to lib/raid/raid6/x86/avx2.c index 059024234dce..7d829c669ea7 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid/raid6/x86/avx2.c @@ -1,20 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright (C) 2012 Intel Corporation - * Author: Yuanhan Liu +/* + * Copyright (C) 2012 Intel Corporation + * Author: Yuanhan Liu * - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- */ - -/* * AVX2 implementation of RAID-6 syndrome functions - * */ -#include -#include "x86.h" +#include +#include +#include "algos.h" static const struct raid6_avx2_constants { u64 x1d[4]; @@ -23,11 +19,6 @@ static const struct raid6_avx2_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); -} - /* * Plain AVX2 implementation */ @@ -128,11 +119,9 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x1 = { - raid6_avx21_gen_syndrome, - raid6_avx21_xor_syndrome, - raid6_have_avx2, - "avx2x1", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx21_gen_syndrome, + .xor_syndrome = raid6_avx21_xor_syndrome, + .name = "avx2x1", }; /* @@ -258,11 +247,9 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x2 = { - raid6_avx22_gen_syndrome, - raid6_avx22_xor_syndrome, - raid6_have_avx2, - "avx2x2", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx22_gen_syndrome, + .xor_syndrome = raid6_avx22_xor_syndrome, + .name = "avx2x2", }; #ifdef CONFIG_X86_64 @@ -461,10 +448,8 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x4 = { - raid6_avx24_gen_syndrome, - raid6_avx24_xor_syndrome, - raid6_have_avx2, - "avx2x4", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx24_gen_syndrome, + .xor_syndrome = raid6_avx24_xor_syndrome, + .name = "avx2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/avx512.c b/lib/raid/raid6/x86/avx512.c similarity index 93% rename from lib/raid6/avx512.c rename to lib/raid/raid6/x86/avx512.c index 009bd0adeebf..e671eb5bde63 100644 --- a/lib/raid6/avx512.c +++ b/lib/raid/raid6/x86/avx512.c @@ -1,24 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- -------------------------------------------------------- - * - * Copyright (C) 2016 Intel Corporation +/* + * Copyright (C) 2016 Intel Corporation * - * Author: Gayatri Kammela - * Author: Megha Dey + * Author: Gayatri Kammela + * Author: Megha Dey * - * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- - */ - -/* * AVX512 implementation of RAID-6 syndrome functions - * */ -#include -#include "x86.h" +#include +#include +#include "algos.h" static const struct raid6_avx512_constants { u64 x1d[8]; @@ -29,16 +24,6 @@ static const struct raid6_avx512_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} - static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -156,11 +141,9 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x1 = { - raid6_avx5121_gen_syndrome, - raid6_avx5121_xor_syndrome, - raid6_have_avx512, - "avx512x1", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5121_gen_syndrome, + .xor_syndrome = raid6_avx5121_xor_syndrome, + .name = "avx512x1", }; /* @@ -313,11 +296,9 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x2 = { - raid6_avx5122_gen_syndrome, - raid6_avx5122_xor_syndrome, - raid6_have_avx512, - "avx512x2", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5122_gen_syndrome, + .xor_syndrome = raid6_avx5122_xor_syndrome, + .name = "avx512x2", }; #ifdef CONFIG_X86_64 @@ -551,10 +532,8 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(); } const struct raid6_calls raid6_avx512x4 = { - raid6_avx5124_gen_syndrome, - raid6_avx5124_xor_syndrome, - raid6_have_avx512, - "avx512x4", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5124_gen_syndrome, + .xor_syndrome = raid6_avx5124_xor_syndrome, + .name = "avx512x4", }; #endif diff --git a/lib/raid6/mmx.c b/lib/raid/raid6/x86/mmx.c similarity index 81% rename from lib/raid6/mmx.c rename to lib/raid/raid6/x86/mmx.c index 3a5bf53a297b..afa82536142d 100644 --- a/lib/raid6/mmx.c +++ b/lib/raid/raid6/x86/mmx.c @@ -1,20 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/mmx.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * MMX implementation of RAID-6 syndrome functions + * MMX implementation of RAID-6 syndrome functions. */ -#ifdef CONFIG_X86_32 - -#include -#include "x86.h" +#include +#include +#include "algos.h" /* Shared with raid6/sse1.c */ const struct raid6_mmx_constants { @@ -23,12 +16,6 @@ const struct raid6_mmx_constants { 0x1d1d1d1d1d1d1d1dULL, }; -static int raid6_have_mmx(void) -{ - /* Not really "boot_cpu" but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX); -} - /* * Plain MMX implementation */ @@ -70,11 +57,8 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx1 = { - raid6_mmx1_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx1", - 0 + .gen_syndrome = raid6_mmx1_gen_syndrome, + .name = "mmxx1", }; /* @@ -129,11 +113,6 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx2 = { - raid6_mmx2_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx2", - 0 + .gen_syndrome = raid6_mmx2_gen_syndrome, + .name = "mmxx2", }; - -#endif diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h new file mode 100644 index 000000000000..02f8843b0537 --- /dev/null +++ b/lib/raid/raid6/x86/pq_arch.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include + +extern const struct raid6_calls raid6_mmxx1; +extern const struct raid6_calls raid6_mmxx2; +extern const struct raid6_calls raid6_sse1x1; +extern const struct raid6_calls raid6_sse1x2; +extern const struct raid6_calls raid6_sse2x1; +extern const struct raid6_calls raid6_sse2x2; +extern const struct raid6_calls raid6_sse2x4; +extern const struct raid6_calls raid6_avx2x1; +extern const struct raid6_calls raid6_avx2x2; +extern const struct raid6_calls raid6_avx2x4; +extern const struct raid6_calls raid6_avx512x1; +extern const struct raid6_calls raid6_avx512x2; +extern const struct raid6_calls raid6_avx512x4; + +extern const struct raid6_recov_calls raid6_recov_ssse3; +extern const struct raid6_recov_calls raid6_recov_avx2; +extern const struct raid6_recov_calls raid6_recov_avx512; + +static inline int raid6_has_avx512(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512BW) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + boot_cpu_has(X86_FEATURE_AVX512DQ); +} + +static inline bool raid6_has_avx2(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); +} + +static inline bool raid6_has_ssse3(void) +{ + return boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2) && + boot_cpu_has(X86_FEATURE_SSSE3); +} + +static inline bool raid6_has_sse2(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + boot_cpu_has(X86_FEATURE_FXSR) && + boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2); +} + +static inline bool raid6_has_sse1_or_mmxext(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + (boot_cpu_has(X86_FEATURE_XMM) || + boot_cpu_has(X86_FEATURE_MMXEXT)); +} + +static __always_inline void __init arch_raid6_init(void) +{ + if (raid6_has_avx2()) { + raid6_algo_add(&raid6_avx2x1); + raid6_algo_add(&raid6_avx2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx2x4); + if (raid6_has_avx512()) { + raid6_algo_add(&raid6_avx512x1); + raid6_algo_add(&raid6_avx512x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx512x4); + } + } else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) { + /* x86_64 can assume SSE2 as baseline */ + raid6_algo_add(&raid6_sse2x1); + raid6_algo_add(&raid6_sse2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_sse2x4); + } else { + raid6_algo_add_default(); + if (raid6_has_sse1_or_mmxext()) { + raid6_algo_add(&raid6_sse1x1); + raid6_algo_add(&raid6_sse1x2); + } else if (boot_cpu_has(X86_FEATURE_MMX)) { + raid6_algo_add(&raid6_mmxx1); + raid6_algo_add(&raid6_mmxx2); + } + } + + if (raid6_has_avx512()) + raid6_recov_algo_add(&raid6_recov_avx512); + else if (raid6_has_avx2()) + raid6_recov_algo_add(&raid6_recov_avx2); + else if (raid6_has_ssse3()) + raid6_recov_algo_add(&raid6_recov_ssse3); +} diff --git a/lib/raid6/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c similarity index 95% rename from lib/raid6/recov_avx2.c rename to lib/raid/raid6/x86/recov_avx2.c index 97d598d2535c..a714a780a2d8 100644 --- a/lib/raid6/recov_avx2.c +++ b/lib/raid/raid6/x86/recov_avx2.c @@ -4,14 +4,10 @@ * Author: Jim Kukunas */ +#include #include -#include "x86.h" - -static int raid6_has_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX); -} +#include +#include "algos.h" static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -28,13 +24,13 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +192,10 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -303,11 +299,9 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx2 = { .data2 = raid6_2data_recov_avx2, .datap = raid6_datap_recov_avx2, - .valid = raid6_has_avx2, #ifdef CONFIG_X86_64 .name = "avx2x2", #else .name = "avx2x1", #endif - .priority = 2, }; diff --git a/lib/raid6/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c similarity index 94% rename from lib/raid6/recov_avx512.c rename to lib/raid/raid6/x86/recov_avx512.c index 7986120ca444..ec72d5a30c01 100644 --- a/lib/raid6/recov_avx512.c +++ b/lib/raid/raid6/x86/recov_avx512.c @@ -6,18 +6,10 @@ * Author: Megha Dey */ +#include #include -#include "x86.h" - -static int raid6_has_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} +#include +#include "algos.h" static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -37,13 +29,13 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -238,10 +230,10 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -367,11 +359,9 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx512 = { .data2 = raid6_2data_recov_avx512, .datap = raid6_datap_recov_avx512, - .valid = raid6_has_avx512, #ifdef CONFIG_X86_64 .name = "avx512x2", #else .name = "avx512x1", #endif - .priority = 3, }; diff --git a/lib/raid6/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c similarity index 95% rename from lib/raid6/recov_ssse3.c rename to lib/raid/raid6/x86/recov_ssse3.c index 2e849185c32b..700bd2c865ec 100644 --- a/lib/raid6/recov_ssse3.c +++ b/lib/raid/raid6/x86/recov_ssse3.c @@ -3,15 +3,10 @@ * Copyright (C) 2012 Intel Corporation */ +#include #include -#include "x86.h" - -static int raid6_has_ssse3(void) -{ - return boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2) && - boot_cpu_has(X86_FEATURE_SSSE3); -} +#include +#include "algos.h" static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -30,13 +25,13 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -203,10 +198,10 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -318,11 +313,9 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_ssse3 = { .data2 = raid6_2data_recov_ssse3, .datap = raid6_datap_recov_ssse3, - .valid = raid6_has_ssse3, #ifdef CONFIG_X86_64 .name = "ssse3x2", #else .name = "ssse3x1", #endif - .priority = 1, }; diff --git a/lib/raid6/sse1.c b/lib/raid/raid6/x86/sse1.c similarity index 78% rename from lib/raid6/sse1.c rename to lib/raid/raid6/x86/sse1.c index 692fa3a93bf0..f4b260df522a 100644 --- a/lib/raid6/sse1.c +++ b/lib/raid/raid6/x86/sse1.c @@ -1,39 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse1.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * SSE-1/MMXEXT implementation of RAID-6 syndrome functions + * SSE-1/MMXEXT implementation of RAID-6 syndrome functions. * - * This is really an MMX implementation, but it requires SSE-1 or - * AMD MMXEXT for prefetch support and a few other features. The - * support for nontemporal memory accesses is enough to make this - * worthwhile as a separate implementation. + * This is really an MMX implementation, but it requires SSE-1 or AMD MMXEXT for + * prefetch support and a few other features. The support for nontemporal + * memory accesses is enough to make this worthwhile as a separate + * implementation. */ -#ifdef CONFIG_X86_32 - -#include -#include "x86.h" +#include +#include +#include "algos.h" /* Defined in raid6/mmx.c */ extern const struct raid6_mmx_constants { u64 x1d; } raid6_mmx_constants; -static int raid6_have_sse1_or_mmxext(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - (boot_cpu_has(X86_FEATURE_XMM) || - boot_cpu_has(X86_FEATURE_MMXEXT)); -} - /* * Plain SSE1 implementation */ @@ -86,11 +71,8 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x1 = { - raid6_sse11_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse11_gen_syndrome, + .name = "sse1x1", }; /* @@ -149,11 +131,6 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x2 = { - raid6_sse12_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse12_gen_syndrome, + .name = "sse1x2", }; - -#endif diff --git a/lib/raid6/sse2.c b/lib/raid/raid6/x86/sse2.c similarity index 94% rename from lib/raid6/sse2.c rename to lib/raid/raid6/x86/sse2.c index 2930220249c9..43b09ce58270 100644 --- a/lib/raid6/sse2.c +++ b/lib/raid/raid6/x86/sse2.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse2.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * * SSE-2 implementation of RAID-6 syndrome functions - * */ -#include -#include "x86.h" +#include +#include +#include "algos.h" static const struct raid6_sse_constants { u64 x1d[2]; @@ -21,15 +15,6 @@ static const struct raid6_sse_constants { { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, }; -static int raid6_have_sse2(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - boot_cpu_has(X86_FEATURE_FXSR) && - boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2); -} - /* * Plain SSE2 implementation */ @@ -133,11 +118,9 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x1 = { - raid6_sse21_gen_syndrome, - raid6_sse21_xor_syndrome, - raid6_have_sse2, - "sse2x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse21_gen_syndrome, + .xor_syndrome = raid6_sse21_xor_syndrome, + .name = "sse2x1", }; /* @@ -263,11 +246,9 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x2 = { - raid6_sse22_gen_syndrome, - raid6_sse22_xor_syndrome, - raid6_have_sse2, - "sse2x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse22_gen_syndrome, + .xor_syndrome = raid6_sse22_xor_syndrome, + .name = "sse2x2", }; #ifdef CONFIG_X86_64 @@ -470,11 +451,9 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop, const struct raid6_calls raid6_sse2x4 = { - raid6_sse24_gen_syndrome, - raid6_sse24_xor_syndrome, - raid6_have_sse2, - "sse2x4", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse24_gen_syndrome, + .xor_syndrome = raid6_sse24_xor_syndrome, + .name = "sse2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c index bd4e6e434418..50931fbf0324 100644 --- a/lib/raid/xor/xor-core.c +++ b/lib/raid/xor/xor-core.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -114,7 +115,7 @@ static int __init calibrate_xor_blocks(void) if (forced_template) return 0; - b1 = (void *) __get_free_pages(GFP_KERNEL, 2); + b1 = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); if (!b1) { pr_warn("xor: Yikes! No memory available.\n"); return -ENOMEM; @@ -132,7 +133,7 @@ static int __init calibrate_xor_blocks(void) pr_info("xor: using function: %s (%d MB/sec)\n", fastest->name, fastest->speed); - free_pages((unsigned long)b1, 2); + kfree(b1); return 0; } diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile deleted file mode 100644 index 5be0a4e60ab1..000000000000 --- a/lib/raid6/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_RAID6_PQ) += raid6_pq.o - -raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ - int8.o - -raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o -raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o -raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o -raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o - -hostprogs += mktables - -ifeq ($(CONFIG_ALTIVEC),y) -altivec_flags := -maltivec $(call cc-option,-mabi=altivec) -# Enable -altivec_flags += -isystem $(shell $(CC) -print-file-name=include) - -ifdef CONFIG_CC_IS_CLANG -# clang ppc port does not yet support -maltivec when -msoft-float is -# enabled. A future release of clang will resolve this -# https://llvm.org/pr31177 -CFLAGS_REMOVE_altivec1.o += -msoft-float -CFLAGS_REMOVE_altivec2.o += -msoft-float -CFLAGS_REMOVE_altivec4.o += -msoft-float -CFLAGS_REMOVE_altivec8.o += -msoft-float -CFLAGS_REMOVE_vpermxor1.o += -msoft-float -CFLAGS_REMOVE_vpermxor2.o += -msoft-float -CFLAGS_REMOVE_vpermxor4.o += -msoft-float -CFLAGS_REMOVE_vpermxor8.o += -msoft-float -endif -endif - -quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ - -targets += int1.c int2.c int4.c int8.c -$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_altivec1.o += $(altivec_flags) -CFLAGS_altivec2.o += $(altivec_flags) -CFLAGS_altivec4.o += $(altivec_flags) -CFLAGS_altivec8.o += $(altivec_flags) -targets += altivec1.c altivec2.c altivec4.c altivec8.c -$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_vpermxor1.o += $(altivec_flags) -CFLAGS_vpermxor2.o += $(altivec_flags) -CFLAGS_vpermxor4.o += $(altivec_flags) -CFLAGS_vpermxor8.o += $(altivec_flags) -targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c -$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_neon1.o += $(CC_FLAGS_FPU) -CFLAGS_neon2.o += $(CC_FLAGS_FPU) -CFLAGS_neon4.o += $(CC_FLAGS_FPU) -CFLAGS_neon8.o += $(CC_FLAGS_FPU) -CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU) -targets += neon1.c neon2.c neon4.c neon8.c -$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += s390vx8.c -$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -quiet_cmd_mktable = TABLE $@ - cmd_mktable = $(obj)/mktables > $@ - -targets += tables.c -$(obj)/tables.c: $(obj)/mktables FORCE - $(call if_changed,mktable) diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c deleted file mode 100644 index 799e0e5eac26..000000000000 --- a/lib/raid6/algos.c +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/algos.c - * - * Algorithm list and algorithm selection for RAID-6 - */ - -#include -#ifndef __KERNEL__ -#include -#include -#else -#include -#include -#endif - -struct raid6_calls raid6_call; -EXPORT_SYMBOL_GPL(raid6_call); - -const struct raid6_calls * const raid6_algos[] = { -#if defined(__i386__) && !defined(__arch_um__) - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x2, - &raid6_sse2x1, - &raid6_sse1x2, - &raid6_sse1x1, - &raid6_mmxx2, - &raid6_mmxx1, -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_avx512x4, - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x4, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x4, - &raid6_sse2x2, - &raid6_sse2x1, -#endif -#ifdef CONFIG_ALTIVEC - &raid6_vpermxor8, - &raid6_vpermxor4, - &raid6_vpermxor2, - &raid6_vpermxor1, - &raid6_altivec8, - &raid6_altivec4, - &raid6_altivec2, - &raid6_altivec1, -#endif -#if defined(CONFIG_S390) - &raid6_s390vx8, -#endif -#ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx8, - &raid6_neonx4, - &raid6_neonx2, - &raid6_neonx1, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_rvvx1, - &raid6_rvvx2, - &raid6_rvvx4, - &raid6_rvvx8, -#endif - &raid6_intx8, - &raid6_intx4, - &raid6_intx2, - &raid6_intx1, - NULL -}; - -void (*raid6_2data_recov)(int, size_t, int, int, void **); -EXPORT_SYMBOL_GPL(raid6_2data_recov); - -void (*raid6_datap_recov)(int, size_t, int, void **); -EXPORT_SYMBOL_GPL(raid6_datap_recov); - -const struct raid6_recov_calls *const raid6_recov_algos[] = { -#ifdef CONFIG_X86 - &raid6_recov_avx512, - &raid6_recov_avx2, - &raid6_recov_ssse3, -#endif -#ifdef CONFIG_S390 - &raid6_recov_s390xc, -#endif -#if defined(CONFIG_KERNEL_MODE_NEON) - &raid6_recov_neon, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_recov_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_recov_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_recov_rvv, -#endif - &raid6_recov_intx1, - NULL -}; - -#ifdef __KERNEL__ -#define RAID6_TIME_JIFFIES_LG2 4 -#else -/* Need more time to be stable in userspace */ -#define RAID6_TIME_JIFFIES_LG2 9 -#define time_before(x, y) ((x) < (y)) -#endif - -#define RAID6_TEST_DISKS 8 -#define RAID6_TEST_DISKS_ORDER 3 - -static inline const struct raid6_recov_calls *raid6_choose_recov(void) -{ - const struct raid6_recov_calls *const *algo; - const struct raid6_recov_calls *best; - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) - if (!(*algo)->valid || (*algo)->valid()) - best = *algo; - - if (best) { - raid6_2data_recov = best->data2; - raid6_datap_recov = best->datap; - - pr_info("raid6: using %s recovery algorithm\n", best->name); - } else - pr_err("raid6: Yikes! No recovery algorithm found!\n"); - - return best; -} - -static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) -{ - unsigned long perf, bestgenperf, j0, j1; - int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ - const struct raid6_calls *const *algo; - const struct raid6_calls *best; - - for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { - if (!best || (*algo)->priority >= best->priority) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - best = *algo; - break; - } - - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1<gen_syndrome(disks, PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - if (perf > bestgenperf) { - bestgenperf = perf; - best = *algo; - } - pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf * HZ * (disks-2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - } - } - - if (!best) { - pr_err("raid6: Yikes! No algorithm found!\n"); - goto out; - } - - raid6_call = *best; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - pr_info("raid6: skipped pq benchmark and selected %s\n", - best->name); - goto out; - } - - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - - if (best->xor_syndrome) { - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { - best->xor_syndrome(disks, start, stop, - PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (perf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); - } - -out: - return best; -} - - -/* Try to pick the best algorithm */ -/* This code uses the gfmul table as convenient data set to abuse */ - -int __init raid6_select_algo(void) -{ - const int disks = RAID6_TEST_DISKS; - - const struct raid6_calls *gen_best; - const struct raid6_recov_calls *rec_best; - char *disk_ptr, *p; - void *dptrs[RAID6_TEST_DISKS]; - int i, cycle; - - /* prepare the buffer and fill it circularly with gfmul table */ - disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); - if (!disk_ptr) { - pr_err("raid6: Yikes! No memory available.\n"); - return -ENOMEM; - } - - p = disk_ptr; - for (i = 0; i < disks; i++) - dptrs[i] = p + PAGE_SIZE * i; - - cycle = ((disks - 2) * PAGE_SIZE) / 65536; - for (i = 0; i < cycle; i++) { - memcpy(p, raid6_gfmul, 65536); - p += 65536; - } - - if ((disks - 2) * PAGE_SIZE % 65536) - memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); - - /* select raid gen_syndrome function */ - gen_best = raid6_choose_gen(&dptrs, disks); - - /* select raid recover functions */ - rec_best = raid6_choose_recov(); - - free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); - - return gen_best && rec_best ? 0 : -EINVAL; -} - -static void raid6_exit(void) -{ - do { } while (0); -} - -subsys_initcall(raid6_select_algo); -module_exit(raid6_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h deleted file mode 100644 index acfc33ce7056..000000000000 --- a/lib/raid6/loongarch.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2023 WANG Xuerui - * - * raid6/loongarch.h - * - * Definitions common to LoongArch RAID-6 code only - */ - -#ifndef _LIB_RAID6_LOONGARCH_H -#define _LIB_RAID6_LOONGARCH_H - -#ifdef __KERNEL__ - -#include -#include - -#else /* for user-space testing */ - -#include - -/* have to supply these defines for glibc 2.37- and musl */ -#ifndef HWCAP_LOONGARCH_LSX -#define HWCAP_LOONGARCH_LSX (1 << 4) -#endif -#ifndef HWCAP_LOONGARCH_LASX -#define HWCAP_LOONGARCH_LASX (1 << 5) -#endif - -#define kernel_fpu_begin() -#define kernel_fpu_end() - -#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) -#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) - -#endif /* __KERNEL__ */ - -#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore deleted file mode 100644 index 1b68a77f348f..000000000000 --- a/lib/raid6/test/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/int.uc -/neon.uc -/raid6test diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile deleted file mode 100644 index 09bbe2b14cce..000000000000 --- a/lib/raid6/test/Makefile +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# This is a simple Makefile to test some of the RAID-6 code -# from userspace. -# - -pound := \# - -# Adjust as desired -CC = gcc -OPTFLAGS = -O2 -CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) -LD = ld -AWK = awk -f -AR = ar -RANLIB = ranlib -OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o - -ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/) -ifeq ($(ARCH),i386) - CFLAGS += -DCONFIG_X86_32 - IS_X86 = yes -endif -ifeq ($(ARCH),x86_64) - CFLAGS += -DCONFIG_X86_64 - IS_X86 = yes -endif - -ifeq ($(ARCH),arm) - CFLAGS += -I../../../arch/arm/include -mfpu=neon - HAS_NEON = yes -endif -ifeq ($(ARCH),aarch64) - CFLAGS += -I../../../arch/arm64/include - HAS_NEON = yes -endif - -ifeq ($(findstring riscv,$(ARCH)),riscv) - CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1 - HAS_RVV = yes -endif - -ifeq ($(findstring ppc,$(ARCH)),ppc) - CFLAGS += -I../../../arch/powerpc/include - HAS_ALTIVEC := $(shell printf '$(pound)include \nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) -endif - -ifeq ($(ARCH),loongarch64) - CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 - CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) - CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) -endif - -ifeq ($(IS_X86),yes) - OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o - CFLAGS += -DCONFIG_X86 -else ifeq ($(HAS_NEON),yes) - OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o - CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -else ifeq ($(ARCH),loongarch64) - OBJS += loongarch_simd.o recov_loongarch_simd.o -else ifeq ($(HAS_RVV),yes) - OBJS += rvv.o recov_rvv.o - CFLAGS += -DCONFIG_RISCV_ISA_V=1 -endif - -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< - -%.c: ../%.c - cp -f $< $@ - -%.uc: ../%.uc - cp -f $< $@ - -all: raid6.a raid6test - -raid6.a: $(OBJS) - rm -f $@ - $(AR) cq $@ $^ - $(RANLIB) $@ - -raid6test: test.c raid6.a - $(CC) $(CFLAGS) -o raid6test $^ - -neon1.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < neon.uc > $@ - -neon2.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < neon.uc > $@ - -neon4.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < neon.uc > $@ - -neon8.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < neon.uc > $@ - -altivec1.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@ - -altivec2.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < altivec.uc > $@ - -altivec4.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < altivec.uc > $@ - -altivec8.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < altivec.uc > $@ - -vpermxor1.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@ - -vpermxor2.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@ - -vpermxor4.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@ - -vpermxor8.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@ - -int1.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < int.uc > $@ - -int2.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < int.uc > $@ - -int4.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < int.uc > $@ - -int8.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < int.uc > $@ - -int16.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=16 < int.uc > $@ - -int32.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=32 < int.uc > $@ - -tables.c: mktables - ./mktables > tables.c - -clean: - rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test - -spotless: clean - rm -f *~ diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c deleted file mode 100644 index 841a55242aba..000000000000 --- a/lib/raid6/test/test.c +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6test.c - * - * Test RAID-6 recovery with various algorithms - */ - -#include -#include -#include -#include - -#define NDISKS 16 /* Including P and Q */ - -const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -char *dataptrs[NDISKS]; -char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -static void makedata(int start, int stop) -{ - int i, j; - - for (i = start; i <= stop; i++) { - for (j = 0; j < PAGE_SIZE; j++) - data[i][j] = rand(); - - dataptrs[i] = data[i]; - } -} - -static char disk_type(int d) -{ - switch (d) { - case NDISKS-2: - return 'P'; - case NDISKS-1: - return 'Q'; - default: - return 'D'; - } -} - -static int test_disks(int i, int j) -{ - int erra, errb; - - memset(recovi, 0xf0, PAGE_SIZE); - memset(recovj, 0xba, PAGE_SIZE); - - dataptrs[i] = recovi; - dataptrs[j] = recovj; - - raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs); - - erra = memcmp(data[i], recovi, PAGE_SIZE); - errb = memcmp(data[j], recovj, PAGE_SIZE); - - if (i < NDISKS-2 && j == NDISKS-1) { - /* We don't implement the DQ failure scenario, since it's - equivalent to a RAID-5 failure (XOR, then recompute Q) */ - erra = errb = 0; - } else { - printf("algo=%-8s faila=%3d(%c) failb=%3d(%c) %s\n", - raid6_call.name, - i, disk_type(i), - j, disk_type(j), - (!erra && !errb) ? "OK" : - !erra ? "ERRB" : - !errb ? "ERRA" : "ERRAB"); - } - - dataptrs[i] = data[i]; - dataptrs[j] = data[j]; - - return erra || errb; -} - -int main(int argc, char *argv[]) -{ - const struct raid6_calls *const *algo; - const struct raid6_recov_calls *const *ra; - int i, j, p1, p2; - int err = 0; - - makedata(0, NDISKS-1); - - for (ra = raid6_recov_algos; *ra; ra++) { - if ((*ra)->valid && !(*ra)->valid()) - continue; - - raid6_2data_recov = (*ra)->data2; - raid6_datap_recov = (*ra)->datap; - - printf("using recovery %s\n", (*ra)->name); - - for (algo = raid6_algos; *algo; algo++) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - raid6_call = **algo; - - /* Nuke syndromes */ - memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE); - - /* Generate assumed good syndrome */ - raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - - if (!raid6_call.xor_syndrome) - continue; - - for (p1 = 0; p1 < NDISKS-2; p1++) - for (p2 = p1; p2 < NDISKS-2; p2++) { - - /* Simulate rmw run */ - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - makedata(p1, p2); - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - } - - } - printf("\n"); - } - - printf("\n"); - /* Pick the best algorithm test */ - raid6_select_algo(); - - if (err) - printf("\n*** ERRORS FOUND ***\n"); - - return err; -} diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h deleted file mode 100644 index 9a6ff37115e7..000000000000 --- a/lib/raid6/x86.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* ----------------------------------------------------------------------- * - * - * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/x86.h - * - * Definitions common to x86 and x86-64 RAID-6 code only - */ - -#ifndef LINUX_RAID_RAID6X86_H -#define LINUX_RAID_RAID6X86_H - -#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__) - -#ifdef __KERNEL__ /* Real code */ - -#include - -#else /* Dummy code for user space testing */ - -static inline void kernel_fpu_begin(void) -{ -} - -static inline void kernel_fpu_end(void) -{ -} - -#define __aligned(x) __attribute__((aligned(x))) - -#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ -#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions - * (fast save and restore) */ -#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ -#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ -#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ -#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ -#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ -#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ -#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ -#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular) - * Instructions - */ -#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular) - * Instructions - */ -#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length) - * Extensions - */ -#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ - -/* Should work well enough on modern CPUs for testing */ -static inline int boot_cpu_has(int flag) -{ - u32 eax, ebx, ecx, edx; - - eax = (flag & 0x100) ? 7 : - (flag & 0x20) ? 0x80000001 : 1; - ecx = 0; - - asm volatile("cpuid" - : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx)); - - return ((flag & 0x100 ? ebx : - (flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1; -} - -#endif /* ndef __KERNEL__ */ - -#endif -#endif diff --git a/lib/random32.c b/lib/random32.c index 24e7acd9343f..dad90219c351 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -41,6 +41,7 @@ #include #include #include +#include /** * prandom_u32_state - seeded pseudo-random number generator. @@ -92,7 +93,14 @@ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes) } EXPORT_SYMBOL(prandom_bytes_state); -static void prandom_warmup(struct rnd_state *state) +/* + * Only declared here so that it has a prototype when made + * non-static for KUnit testing (avoids -Wmissing-prototypes). + */ +#if IS_ENABLED(CONFIG_KUNIT) +void prandom_warmup(struct rnd_state *state); +#endif +VISIBLE_IF_KUNIT void prandom_warmup(struct rnd_state *state) { /* Calling RNG ten times to satisfy recurrence condition */ prandom_u32_state(state); @@ -106,6 +114,7 @@ static void prandom_warmup(struct rnd_state *state) prandom_u32_state(state); prandom_u32_state(state); } +EXPORT_SYMBOL_IF_KUNIT(prandom_warmup); void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) { @@ -125,176 +134,3 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) } } EXPORT_SYMBOL(prandom_seed_full_state); - -#ifdef CONFIG_RANDOM32_SELFTEST -static struct prandom_test1 { - u32 seed; - u32 result; -} test1[] = { - { 1U, 3484351685U }, - { 2U, 2623130059U }, - { 3U, 3125133893U }, - { 4U, 984847254U }, -}; - -static struct prandom_test2 { - u32 seed; - u32 iteration; - u32 result; -} test2[] = { - /* Test cases against taus113 from GSL library. */ - { 931557656U, 959U, 2975593782U }, - { 1339693295U, 876U, 3887776532U }, - { 1545556285U, 961U, 1615538833U }, - { 601730776U, 723U, 1776162651U }, - { 1027516047U, 687U, 511983079U }, - { 416526298U, 700U, 916156552U }, - { 1395522032U, 652U, 2222063676U }, - { 366221443U, 617U, 2992857763U }, - { 1539836965U, 714U, 3783265725U }, - { 556206671U, 994U, 799626459U }, - { 684907218U, 799U, 367789491U }, - { 2121230701U, 931U, 2115467001U }, - { 1668516451U, 644U, 3620590685U }, - { 768046066U, 883U, 2034077390U }, - { 1989159136U, 833U, 1195767305U }, - { 536585145U, 996U, 3577259204U }, - { 1008129373U, 642U, 1478080776U }, - { 1740775604U, 939U, 1264980372U }, - { 1967883163U, 508U, 10734624U }, - { 1923019697U, 730U, 3821419629U }, - { 442079932U, 560U, 3440032343U }, - { 1961302714U, 845U, 841962572U }, - { 2030205964U, 962U, 1325144227U }, - { 1160407529U, 507U, 240940858U }, - { 635482502U, 779U, 4200489746U }, - { 1252788931U, 699U, 867195434U }, - { 1961817131U, 719U, 668237657U }, - { 1071468216U, 983U, 917876630U }, - { 1281848367U, 932U, 1003100039U }, - { 582537119U, 780U, 1127273778U }, - { 1973672777U, 853U, 1071368872U }, - { 1896756996U, 762U, 1127851055U }, - { 847917054U, 500U, 1717499075U }, - { 1240520510U, 951U, 2849576657U }, - { 1685071682U, 567U, 1961810396U }, - { 1516232129U, 557U, 3173877U }, - { 1208118903U, 612U, 1613145022U }, - { 1817269927U, 693U, 4279122573U }, - { 1510091701U, 717U, 638191229U }, - { 365916850U, 807U, 600424314U }, - { 399324359U, 702U, 1803598116U }, - { 1318480274U, 779U, 2074237022U }, - { 697758115U, 840U, 1483639402U }, - { 1696507773U, 840U, 577415447U }, - { 2081979121U, 981U, 3041486449U }, - { 955646687U, 742U, 3846494357U }, - { 1250683506U, 749U, 836419859U }, - { 595003102U, 534U, 366794109U }, - { 47485338U, 558U, 3521120834U }, - { 619433479U, 610U, 3991783875U }, - { 704096520U, 518U, 4139493852U }, - { 1712224984U, 606U, 2393312003U }, - { 1318233152U, 922U, 3880361134U }, - { 855572992U, 761U, 1472974787U }, - { 64721421U, 703U, 683860550U }, - { 678931758U, 840U, 380616043U }, - { 692711973U, 778U, 1382361947U }, - { 677703619U, 530U, 2826914161U }, - { 92393223U, 586U, 1522128471U }, - { 1222592920U, 743U, 3466726667U }, - { 358288986U, 695U, 1091956998U }, - { 1935056945U, 958U, 514864477U }, - { 735675993U, 990U, 1294239989U }, - { 1560089402U, 897U, 2238551287U }, - { 70616361U, 829U, 22483098U }, - { 368234700U, 731U, 2913875084U }, - { 20221190U, 879U, 1564152970U }, - { 539444654U, 682U, 1835141259U }, - { 1314987297U, 840U, 1801114136U }, - { 2019295544U, 645U, 3286438930U }, - { 469023838U, 716U, 1637918202U }, - { 1843754496U, 653U, 2562092152U }, - { 400672036U, 809U, 4264212785U }, - { 404722249U, 965U, 2704116999U }, - { 600702209U, 758U, 584979986U }, - { 519953954U, 667U, 2574436237U }, - { 1658071126U, 694U, 2214569490U }, - { 420480037U, 749U, 3430010866U }, - { 690103647U, 969U, 3700758083U }, - { 1029424799U, 937U, 3787746841U }, - { 2012608669U, 506U, 3362628973U }, - { 1535432887U, 998U, 42610943U }, - { 1330635533U, 857U, 3040806504U }, - { 1223800550U, 539U, 3954229517U }, - { 1322411537U, 680U, 3223250324U }, - { 1877847898U, 945U, 2915147143U }, - { 1646356099U, 874U, 965988280U }, - { 805687536U, 744U, 4032277920U }, - { 1948093210U, 633U, 1346597684U }, - { 392609744U, 783U, 1636083295U }, - { 690241304U, 770U, 1201031298U }, - { 1360302965U, 696U, 1665394461U }, - { 1220090946U, 780U, 1316922812U }, - { 447092251U, 500U, 3438743375U }, - { 1613868791U, 592U, 828546883U }, - { 523430951U, 548U, 2552392304U }, - { 726692899U, 810U, 1656872867U }, - { 1364340021U, 836U, 3710513486U }, - { 1986257729U, 931U, 935013962U }, - { 407983964U, 921U, 728767059U }, -}; - -static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed) -{ -#define LCG(x) ((x) * 69069U) /* super-duper LCG */ - state->s1 = __seed(LCG(seed), 2U); - state->s2 = __seed(LCG(state->s1), 8U); - state->s3 = __seed(LCG(state->s2), 16U); - state->s4 = __seed(LCG(state->s3), 128U); -} - -static int __init prandom_state_selftest(void) -{ - int i, j, errors = 0, runs = 0; - bool error = false; - - for (i = 0; i < ARRAY_SIZE(test1); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test1[i].seed); - prandom_warmup(&state); - - if (test1[i].result != prandom_u32_state(&state)) - error = true; - } - - if (error) - pr_warn("prandom: seed boundary self test failed\n"); - else - pr_info("prandom: seed boundary self test passed\n"); - - for (i = 0; i < ARRAY_SIZE(test2); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test2[i].seed); - prandom_warmup(&state); - - for (j = 0; j < test2[i].iteration - 1; j++) - prandom_u32_state(&state); - - if (test2[i].result != prandom_u32_state(&state)) - errors++; - - runs++; - cond_resched(); - } - - if (errors) - pr_warn("prandom: %d/%d self tests failed\n", errors, runs); - else - pr_info("prandom: %d self tests passed\n", runs); - return 0; -} -core_initcall(prandom_state_selftest); -#endif diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 40cfb38ac919..ef975510ec38 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #define HASH_DEFAULT_SIZE 64UL #define HASH_MIN_SIZE 4U diff --git a/lib/scatterlist.c b/lib/scatterlist.c index b7fe91ef35b8..6ea40d2e6247 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1366,6 +1366,7 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter, sg_max--; maxsize -= len; + start += len; ret += len; if (maxsize <= 0 || sg_max == 0) break; diff --git a/lib/string.c b/lib/string.c index 45ca1cfe0184..a3778d5aab4a 100644 --- a/lib/string.c +++ b/lib/string.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -109,11 +110,8 @@ ssize_t sized_strscpy(char *dest, const char *src, size_t count) * If src is unaligned, don't cross a page boundary, * since we don't know if the next page is mapped. */ - if ((long)src & (sizeof(long) - 1)) { - size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)); - if (limit < max) - max = limit; - } + if ((long)src & (sizeof(long) - 1)) + max = min(PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)), max); #else /* If src or dest is unaligned, don't do word-at-a-time. */ if (((long) dest | (long) src) & (sizeof(long) - 1)) @@ -823,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index ee87fef66cb5..9255234752db 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c @@ -198,6 +198,7 @@ static void __init test_kstrtoull_fail(void) {"10000000000000000000000000000000000000000000000000000000000000000", 2}, {"2000000000000000000000", 8}, {"18446744073709551616", 10}, + {"569202370375329612767", 10}, {"10000000000000000", 16}, /* negative */ {"-0", 0}, @@ -275,9 +276,11 @@ static void __init test_kstrtoll_fail(void) {"9223372036854775809", 10}, {"18446744073709551614", 10}, {"18446744073709551615", 10}, + {"569202370375329612767", 10}, {"-9223372036854775809", 10}, {"-18446744073709551614", 10}, {"-18446744073709551615", 10}, + {"-569202370375329612767", 10}, /* sign is first character if any */ {"-+1", 0}, {"-+1", 8}, @@ -334,6 +337,7 @@ static void __init test_kstrtou64_fail(void) {"-1", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, }; TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail); } @@ -386,6 +390,8 @@ static void __init test_kstrtos64_fail(void) {"18446744073709551615", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, + {"-569202370375329612767", 10}, }; TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail); } @@ -703,6 +709,182 @@ static void __init test_kstrtos8_fail(void) TEST_FAIL(kstrtos8, s8, "%hhd", test_s8_fail); } +static void __init test_kstrtoudec64_ok(void) +{ + DECLARE_TEST_OK(u64, struct test_udec64); + static DEFINE_TEST_OK(struct test_udec64, test_udec64_ok) = { + /* basic: integer.fraction, exact digits */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + {"42.0", 1, 420}, + /* zero */ + {"0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"0", 1, 0}, + {"42", 3, 42000}, + {"123.", 2, 12300}, + {"1", 1, 10}, + /* fractional only (leading dot) */ + {".5", 1, 5}, + {".5", 0, 0}, + {".123", 3, 123}, + {".001", 3, 1}, + /* zero padding: fewer fractional digits than scale */ + {"1.2", 3, 1200}, + {"1.2", 6, 1200000}, + {"0.01", 3, 10}, + {"0.1", 9, 100000000ULL}, + {"0.01", 9, 10000000}, + /* truncation: more fractional digits than scale */ + {"1.23456", 3, 1234}, + {"3.1415926535", 6, 3141592}, + {"0.999999999", 3, 999}, + {"1.99", 1, 19}, + {"1.234", 0, 1}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"42\n", 3, 42000}, + /* plus sign */ + {"+1.5", 1, 15}, + {"+.5", 1, 5}, + /* scale progression */ + {"1.", 0, 1}, + {"1.0", 1, 10}, + {"1.00", 2, 100}, + {"1.000", 3, 1000}, + {"1.000000", 6, 1000000}, + {"1.000000000", 9, 1000000000ULL}, + /* max limit check */ + {"18446744073.709551615", 9, ULLONG_MAX}, + {"18446744073709.551615", 6, ULLONG_MAX}, + {"0.18446744073709551615", 20, ULLONG_MAX}, + /* scale > 19: representable when integer part is small */ + {"0.00000000000000000001", 20, 1}, + {"0.1", 20, 10000000000000000000ULL}, + {"0.00000000000000000000001", 23, 1}, + /* truncation with scale > 19 */ + {"0.0000000000000000000012345", 23, 123}, + /* truncation with many excess digits */ + {"0.00000000000000000000000000000000423", 34, 42}, + {"1.99999999999999999999999999999999999", 3, 1999}, + }; + TEST_OK(kstrtoudec64, u64, "%llu", test_udec64_ok); +} + +static void __init test_kstrtoudec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_udec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* minus sign (unsigned) */ + {"-1.5", 1}, + {"-0.5", 1}, + /* only a decimal point */ + {".", 3}, + {".", 0}, + /* only a sign */ + {"+", 3}, + /* non-digit characters */ + {"abc", 3}, + {"1.2x", 3}, + /* leading/trailing space */ + {" 1.5", 1}, + {"1.5 ", 1}, + /* overflow */ + {"18446744073710.551615", 6}, + {"99999999999999999999", 1}, + /* overflow with scale > 19 */ + {"1.0", 21}, + {"0.2", 20}, + {"0.18446744073709551616", 20}, + {"1", 20}, + }; + TEST_FAIL(kstrtoudec64, u64, "%llu", test_udec64_fail); +} + +static void __init test_kstrtodec64_ok(void) +{ + DECLARE_TEST_OK(s64, struct test_dec64); + static DEFINE_TEST_OK(struct test_dec64, test_dec64_ok) = { + /* basic positive */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + /* basic negative */ + {"-1.5", 1, -15}, + {"-1.234", 3, -1234}, + {"-0.5", 1, -5}, + {"-0.001", 3, -1}, + /* zero (signed) */ + {"-0", 1, 0}, + {"-0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"42", 3, 42000}, + {"-42", 3, -42000}, + /* fractional only */ + {".5", 1, 5}, + {"-.5", 1, -5}, + /* zero padding */ + {"1.2", 3, 1200}, + {"-1.2", 3, -1200}, + {"0.01", 3, 10}, + {"-0.01", 3, -10}, + /* truncation */ + {"1.23456", 3, 1234}, + {"-1.23456", 3, -1234}, + {"0.999999999", 3, 999}, + {"-0.999999999", 3, -999}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"-1.5\n", 1, -15}, + /* plus sign */ + {"+1.5", 1, 15}, + /* limits */ + {"9223372036.854775807", 9, LLONG_MAX}, + {"-9223372036.854775808", 9, LLONG_MIN}, + /* scale > 19 */ + {"0.0", 20, 0}, + {"-0.0", 20, 0}, + {"0.00000000000000000001", 20, 1}, + {"-0.00000000000000000001", 20, -1}, + {"0.009223372036854775807", 21, LLONG_MAX}, + {"-0.009223372036854775808", 21, LLONG_MIN}, + }; + TEST_OK(kstrtodec64, s64, "%lld", test_dec64_ok); +} + +static void __init test_kstrtodec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_dec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* no digits after dot */ + {".", 3}, + {"-.", 3}, + /* no digits at all */ + {"+", 3}, + {"-", 3}, + /* non-digit characters */ + {"abc", 3}, + {"-1.2x", 3}, + /* signed overflow */ + {"9223372036.854775808", 9}, + {"-9223372036.854775809", 9}, + {"99999999999999999999", 1}, + /* signed overflow with scale > 19 */ + {"0.1", 20}, + {"-0.1", 20}, + {"0.09223372036854775808", 20}, + {"-0.09223372036854775809", 20}, + }; + TEST_FAIL(kstrtodec64, s64, "%lld", test_dec64_fail); +} + static int __init test_kstrtox_init(void) { test_kstrtoull_ok(); @@ -729,6 +911,12 @@ static int __init test_kstrtox_init(void) test_kstrtou8_fail(); test_kstrtos8_ok(); test_kstrtos8_fail(); + + test_kstrtoudec64_ok(); + test_kstrtoudec64_fail(); + test_kstrtodec64_ok(); + test_kstrtodec64_fail(); + return -EINVAL; } module_init(test_kstrtox_init); diff --git a/lib/test_firmware.c b/lib/test_firmware.c index b471d720879a..7459bba65444 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -867,7 +867,7 @@ static int test_fw_run_batch_request(void *data) if (test_fw_config->into_buf) { void *test_buf; - test_buf = kzalloc(TEST_FIRMWARE_BUF_SIZE, GFP_KERNEL); + test_buf = kzalloc(test_fw_config->buf_size, GFP_KERNEL); if (!test_buf) return -ENOMEM; diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 9c59d1ceb5b5..45c0cb992218 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -407,7 +407,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; @@ -581,7 +581,7 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice, devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; break; case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: - devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? + devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? spm_addr_dev0 : spm_addr_dev1; devmem->pagemap.range.end = devmem->pagemap.range.start + diff --git a/lib/test_ida.c b/lib/test_ida.c index 63078f8dc13f..c400b24f88b6 100644 --- a/lib/test_ida.c +++ b/lib/test_ida.c @@ -256,6 +256,20 @@ static void ida_check_find_first(struct ida *ida) ida_free(ida, (1 << 20) - 1); IDA_BUG_ON(ida, !ida_is_empty(ida)); + + /* + * Test cross-chunk search. + * Allocate ID in chunk 0 and ID in chunk 1. + * Search for ID >= 1. min=1 maps to chunk 0. Chunk 0 has no IDs >= 1. + * It should continue to chunk 1 and return 1024. + */ + IDA_BUG_ON(ida, ida_alloc_min(ida, 0, GFP_KERNEL) != 0); + IDA_BUG_ON(ida, ida_alloc_min(ida, 1024, GFP_KERNEL) != 1024); + IDA_BUG_ON(ida, ida_find_first_range(ida, 1, INT_MAX) != 1024); + ida_free(ida, 0); + ida_free(ida, 1024); + + IDA_BUG_ON(ida, !ida_is_empty(ida)); } static DEFINE_IDA(ida); diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 7e9c2fa52e35..c045b82169bc 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_BITS_TEST) += test_bits.o +obj-$(CONFIG_SHDI3_KUNIT_TEST) += shdi3_kunit.o obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o @@ -42,6 +43,7 @@ CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-o obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o # GCC < 12.1 can miscompile errptr() test when branch profiling is enabled. CFLAGS_printf_kunit.o += -DDISABLE_BRANCH_PROFILING +obj-$(CONFIG_PRANDOM_KUNIT_TEST) += random32_kunit.o obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o obj-$(CONFIG_RANDSTRUCT_KUNIT_TEST) += randstruct_kunit.o obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o diff --git a/lib/tests/cmdline_kunit.c b/lib/tests/cmdline_kunit.c index c1602f797637..3f61ff8d3178 100644 --- a/lib/tests/cmdline_kunit.c +++ b/lib/tests/cmdline_kunit.c @@ -6,6 +6,7 @@ #include #include #include +#include #include static const char *cmdline_test_strings[] = { @@ -139,11 +140,128 @@ static void cmdline_test_range(struct kunit *test) } while (++i < ARRAY_SIZE(cmdline_test_range_strings)); } +static void cmdline_test_next_arg_quoted_value(struct kunit *test) +{ + char in[] = "foo=\"bar baz\" qux=1"; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar baz"); + KUNIT_EXPECT_STREQ(test, next, "qux=1"); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "qux"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "1"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_bare_quote_regression(struct kunit *test) +{ + char in[] = "foo=bar \""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar"); + KUNIT_EXPECT_STREQ(test, next, "\""); + + /* This hits the i == 0 quoted-token case fixed by 9847f21225c4. */ + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, ""); + KUNIT_EXPECT_PTR_EQ(test, val, NULL); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_mixed_tokens(struct kunit *test) +{ + char in[] = "bbb= jjj kkk=\"a=b\""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "bbb"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, ""); + KUNIT_EXPECT_STREQ(test, next, "jjj kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "jjj"); + KUNIT_EXPECT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, next, "kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "kkk"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "a=b"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +struct cmdline_test_memparse_entry { + const char *input; + const char *unrecognized; + unsigned long long result; +}; + +static const struct cmdline_test_memparse_entry testdata[] = { + { "0", "", 0ULL }, + { "1", "", 1ULL }, + { "a", "a", 0ULL }, + { "k", "k", 0ULL }, + { "E", "E", 0ULL }, + { "0xb", "", 11ULL }, + { "0xz", "x", 0ULL }, + { "1234", "", 1234ULL }, + { "04567", "", 2423ULL }, + { "0x9876", "", 39030LL }, + { "05678", "8", 375ULL }, + { "0xabcdefz", "z", 11259375ULL }, + { "0cdba", "c", 0ULL }, + { "4K", "", SZ_4K }, + { "0x10k@0xaaaabbbb", "@", SZ_16K }, + { "32M", "", SZ_32M }, + { "067m:foo", ":", 55 * SZ_1M }, + { "2G;bar=baz", ";", SZ_2G }, + { "07gz", "z", 7ULL * SZ_1G }, + { "3T+data", "+", 3 * SZ_1T }, + { "04t,ro", ",", SZ_4T }, + { "012p", "", 11258999068426240ULL }, + { "7P,sync", ",", 7881299347898368ULL }, + { "0x2e", "", 46ULL }, + { "2E and more", " ", 2305843009213693952ULL }, + { "18446744073709551615", "", ULLONG_MAX }, + { "0xffffffffffffffff0", "", ULLONG_MAX }, + { "1111111111111111111T", "", ULLONG_MAX }, + { "222222222222222222222G", "", ULLONG_MAX }, + { "3333333333333333333333M", "", ULLONG_MAX }, +}; + +static void cmdline_test_memparse(struct kunit *test) +{ + const struct cmdline_test_memparse_entry *e; + unsigned long long ret; + char *retptr; + + for (e = testdata; e < testdata + ARRAY_SIZE(testdata); e++) { + ret = memparse(e->input, &retptr); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when parsing '%s'", e->input); + KUNIT_EXPECT_EQ_MSG(test, *retptr, *e->unrecognized, + " when parsing '%s'", e->input); + } +} + static struct kunit_case cmdline_test_cases[] = { KUNIT_CASE(cmdline_test_noint), KUNIT_CASE(cmdline_test_lead_int), KUNIT_CASE(cmdline_test_tail_int), KUNIT_CASE(cmdline_test_range), + KUNIT_CASE(cmdline_test_next_arg_quoted_value), + KUNIT_CASE(cmdline_test_next_arg_bare_quote_regression), + KUNIT_CASE(cmdline_test_next_arg_mixed_tokens), + KUNIT_CASE(cmdline_test_memparse), {} }; diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index f02f7b7aa796..d9690ba1db88 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -53,7 +53,7 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, size_t npages) { struct page **pages; - unsigned long got; + unsigned long got, last; void *buffer; unsigned int i; @@ -61,7 +61,15 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); *ppages = pages; - got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + got = 0; + while (true) { + last = got; + got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + + if (last == got || got == npages) + break; + } + if (got != npages) { release_pages(pages, got); kvfree(pages); @@ -275,7 +283,7 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) struct page **spages, **bpages; u8 *scratch, *buffer; size_t bufsize, npages, size, copied; - int i, b, patt; + int i, patt; bufsize = 0x100000; npages = bufsize / PAGE_SIZE; @@ -298,10 +306,9 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); /* Build the expected image in the scratch buffer. */ - b = 0; patt = 0; memset(scratch, 0, bufsize); - for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { + for (pr = bvec_test_ranges; pr->from >= 0; pr++) { u8 *p = scratch + pr->page * PAGE_SIZE; for (i = pr->from; i < pr->to; i++) diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c index e4b0ecbee32f..4c08a7c6fb78 100644 --- a/lib/tests/liveupdate.c +++ b/lib/tests/liveupdate.c @@ -105,6 +105,9 @@ static void liveupdate_test_init(void) pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n", flb->compatible, ERR_PTR(err)); } + + if (!err) + liveupdate_flb_put_incoming(flb); } initialized = true; } diff --git a/lib/tests/random32_kunit.c b/lib/tests/random32_kunit.c new file mode 100644 index 000000000000..0b4af2b09c01 --- /dev/null +++ b/lib/tests/random32_kunit.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for random32 functions. + */ + +#include +#include + +/* prandom_warmup() is static in lib/random32.c; exposed for testing only. */ +void prandom_warmup(struct rnd_state *state); + +static const struct prandom_test1 { + u32 seed; + u32 result; +} test1[] = { + { 1U, 3484351685U }, + { 2U, 2623130059U }, + { 3U, 3125133893U }, + { 4U, 984847254U }, +}; + +static const struct prandom_test2 { + u32 seed; + u32 iteration; + u32 result; +} test2[] = { + /* Test cases against taus113 from GSL library. */ + { 931557656U, 959U, 2975593782U }, + { 1339693295U, 876U, 3887776532U }, + { 1545556285U, 961U, 1615538833U }, + { 601730776U, 723U, 1776162651U }, + { 1027516047U, 687U, 511983079U }, + { 416526298U, 700U, 916156552U }, + { 1395522032U, 652U, 2222063676U }, + { 366221443U, 617U, 2992857763U }, + { 1539836965U, 714U, 3783265725U }, + { 556206671U, 994U, 799626459U }, + { 684907218U, 799U, 367789491U }, + { 2121230701U, 931U, 2115467001U }, + { 1668516451U, 644U, 3620590685U }, + { 768046066U, 883U, 2034077390U }, + { 1989159136U, 833U, 1195767305U }, + { 536585145U, 996U, 3577259204U }, + { 1008129373U, 642U, 1478080776U }, + { 1740775604U, 939U, 1264980372U }, + { 1967883163U, 508U, 10734624U }, + { 1923019697U, 730U, 3821419629U }, + { 442079932U, 560U, 3440032343U }, + { 1961302714U, 845U, 841962572U }, + { 2030205964U, 962U, 1325144227U }, + { 1160407529U, 507U, 240940858U }, + { 635482502U, 779U, 4200489746U }, + { 1252788931U, 699U, 867195434U }, + { 1961817131U, 719U, 668237657U }, + { 1071468216U, 983U, 917876630U }, + { 1281848367U, 932U, 1003100039U }, + { 582537119U, 780U, 1127273778U }, + { 1973672777U, 853U, 1071368872U }, + { 1896756996U, 762U, 1127851055U }, + { 847917054U, 500U, 1717499075U }, + { 1240520510U, 951U, 2849576657U }, + { 1685071682U, 567U, 1961810396U }, + { 1516232129U, 557U, 3173877U }, + { 1208118903U, 612U, 1613145022U }, + { 1817269927U, 693U, 4279122573U }, + { 1510091701U, 717U, 638191229U }, + { 365916850U, 807U, 600424314U }, + { 399324359U, 702U, 1803598116U }, + { 1318480274U, 779U, 2074237022U }, + { 697758115U, 840U, 1483639402U }, + { 1696507773U, 840U, 577415447U }, + { 2081979121U, 981U, 3041486449U }, + { 955646687U, 742U, 3846494357U }, + { 1250683506U, 749U, 836419859U }, + { 595003102U, 534U, 366794109U }, + { 47485338U, 558U, 3521120834U }, + { 619433479U, 610U, 3991783875U }, + { 704096520U, 518U, 4139493852U }, + { 1712224984U, 606U, 2393312003U }, + { 1318233152U, 922U, 3880361134U }, + { 855572992U, 761U, 1472974787U }, + { 64721421U, 703U, 683860550U }, + { 678931758U, 840U, 380616043U }, + { 692711973U, 778U, 1382361947U }, + { 677703619U, 530U, 2826914161U }, + { 92393223U, 586U, 1522128471U }, + { 1222592920U, 743U, 3466726667U }, + { 358288986U, 695U, 1091956998U }, + { 1935056945U, 958U, 514864477U }, + { 735675993U, 990U, 1294239989U }, + { 1560089402U, 897U, 2238551287U }, + { 70616361U, 829U, 22483098U }, + { 368234700U, 731U, 2913875084U }, + { 20221190U, 879U, 1564152970U }, + { 539444654U, 682U, 1835141259U }, + { 1314987297U, 840U, 1801114136U }, + { 2019295544U, 645U, 3286438930U }, + { 469023838U, 716U, 1637918202U }, + { 1843754496U, 653U, 2562092152U }, + { 400672036U, 809U, 4264212785U }, + { 404722249U, 965U, 2704116999U }, + { 600702209U, 758U, 584979986U }, + { 519953954U, 667U, 2574436237U }, + { 1658071126U, 694U, 2214569490U }, + { 420480037U, 749U, 3430010866U }, + { 690103647U, 969U, 3700758083U }, + { 1029424799U, 937U, 3787746841U }, + { 2012608669U, 506U, 3362628973U }, + { 1535432887U, 998U, 42610943U }, + { 1330635533U, 857U, 3040806504U }, + { 1223800550U, 539U, 3954229517U }, + { 1322411537U, 680U, 3223250324U }, + { 1877847898U, 945U, 2915147143U }, + { 1646356099U, 874U, 965988280U }, + { 805687536U, 744U, 4032277920U }, + { 1948093210U, 633U, 1346597684U }, + { 392609744U, 783U, 1636083295U }, + { 690241304U, 770U, 1201031298U }, + { 1360302965U, 696U, 1665394461U }, + { 1220090946U, 780U, 1316922812U }, + { 447092251U, 500U, 3438743375U }, + { 1613868791U, 592U, 828546883U }, + { 523430951U, 548U, 2552392304U }, + { 726692899U, 810U, 1656872867U }, + { 1364340021U, 836U, 3710513486U }, + { 1986257729U, 931U, 935013962U }, + { 407983964U, 921U, 728767059U }, +}; + +static void prandom_state_test_seed(struct rnd_state *state, u32 seed) +{ +#define LCG(x) ((x) * 69069U) /* super-duper LCG */ + state->s1 = __seed(LCG(seed), 2U); + state->s2 = __seed(LCG(state->s1), 8U); + state->s3 = __seed(LCG(state->s2), 16U); + state->s4 = __seed(LCG(state->s3), 128U); +} + +static void test_prandom_seed_boundary(struct kunit *test) +{ + int i; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test1); i++) { + prandom_state_test_seed(&state, test1[i].seed); + prandom_warmup(&state); + KUNIT_EXPECT_EQ(test, test1[i].result, prandom_u32_state(&state)); + } +} + +static void test_prandom_taus113(struct kunit *test) +{ + int i, j; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test2); i++) { + prandom_state_test_seed(&state, test2[i].seed); + prandom_warmup(&state); + + for (j = 0; j < test2[i].iteration - 1; j++) + prandom_u32_state(&state); + + KUNIT_EXPECT_EQ(test, test2[i].result, prandom_u32_state(&state)); + } +} + +static struct kunit_case prandom_test_cases[] = { + KUNIT_CASE(test_prandom_seed_boundary), + KUNIT_CASE(test_prandom_taus113), + {} +}; + +static struct kunit_suite prandom_test_suite = { + .name = "prandom", + .test_cases = prandom_test_cases, +}; + +kunit_test_suite(prandom_test_suite); + +MODULE_DESCRIPTION("KUnit test for prandom"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/lib/tests/shdi3_kunit.c b/lib/tests/shdi3_kunit.c new file mode 100644 index 000000000000..44f65e66512b --- /dev/null +++ b/lib/tests/shdi3_kunit.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR Apache-2.0 +/* + * Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3(). + */ + +#include +#include +#include +#include + +struct shdi3_test_entry { + long long input; + int shift; + long long result; +}; + +static const struct shdi3_test_entry ashldi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashldi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x2468ACF13579BDELL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BCLL }, + { 0x0123456789ABCDEFLL, 3, 0x91A2B3C4D5E6F78LL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDEF0LL }, + { 0x0123456789ABCDEFLL, 28, 0x789ABCDEF0000000LL }, + { 0x0123456789ABCDEFLL, 29, 0xF13579BDE0000000LL }, + { 0x0123456789ABCDEFLL, 30, 0xE26AF37BC0000000LL }, + { 0x0123456789ABCDEFLL, 31, 0xC4D5E6F780000000LL }, + { 0x0123456789ABCDEFLL, 32, 0x89ABCDEF00000000LL }, + { 0x0123456789ABCDEFLL, 33, 0x13579BDE00000000LL }, + { 0x0123456789ABCDEFLL, 34, 0x26AF37BC00000000LL }, + { 0x0123456789ABCDEFLL, 35, 0x4D5E6F7800000000LL }, + { 0x0123456789ABCDEFLL, 36, 0x9ABCDEF000000000LL }, + { 0x0123456789ABCDEFLL, 60, 0xF000000000000000LL }, + { 0x0123456789ABCDEFLL, 61, 0xE000000000000000LL }, + { 0x0123456789ABCDEFLL, 62, 0xC000000000000000LL }, + { 0x0123456789ABCDEFLL, 63, 0x8000000000000000LL }, +}; + +static void shdi3_test_ashldi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashldi3_testdata; + e < ashldi3_testdata + ARRAY_SIZE(ashldi3_testdata); e++) { + ret = __ashldi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashldi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry ashrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0xFF6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0xFFB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0xFFDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFFFFFFFFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0xFFFFFFFFF6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0xFFFFFFFFFB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0xFFFFFFFFFDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFFFFFFFFFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0xFFFFFFFFFF6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0xFFFFFFFFFFB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0xFFFFFFFFFFDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFFFFFFFFFFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xFFFFFFFFFFFFFFFALL }, + { 0xAEDCBA9876543210LL, 61, 0xFFFFFFFFFFFFFFFDLL }, + { 0xAEDCBA9876543210LL, 62, 0xFFFFFFFFFFFFFFFELL }, + { 0xAEDCBA9876543210LL, 63, 0xFFFFFFFFFFFFFFFFLL }, +}; + +static void shdi3_test_ashrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashrdi3_testdata; + e < ashrdi3_testdata + ARRAY_SIZE(ashrdi3_testdata); e++) { + ret = __ashrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry lshrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/lshrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0x7F6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0x3FB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0x1FDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0x7F6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0x3FB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0x1FDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0x7F6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0x3FB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0x1FDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xALL }, + { 0xAEDCBA9876543210LL, 61, 0x5LL }, + { 0xAEDCBA9876543210LL, 62, 0x2LL }, + { 0xAEDCBA9876543210LL, 63, 0x1LL }, +}; + +static void shdi3_test_lshrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = lshrdi3_testdata; + e < lshrdi3_testdata + ARRAY_SIZE(lshrdi3_testdata); e++) { + ret = __lshrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __lshrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static struct kunit_case shdi3_test_cases[] = { + KUNIT_CASE(shdi3_test_ashldi3), + KUNIT_CASE(shdi3_test_ashrdi3), + KUNIT_CASE(shdi3_test_lshrdi3), + {} +}; + +static struct kunit_suite shdi3_test_suite = { + .name = "shdi3", + .test_cases = shdi3_test_cases, +}; +kunit_test_suite(shdi3_test_suite); + +MODULE_DESCRIPTION("Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3()"); +MODULE_LICENSE("GPL"); diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c index c853046183d2..9fbe91079c7e 100644 --- a/lib/tests/string_helpers_kunit.c +++ b/lib/tests/string_helpers_kunit.c @@ -5,11 +5,16 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include + #include -#include +#include +#include +#include #include -#include +#include +#include #include +#include static void test_string_check_buf(struct kunit *test, const char *name, unsigned int flags, @@ -601,6 +606,11 @@ static void test_unescape(struct kunit *test) test_string_unescape(test, "unescape", i, false); test_string_unescape(test, "unescape inplace", get_random_u32_below(UNESCAPE_ALL_MASK + 1), true); +} + +static void test_escape(struct kunit *test) +{ + unsigned int i; /* Without dictionary */ for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) @@ -615,6 +625,7 @@ static struct kunit_case string_helpers_test_cases[] = { KUNIT_CASE(test_get_size), KUNIT_CASE(test_upper_lower), KUNIT_CASE(test_unescape), + KUNIT_CASE(test_escape), {} }; diff --git a/lib/tests/test_ratelimit.c b/lib/tests/test_ratelimit.c index 33cea5f3d28b..e244f8cd47d7 100644 --- a/lib/tests/test_ratelimit.c +++ b/lib/tests/test_ratelimit.c @@ -68,7 +68,6 @@ static void test_ratelimit_smoke(struct kunit *test) static struct ratelimit_state stressrl = RATELIMIT_STATE_INIT_FLAGS("stressrl", HZ / 10, 3, RATELIMIT_MSG_ON_RELEASE); -static int doneflag; static const int stress_duration = 2 * HZ; struct stress_kthread { @@ -84,9 +83,8 @@ static int test_ratelimit_stress_child(void *arg) struct stress_kthread *sktp = arg; set_user_nice(current, MAX_NICE); - WARN_ON_ONCE(!sktp->tp); - while (!READ_ONCE(doneflag)) { + while (!kthread_should_stop()) { sktp->nattempts++; if (___ratelimit(&stressrl, __func__)) sktp->nunlimited++; @@ -105,26 +103,37 @@ static void test_ratelimit_stress(struct kunit *test) const int n_stress_kthread = cpumask_weight(cpu_online_mask); struct stress_kthread skt = { 0 }; struct stress_kthread *sktp = kzalloc_objs(*sktp, n_stress_kthread); + int n_started = 0; - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); + KUNIT_ASSERT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); for (i = 0; i < n_stress_kthread; i++) { sktp[i].tp = kthread_run(test_ratelimit_stress_child, &sktp[i], "%s/%i", "test_ratelimit_stress_child", i); - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "kthread creation failure"); + if (IS_ERR(sktp[i].tp)) { + KUNIT_FAIL(test, "kthread_run failed: %ld", PTR_ERR(sktp[i].tp)); + goto out_stop; + } + n_started++; pr_alert("Spawned test_ratelimit_stress_child %d\n", i); } schedule_timeout_idle(stress_duration); - WRITE_ONCE(doneflag, 1); - for (i = 0; i < n_stress_kthread; i++) { + +out_stop: + for (i = 0; i < n_started; i++) { kthread_stop(sktp[i].tp); skt.nattempts += sktp[i].nattempts; skt.nunlimited += sktp[i].nunlimited; skt.nlimited += sktp[i].nlimited; skt.nmissed += sktp[i].nmissed; } - KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, - "Outcomes not equal to attempts"); - KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, "Misses not equal to limits"); + if (n_started == n_stress_kthread) { + KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, + "Outcomes not equal to attempts"); + KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, + "Misses not equal to limits"); + } + + kfree(sktp); } static struct kunit_case ratelimit_test_cases[] = { diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c index de71b2649dac..2ef64fbe67d6 100644 --- a/lib/tests/uuid_kunit.c +++ b/lib/tests/uuid_kunit.c @@ -86,11 +86,67 @@ static void uuid_test_uuid_invalid(struct kunit *test) } } +/* + * RFC 4122 section 4.4 says random UUIDs/GUIDs (version 4) must have: + * - version 4 in the high nibble of the version byte, + * - variant DCE 1.1 (binary 10x) in the high bits of byte 8. + * + * The version byte is byte 6 in the "wire" uuid_t layout and byte 7 in + * the byte-swapped guid_t layout. + */ +static void uuid_test_uuid_gen(struct kunit *test) +{ + uuid_t u; + + for (unsigned int i = 0; i < 8; i++) { + uuid_gen(&u); + KUNIT_EXPECT_EQ(test, u.b[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, u.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_guid_gen(struct kunit *test) +{ + guid_t g; + + for (unsigned int i = 0; i < 8; i++) { + guid_gen(&g); + KUNIT_EXPECT_EQ(test, g.b[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, g.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_uuid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_uuid(buf); + KUNIT_EXPECT_EQ(test, buf[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_guid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_guid(buf); + KUNIT_EXPECT_EQ(test, buf[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + static struct kunit_case uuid_test_cases[] = { KUNIT_CASE(uuid_test_guid_valid), KUNIT_CASE(uuid_test_uuid_valid), KUNIT_CASE(uuid_test_guid_invalid), KUNIT_CASE(uuid_test_uuid_invalid), + KUNIT_CASE(uuid_test_uuid_gen), + KUNIT_CASE(uuid_test_guid_gen), + KUNIT_CASE(uuid_test_generate_random_uuid), + KUNIT_CASE(uuid_test_generate_random_guid), {}, }; diff --git a/lib/usercopy.c b/lib/usercopy.c index b00a3a957de6..e2f0bf104a59 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,15 +12,13 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) +#if !defined(INLINE_COPY_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); } EXPORT_SYMBOL(_copy_from_user); -#endif -#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 2bc6ef483576..1d95ab9314bf 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -86,7 +86,7 @@ static unsigned long long simple_strntoull(const char *startp, char **endp, unsi cp = _parse_integer_fixup_radix(startp, &base); prefix_chars = cp - startp; if (prefix_chars < max_chars) { - rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars); + rv = _parse_integer(cp, base, &result, max_chars - prefix_chars); /* FIXME */ cp += (rv & ~KSTRTOX_OVERFLOW); } else { @@ -2360,6 +2360,9 @@ void __init hash_pointers_finalize(bool slub_debug) static int __init hash_pointers_mode_parse(char *str) { + /* Avoid stale no_hash_pointers state when hash_pointers overrides it */ + no_hash_pointers = false; + if (!str) { pr_warn("Hash pointers mode empty; falling back to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; @@ -2369,6 +2372,7 @@ static int __init hash_pointers_mode_parse(char *str) } else if (strcmp(str, "never") == 0) { pr_info("Hash pointers mode set to never.\n"); hash_pointers_mode = HASH_PTR_NEVER; + no_hash_pointers = true; } else if (strcmp(str, "always") == 0) { pr_info("Hash pointers mode set to always.\n"); hash_pointers_mode = HASH_PTR_ALWAYS; diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index cc49a300a5b2..88922323f96e 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -466,7 +466,7 @@ static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b) { size_t copy_size; - copy_size = min_t(size_t, s->temp.filtered, b->out_size - b->out_pos); + copy_size = min(s->temp.filtered, b->out_size - b->out_pos); memcpy(b->out + b->out_pos, s->temp.buf, copy_size); b->out_pos += copy_size; diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 4b783ac94e71..46c2df6ad6f5 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -135,14 +135,16 @@ struct lzma_dec { uint32_t rep2; uint32_t rep3; - /* Types of the most recently seen LZMA symbols */ - enum lzma_state state; - /* * Length of a match. This is updated so that dict_repeat can - * be called again to finish repeating the whole match. + * be called again to finish repeating the whole match. This is + * size_t because a pointer to this is passed to dict_repeat, + * and there it's nicer to have size_t instead of uint32_t. */ - uint32_t len; + size_t len; + + /* Types of the most recently seen LZMA symbols */ + enum lzma_state state; /* * LZMA properties or related bit masks (number of literal @@ -228,13 +230,13 @@ struct lzma2_dec { enum lzma2_seq next_sequence; /* Uncompressed size of LZMA chunk (2 MiB at maximum) */ - uint32_t uncompressed; + size_t uncompressed; /* * Compressed size of LZMA chunk or compressed/uncompressed * size of uncompressed chunk (64 KiB at maximum) */ - uint32_t compressed; + size_t compressed; /* * True if dictionary reset is needed. This is false before @@ -269,11 +271,11 @@ struct xz_dec_lzma2 { struct lzma_dec lzma; /* - * Temporary buffer which holds small number of input bytes between + * Temporary buffer which holds a small number of input bytes between * decoder calls. See lzma2_lzma() for details. */ struct { - uint32_t size; + size_t size; uint8_t buf[3 * LZMA_IN_REQUIRED]; } temp; }; @@ -320,7 +322,7 @@ static inline bool dict_has_space(const struct dictionary *dict) * still empty. This special case is needed for single-call decoding to * avoid writing a '\0' to the end of the destination buffer. */ -static inline uint32_t dict_get(const struct dictionary *dict, uint32_t dist) +static inline uint32_t dict_get(const struct dictionary *dict, size_t dist) { size_t offset = dict->pos - dist - 1; @@ -346,15 +348,15 @@ static inline void dict_put(struct dictionary *dict, uint8_t byte) * invalid, false is returned. On success, true is returned and *len is * updated to indicate how many bytes were left to be repeated. */ -static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) +static bool dict_repeat(struct dictionary *dict, size_t *len, size_t dist) { size_t back; - uint32_t left; + size_t left; if (dist >= dict->full || dist >= dict->size) return false; - left = min_t(size_t, dict->limit - dict->pos, *len); + left = min(dict->limit - dict->pos, *len); *len -= left; back = dict->pos - dist - 1; @@ -375,7 +377,7 @@ static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) /* Copy uncompressed data as is from input to dictionary and output buffers. */ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, - uint32_t *left) + size_t *left) { size_t copy_size; @@ -433,7 +435,7 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, * enough space in b->out. This is guaranteed because caller uses dict_limit() * before decoding data into the dictionary. */ -static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b) +static size_t dict_flush(struct dictionary *dict, struct xz_buf *b) { size_t copy_size = dict->pos - dict->start; @@ -755,8 +757,8 @@ static bool lzma_main(struct xz_dec_lzma2 *s) uint32_t pos_state; /* - * If the dictionary was reached during the previous call, try to - * finish the possibly pending repeat in the dictionary. + * If the dictionary write limit was reached during the previous call, + * try to finish the possibly pending repeat in the dictionary. */ if (dict_has_space(&s->dict) && s->lzma.len > 0) dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0); @@ -878,7 +880,7 @@ static bool lzma_props(struct xz_dec_lzma2 *s, uint8_t props) static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b) { size_t in_avail; - uint32_t tmp; + size_t tmp; in_avail = b->in_size - b->in_pos; if (s->temp.size > 0 || s->lzma2.compressed == 0) { @@ -976,7 +978,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * an uncompressed chunk * 0x02 Uncompressed chunk (no dictionary reset) * - * Highest three bits (s->control & 0xE0): + * Highest three bits (tmp & 0xE0): * 0xE0 Dictionary reset, new properties and state * reset, followed by LZMA compressed chunk * 0xC0 New properties and state reset, followed @@ -988,7 +990,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * 0x80 LZMA chunk (no dictionary or state reset) * * For LZMA compressed chunks, the lowest five bits - * (s->control & 1F) are the highest bits of the + * (tmp & 1F) are the highest bits of the * uncompressed size (bits 16-20). * * A new LZMA2 stream must begin with a dictionary @@ -1046,25 +1048,23 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_UNCOMPRESSED_1: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] << 8; + += (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_UNCOMPRESSED_2; break; case SEQ_UNCOMPRESSED_2: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] + 1; + += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = SEQ_COMPRESSED_0; break; case SEQ_COMPRESSED_0: - s->lzma2.compressed - = (uint32_t)b->in[b->in_pos++] << 8; + s->lzma2.compressed = (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_COMPRESSED_1; break; case SEQ_COMPRESSED_1: - s->lzma2.compressed - += (uint32_t)b->in[b->in_pos++] + 1; + s->lzma2.compressed += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = s->lzma2.next_sequence; break; @@ -1091,16 +1091,15 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_LZMA_RUN: /* * Set dictionary limit to indicate how much we want - * to be encoded at maximum. Decode new data into the + * to be decoded at maximum. Decode new data into the * dictionary. Flush the new data from dictionary to * b->out. Check if we finished decoding this chunk. * In case the dictionary got full but we didn't fill * the output buffer yet, we may run this loop * multiple times without changing s->lzma2.sequence. */ - dict_limit(&s->dict, min_t(size_t, - b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; @@ -1260,8 +1259,8 @@ enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr, s->dict.end = b->out_size - b->out_pos; while (true) { - dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index 59bfd54ffee7..0bed6daefac2 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -155,8 +155,8 @@ static const uint8_t check_sizes[16] = { */ static bool fill_temp(struct xz_dec *s, struct xz_buf *b) { - size_t copy_size = min_t(size_t, - b->in_size - b->in_pos, s->temp.size - s->temp.pos); + size_t copy_size = min(b->in_size - b->in_pos, + s->temp.size - s->temp.pos); memcpy(s->temp.buf + s->temp.pos, b->in + b->in_pos, copy_size); b->in_pos += copy_size; diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h index d2632b7dfb9c..a612ce4fd450 100644 --- a/lib/xz/xz_lzma2.h +++ b/lib/xz/xz_lzma2.h @@ -143,7 +143,7 @@ static inline bool lzma_state_is_literal(enum lzma_state state) * Get the index of the appropriate probability array for decoding * the distance slot. */ -static inline uint32_t lzma_get_dist_state(uint32_t len) +static inline size_t lzma_get_dist_state(size_t len) { return len < DIST_STATES + MATCH_LEN_MIN ? len - MATCH_LEN_MIN : DIST_STATES - 1; diff --git a/localversion-next b/localversion-next new file mode 100644 index 000000000000..7218c6c4244e --- /dev/null +++ b/localversion-next @@ -0,0 +1 @@ +-next-20260706 diff --git a/mm/Kconfig b/mm/Kconfig index fe734d9bbe99..04fe5171bb8c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -703,7 +703,7 @@ config DEVICE_MIGRATION config ARCH_ENABLE_HUGEPAGE_MIGRATION bool -config ARCH_ENABLE_THP_MIGRATION +config ARCH_SUPPORTS_PMD_SOFTLEAF bool config HUGETLB_PAGE_SIZE_VARIABLE @@ -989,17 +989,6 @@ config THP_SWAP For selection by architectures with reasonable THP sizes. -config READ_ONLY_THP_FOR_FS - bool "Read-only THP for filesystems (EXPERIMENTAL)" - depends on TRANSPARENT_HUGEPAGE - - help - Allow khugepaged to put read-only file-backed pages in THP. - - This is marked experimental because it is a new feature. Write - support of file THPs will be developed in the next few release - cycles. - config NO_PAGE_MAPCOUNT bool "No per-page mapcount (EXPERIMENTAL)" help @@ -1353,7 +1342,7 @@ config ARCH_HAS_PTE_SPECIAL bool config MAPPING_DIRTY_HELPERS - bool + bool config KMAP_LOCAL bool @@ -1484,7 +1473,7 @@ config ARCH_HAS_USER_SHADOW_STACK bool help The architecture has hardware support for userspace shadow call - stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss). + stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss). config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 91b3e027b753..5737a504efbb 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS overhead in the page fault path. If in doubt, say N. + +config MEM_ALLOC_PROFILING + bool "Enable memory allocation profiling" + default n + depends on MMU + depends on PROC_FS + depends on !DEBUG_FORCE_WEAK_PER_CPU + select CODE_TAGGING + select PAGE_EXTENSION + select SLAB_OBJ_EXT + help + Track allocation source code and record total allocation size + initiated at that code location. The mechanism can be used to track + memory leaks with a low performance and memory impact. + +config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + bool "Enable memory allocation profiling by default" + default y + depends on MEM_ALLOC_PROFILING + +config MEM_ALLOC_PROFILING_DEBUG + bool "Memory allocation profiler debugging" + default n + depends on MEM_ALLOC_PROFILING + select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT + help + Adds warnings with helpful error messages for memory allocation + profiling. diff --git a/mm/Makefile b/mm/Makefile index eff9f9e7e061..4fc713867b9b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o diff --git a/lib/alloc_tag.c b/mm/alloc_tag.c similarity index 87% rename from lib/alloc_tag.c rename to mm/alloc_tag.c index ed1bdcf1f8ab..d9be1cf5187d 100644 --- a/lib/alloc_tag.c +++ b/mm/alloc_tag.c @@ -45,6 +45,7 @@ int alloc_tag_ref_offs; struct allocinfo_private { struct codetag_iterator iter; + struct codetag_iterator reported_iter; bool print_header; }; @@ -54,20 +55,24 @@ static void *allocinfo_start(struct seq_file *m, loff_t *pos) loff_t node = *pos; priv = (struct allocinfo_private *)m->private; - codetag_lock_module_list(alloc_tag_cttype, true); + codetag_lock_module_list(alloc_tag_cttype); if (node == 0) { priv->print_header = true; priv->iter = codetag_get_ct_iter(alloc_tag_cttype); - codetag_next_ct(&priv->iter); + } else { + priv->iter = priv->reported_iter; } + codetag_next_ct(&priv->iter); return priv->iter.ct ? priv : NULL; } static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) { struct allocinfo_private *priv = (struct allocinfo_private *)arg; - struct codetag *ct = codetag_next_ct(&priv->iter); + struct codetag *ct; + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); (*pos)++; if (!ct) return NULL; @@ -77,7 +82,7 @@ static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) static void allocinfo_stop(struct seq_file *m, void *arg) { - codetag_lock_module_list(alloc_tag_cttype, false); + codetag_unlock_module_list(alloc_tag_cttype); } static void print_allocinfo_header(struct seq_buf *buf) @@ -136,7 +141,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl return 0; if (can_sleep) - codetag_lock_module_list(alloc_tag_cttype, true); + codetag_lock_module_list(alloc_tag_cttype); else if (!codetag_trylock_module_list(alloc_tag_cttype)) return 0; @@ -161,7 +166,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl } } - codetag_lock_module_list(alloc_tag_cttype, false); + codetag_unlock_module_list(alloc_tag_cttype); return nr; } @@ -767,50 +772,82 @@ static __init bool need_page_alloc_tagging(void) * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. * - * Early allocations include: - * - Base allocations independent of CPU count - * - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, - * such as trace ring buffers, scheduler per-cpu data) - * - * For simplicity, we fix the size to 8192. - * If insufficient, a warning will be triggered to alert the user. + * Each page is cast to a pfn_pool: the first few bytes hold metadata + * (next pointer and slot count), the remainder stores PFNs. + */ +struct pfn_pool { + struct pfn_pool *next; + atomic_t count; + unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* + * Skip early PFN recording for a page allocation. Reuses the + * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid + * recursion when allocating pages for the early PFN tracking list + * itself. * - * TODO: Replace fixed-size array with dynamic allocation using - * a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. + * Codetags of the pages allocated with __GFP_NO_CODETAG should be + * cleared (via clear_page_tag_ref()) before freeing the pages to prevent + * alloc_tag_sub_check() from triggering a warning. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX] __initdata; -static atomic_t early_pfn_count __initdata = ATOMIC_INIT(0); +static struct pfn_pool *current_pfn_pool __initdata; static void __init __alloc_tag_add_early_pfn(unsigned long pfn) { - int old_idx, new_idx; + struct pfn_pool *pool; + int idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + pool = READ_ONCE(current_pfn_pool); + if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { + struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct pfn_pool *new; + + if (!new_page) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new = page_address(new_page); + new->next = pool; + atomic_set(&new->count, 0); + if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { + clear_page_tag_ref(new_page); + __free_page(new_page); + continue; + } + pool = new; } - new_idx = old_idx + 1; - } while (!atomic_try_cmpxchg(&early_pfn_count, &old_idx, new_idx)); + idx = atomic_read(&pool->count); + if (idx >= PFN_POOL_SIZE) + continue; + if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) + break; + } while (1); - early_pfns[old_idx] = pfn; + pool->pfns[idx] = pfn; } typedef void alloc_tag_add_func(unsigned long pfn); static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = RCU_INITIALIZER(__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn(unsigned long pfn) +void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) { alloc_tag_add_func *alloc_tag_add; if (static_key_enabled(&mem_profiling_compressed)) return; + /* Skip allocations for the tracking list itself to avoid recursion. */ + if (gfp_flags & __GFP_NO_CODETAG) + return; + rcu_read_lock(); alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); if (alloc_tag_add) @@ -820,7 +857,9 @@ void alloc_tag_add_early_pfn(unsigned long pfn) static void __init clear_early_alloc_pfn_tag_refs(void) { - unsigned int i; + struct pfn_pool *pool, *next; + struct page *page; + int i; if (static_key_enabled(&mem_profiling_compressed)) return; @@ -829,37 +868,45 @@ static void __init clear_early_alloc_pfn_tag_refs(void) /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu(); - for (i = 0; i < atomic_read(&early_pfn_count); i++) { - unsigned long pfn = early_pfns[i]; - - if (pfn_valid(pfn)) { - struct page *page = pfn_to_page(pfn); - union pgtag_ref_handle handle; - union codetag_ref ref; - - if (get_page_tag_ref(page, &ref, &handle)) { - /* - * An early-allocated page could be freed and reallocated - * after its page_ext is initialized but before we clear it. - * In that case, it already has a valid tag set. - * We should not overwrite that valid tag with CODETAG_EMPTY. - * - * Note: there is still a small race window between checking - * ref.ct and calling set_codetag_empty(). We accept this - * race as it's unlikely and the extra complexity of atomic - * cmpxchg is not worth it for this debug-only code path. - */ - if (ref.ct) { + for (pool = current_pfn_pool; pool; pool = next) { + int nr_pfns = atomic_read(&pool->count); + + for (i = 0; i < nr_pfns; i++) { + unsigned long pfn = pool->pfns[i]; + + if (pfn_valid(pfn)) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag + * with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); put_page_tag_ref(handle); - continue; } - - set_codetag_empty(&ref); - update_page_tag_ref(handle, &ref); - put_page_tag_ref(handle); } } + next = pool->next; + page = virt_to_page(pool); + clear_page_tag_ref(page); + __free_page(page); } } #else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ diff --git a/mm/cma.c b/mm/cma.c index a13ce4999b39..31073738f2ac 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -126,7 +126,6 @@ bool cma_validate_zones(struct cma *cma) * to be in the same zone. Simplify by forcing the entire * CMA resv range to be in the same zone. */ - WARN_ON_ONCE(!pfn_valid(base_pfn)); if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) { set_bit(CMA_ZONES_INVALID, &cma->flags); return false; @@ -165,6 +164,8 @@ static void __init cma_activate_area(struct cma *cma) bitmap_set(cmr->bitmap, 0, bitmap_count); } + WARN_ON_ONCE(!pfn_valid(cmr->base_pfn)); + for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count; pfn += pageblock_nr_pages) init_cma_reserved_pageblock(pfn_to_page(pfn)); diff --git a/mm/compaction.c b/mm/compaction.c index b776f35ad020..f08765ade014 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1875,15 +1875,14 @@ static void compaction_free(struct folio *dst, unsigned long data) int order = folio_order(dst); struct page *page = &dst->page; - if (folio_put_testzero(dst)) { - free_pages_prepare(page, order); + if (folio_put_testzero(dst) && free_pages_prepare(page, order)) { list_add(&dst->lru, &cc->freepages[order]); cc->nr_freepages += 1 << order; } cc->nr_migratepages += 1 << order; /* - * someone else has referenced the page, we cannot take it back to our - * free list. + * someone else has referenced the page or free_pages_prepare() fails, + * we cannot take it back to our free list. */ } diff --git a/mm/damon/core.c b/mm/damon/core.c index 265d51ade25b..b2fc15a3804f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Data Access Monitor - * - * Author: SeongJae Park */ #define pr_fmt(fmt) "damon: " fmt @@ -210,6 +208,71 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx) return NULL; } +/* + * damon_mvsum() - Returns pseudo moving sum value for a time window. + * @current_nr: The value of the current aggregation window. + * @last_nr: The value of the last aggregation window. + * @left_window_bp: Left time of the current aggregation window. + * + * This function calculates a pseudo moving sum value of a counter that is + * aggregated for each time window. @current_nr is the value of the counter + * that aggregated so far (maybe not yet complete), from the beginning of the + * current aggregation time window. @last_nr is the value of the counter that + * has completely aggregated in the last aggregation time window. + * @left_window_bp represents how much time is left for the current aggregation + * time window in bp (1/10,000). For example, the aggregation time window is + * for every 10 seconds and 7 seconds has passed since the beginning of the + * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000). + * + * The logic assumes the aggregation in the last phase was made in a single + * speed. Based on the assumption, the value from the last window that needs + * to be added to the current value is calculated as a portion of the last + * value based on the remaining time window. + */ +static unsigned long damon_mvsum(unsigned long current_nr, + unsigned long last_nr, unsigned long left_window_bp) +{ + return current_nr + mult_frac(last_nr, left_window_bp, 10000); +} + +/** + * damon_nr_accesses_mvsum() - Returns moving sum access frequency score. + * @r: Region to get the access frequency of. + * @ctx: DAMON context of @r. + * + * This function returns for how many sampling iterations in the last + * aggregation interval (&damon_attrs->aggr_interval) the region was found to + * be accessed. Hence the value can be interpreted as the relative access + * frequency score of the region (@r). The value is calculated as a pseudo + * moving sum, and hence it is not an exact value but just a best-effort + * reasonable estimation. + * + * Return: the pseudo moving sum access frequency score. + */ +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx) +{ + unsigned long sample_interval, aggr_interval; + unsigned long window_len, left_window, left_window_bp; + + sample_interval = ctx->attrs.sample_interval ? : 1; + aggr_interval = ctx->attrs.aggr_interval ? : 1; + window_len = aggr_interval / sample_interval; + if (time_after_eq(ctx->passed_sample_intervals, + ctx->next_aggregation_sis)) + left_window = 0; + else + left_window = ctx->next_aggregation_sis - + ctx->passed_sample_intervals; + left_window_bp = mult_frac(left_window, 10000, window_len); + + if (left_window_bp == 10000) + return r->last_nr_accesses; + + return damon_mvsum(r->nr_accesses, r->last_nr_accesses, + left_window_bp); +} + #ifdef CONFIG_DAMON_DEBUG_SANITY static void damon_verify_new_region(unsigned long start, unsigned long end) { @@ -239,7 +302,6 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->ar.start = start; region->ar.end = end; region->nr_accesses = 0; - region->nr_accesses_bp = 0; for (i = 0; i < DAMON_MAX_PROBES; i++) region->probe_hits[i] = 0; INIT_LIST_HEAD(®ion->list); @@ -356,8 +418,21 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, { struct damon_region *r, *next; unsigned int i; + unsigned long last_end; int err; + for (i = 0; i < nr_ranges; i++) { + unsigned long start, end; + + start = ALIGN_DOWN(ranges[i].start, min_region_sz); + end = ALIGN(ranges[i].end, min_region_sz); + if (start >= end) + return -EINVAL; + if (i > 0 && last_end > start) + return -EINVAL; + last_end = end; + } + /* Remove regions which are not in the new ranges */ damon_for_each_region_safe(r, next, t) { for (i = 0; i < nr_ranges; i++) { @@ -811,22 +886,19 @@ static void damon_update_monitoring_result(struct damon_region *r, struct damon_attrs *old_attrs, struct damon_attrs *new_attrs, bool aggregating) { - if (!aggregating) { + r->last_nr_accesses = damon_nr_accesses_for_new_attrs( + r->last_nr_accesses, old_attrs, new_attrs); + if (!aggregating) r->nr_accesses = damon_nr_accesses_for_new_attrs( r->nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->nr_accesses * 10000; - } else { + else /* * if this is called in the middle of the aggregation, reset * the aggregations we made so far for this aggregation * interval. In other words, make the status like * kdamond_reset_aggregated() is called. */ - r->last_nr_accesses = damon_nr_accesses_for_new_attrs( - r->last_nr_accesses, old_attrs, new_attrs); - r->nr_accesses_bp = r->last_nr_accesses * 10000; r->nr_accesses = 0; - } r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs); } @@ -1349,23 +1421,35 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx) static int damon_commit_target_regions(struct damon_target *dst, struct damon_target *src, unsigned long src_min_region_sz) { - struct damon_region *src_region; + struct damon_region *src_region, *prev = NULL; struct damon_addr_range *ranges; int i = 0, err; - damon_for_each_region(src_region, src) - i++; + damon_for_each_region(src_region, src) { + if (!prev || prev->ar.end != src_region->ar.start) + i++; + prev = src_region; + } if (!i) return 0; - ranges = kmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); + ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN); if (!ranges) return -ENOMEM; + prev = NULL; i = 0; - damon_for_each_region(src_region, src) - ranges[i++] = src_region->ar; + damon_for_each_region(src_region, src) { + if (!prev) { + ranges[i].start = src_region->ar.start; + } else if (prev->ar.end != src_region->ar.start) { + ranges[i++].end = prev->ar.end; + ranges[i].start = src_region->ar.start; + } + prev = src_region; + } + ranges[i++].end = damon_last_region(src)->ar.end; err = damon_set_regions(dst, ranges, i, src_min_region_sz); - kfree(ranges); + kvfree(ranges); return err; } @@ -1387,10 +1471,36 @@ static int damon_commit_target( return 0; } +/* + * damon_revert_target_commits() - revert unsuccessful target commits. + * @dst: Commit destination context + * @failed: Commit failed destination target + * @src: Commit source context + * + * Revert target states that changed by damon_commit_target(), and cannot be + * cleaned up by the destination context's ops.cleanup_target(). + */ +static void damon_revert_target_commits(struct damon_ctx *dst, + struct damon_target *failed, struct damon_ctx *src) +{ + struct damon_target *target; + + if (!damon_target_has_pid(src)) + return; + if (dst->ops.cleanup_target) + return; + damon_for_each_target(target, dst) { + if (target == failed) + return; + put_pid(target->pid); + } +} + static int damon_commit_targets( struct damon_ctx *dst, struct damon_ctx *src) { struct damon_target *dst_target, *next, *src_target, *new_target; + struct damon_target *failed; int i = 0, j = 0, err; damon_for_each_target_safe(dst_target, next, dst) { @@ -1404,8 +1514,10 @@ static int damon_commit_targets( dst_target, damon_target_has_pid(dst), src_target, damon_target_has_pid(src), src->min_region_sz); - if (err) - return err; + if (err) { + failed = dst_target; + goto out; + } } else { struct damos *s; @@ -1419,25 +1531,34 @@ static int damon_commit_targets( } } + failed = NULL; damon_for_each_target_safe(src_target, next, src) { if (j++ < i) continue; /* target to remove has no matching dst */ - if (src_target->obsolete) - return -EINVAL; + if (src_target->obsolete) { + err = -EINVAL; + goto out; + } new_target = damon_new_target(); - if (!new_target) - return -ENOMEM; + if (!new_target) { + err = -ENOMEM; + goto out; + } err = damon_commit_target(new_target, false, src_target, damon_target_has_pid(src), src->min_region_sz); if (err) { damon_destroy_target(new_target, NULL); - return err; + goto out; } damon_add_target(dst, new_target); } return 0; + +out: + damon_revert_target_commits(dst, failed, src); + return err; } static void damon_commit_filter(struct damon_filter *dst, @@ -1571,8 +1692,10 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src) */ if (!damon_attrs_equals(&dst->attrs, &src->attrs)) { err = damon_set_attrs(dst, &src->attrs); - if (err) + if (err) { + damon_revert_target_commits(dst, NULL, src); return err; + } } dst->pause = src->pause; dst->ops = src->ops; @@ -1898,36 +2021,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) return 0; } -/* - * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing - * the problem being propagated. - */ -static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r) -{ - if (r->nr_accesses_bp == r->nr_accesses * 10000) - return; - WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n", - r->nr_accesses_bp, r->nr_accesses); - r->nr_accesses_bp = r->nr_accesses * 10000; -} - -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ - WARN_ONCE(r->nr_accesses_bp != r->last_nr_accesses * 10000, - "nr_accesses_bp %u last_nr_accesses %u sis %lu %lu\n", - r->nr_accesses_bp, r->last_nr_accesses, - c->passed_sample_intervals, c->next_aggregation_sis); -} -#else -static void damon_verify_reset_aggregated(struct damon_region *r, - struct damon_ctx *c) -{ -} -#endif - - /* * Reset the aggregated monitoring results ('nr_accesses' of each region). */ @@ -1952,12 +2045,10 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) trace_damon_aggregated(ti, r, damon_nr_regions(t)); trace_damon_region_aggregated(ti, r, damon_nr_regions(t), nr_probes); - damon_warn_fix_nr_accesses_corruption(r); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; for (i = 0; i < DAMON_MAX_PROBES; i++) r->probe_hits[i] = 0; - damon_verify_reset_aggregated(r, c); } ti++; } @@ -2002,7 +2093,7 @@ static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c) return adaptation_bp; } -static void kdamond_tune_intervals(struct damon_ctx *c) +static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c) { unsigned long adaptation_bp; struct damon_attrs new_attrs; @@ -2024,10 +2115,11 @@ static void kdamond_tune_intervals(struct damon_ctx *c) damon_set_attrs(c, &new_attrs); } -static bool __damos_valid_target(struct damon_region *r, struct damos *s) +static bool __damos_valid_target(struct damon_region *r, struct damos *s, + struct damon_ctx *c) { unsigned long sz; - unsigned int nr_accesses = r->nr_accesses_bp / 10000; + unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c); sz = damon_sz_region(r); return s->pattern.min_sz_region <= sz && @@ -2053,7 +2145,7 @@ static bool damos_quota_is_set(struct damos_quota *quota) static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r, struct damos *s) { - bool ret = __damos_valid_target(r, s); + bool ret = __damos_valid_target(r, s, c); if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score) return ret; @@ -2337,7 +2429,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, struct damos *siter; /* schemes iterator */ unsigned int sidx = 0; struct damon_target *titer; /* targets iterator */ - unsigned int tidx = 0; + unsigned int tidx = 0, nr_accesses = 0; bool do_trace = false; /* get indices for trace_damos_before_apply() */ @@ -2352,6 +2444,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, break; tidx++; } + nr_accesses = damon_nr_accesses_mvsum(r, c); do_trace = true; } @@ -2367,7 +2460,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, if (damos_core_filter_out(c, t, r, s)) return; ktime_get_coarse_ts64(&begin); - trace_damos_before_apply(cidx, sidx, tidx, r, + trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses, damon_nr_regions(t), do_trace); sz_applied = c->ops.apply_scheme(c, t, r, s, &sz_ops_filter_passed); @@ -2639,7 +2732,7 @@ static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c, damon_for_each_region(r, t) { phys_addr_t addr, end_addr; - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; /* Convert from core address units to physical bytes */ @@ -2928,7 +3021,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) (DAMOS_MAX_SCORE + 1)); damon_for_each_target(t, c) { damon_for_each_region(r, t) { - if (!__damos_valid_target(r, s)) + if (!__damos_valid_target(r, s, c)) continue; if (damos_core_filter_out(c, t, r, s)) continue; @@ -3032,7 +3125,6 @@ static void damon_merge_two_regions(struct damon_target *t, l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); - l->nr_accesses_bp = l->nr_accesses * 10000; l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end = r->ar.end; /* todo: do this for only installed probes */ @@ -3043,20 +3135,6 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } -#ifdef CONFIG_DAMON_DEBUG_SANITY -static void damon_verify_merge_regions_of(struct damon_region *r) -{ - WARN_ONCE(r->nr_accesses != r->nr_accesses_bp / 10000, - "nr_accesses (%u) != nr_accesses_bp (%u)\n", - r->nr_accesses, r->nr_accesses_bp); -} -#else -static void damon_verify_merge_regions_of(struct damon_region *r) -{ -} -#endif - - /* * Merge adjacent regions having similar access frequencies * @@ -3070,7 +3148,6 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { - damon_verify_merge_regions_of(r); if (abs(r->nr_accesses - r->last_nr_accesses) > thres) r->age = 0; else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) @@ -3159,7 +3236,6 @@ static void damon_split_region_at(struct damon_target *t, new->age = r->age; new->last_nr_accesses = r->last_nr_accesses; - new->nr_accesses_bp = r->nr_accesses_bp; new->nr_accesses = r->nr_accesses; /* todo: do this for only installed probes */ memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits)); @@ -3197,6 +3273,37 @@ static void damon_split_regions_of(struct damon_ctx *ctx, } } +/* Split one in every @split_step regions into two, from a rotating offset */ +static void damon_split_some_regions(struct damon_ctx *ctx, + unsigned long split_step) +{ + static unsigned long rotation; + struct damon_target *t; + struct damon_region *r, *next; + unsigned long offset = rotation++ % split_step; + unsigned long idx = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) { + unsigned long sz_region, sz_sub; + + if (idx++ % split_step != offset) + continue; + sz_region = damon_sz_region(r); + if (sz_region < 2 * ctx->min_region_sz) + continue; + + sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) * + sz_region / 10, ctx->min_region_sz); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(t, r, sz_sub); + } + } +} + /* * Split every target region into randomly-sized small regions * @@ -3210,25 +3317,33 @@ static void damon_split_regions_of(struct damon_ctx *ctx, static void kdamond_split_regions(struct damon_ctx *ctx) { struct damon_target *t; - unsigned int nr_regions = 0; - static unsigned int last_nr_regions; + unsigned long nr_regions = 0; + unsigned long max_nr_regions = ctx->attrs.max_nr_regions; + static unsigned long last_nr_regions; int nr_subregions = 2; damon_for_each_target(t, ctx) nr_regions += damon_nr_regions(t); - if (nr_regions > ctx->attrs.max_nr_regions / 2) - return; + if (nr_regions >= max_nr_regions) + goto done; + + if (nr_regions > max_nr_regions / 2) { + damon_split_some_regions(ctx, + max_nr_regions / (max_nr_regions - nr_regions)); + goto done; + } /* Maybe the middle of the region has different access frequency */ if (last_nr_regions == nr_regions && - nr_regions < ctx->attrs.max_nr_regions / 3) + nr_regions < max_nr_regions / 3) nr_subregions = 3; damon_for_each_target(t, ctx) damon_split_regions_of(ctx, t, nr_subregions, ctx->min_region_sz); +done: last_nr_regions = nr_regions; } @@ -3534,8 +3649,7 @@ static int kdamond_fn(void *data) * aggregation, and make aggregation * information reset for all regions. Then, * following kdamond_reset_aggregated() call - * will make the region information invalid, - * particularly for ->nr_accesses_bp. + * will make the region information invalid. * * Reset ->next_aggregation_sis to avoid that. * It will anyway correctly updated after this @@ -3674,72 +3788,18 @@ int damon_set_region_system_rams_default(struct damon_target *t, return damon_set_regions(t, &addr_range, 1, min_region_sz); } -/* - * damon_moving_sum() - Calculate an inferred moving sum value. - * @mvsum: Inferred sum of the last @len_window values. - * @nomvsum: Non-moving sum of the last discrete @len_window window values. - * @len_window: The number of last values to take care of. - * @new_value: New value that will be added to the pseudo moving sum. - * - * Moving sum (moving average * window size) is good for handling noise, but - * the cost of keeping past values can be high for arbitrary window size. This - * function implements a lightweight pseudo moving sum function that doesn't - * keep the past window values. - * - * It simply assumes there was no noise in the past, and get the no-noise - * assumed past value to drop from @nomvsum and @len_window. @nomvsum is a - * non-moving sum of the last window. For example, if @len_window is 10 and we - * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25 - * values. Hence, this function simply drops @nomvsum / @len_window from - * given @mvsum and add @new_value. - * - * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for - * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20. For - * calculating next moving sum with a new value, we should drop 0 from 50 and - * add the new value. However, this function assumes it got value 5 for each - * of the last ten times. Based on the assumption, when the next value is - * measured, it drops the assumed past value, 5 from the current sum, and add - * the new value to get the updated pseduo-moving average. - * - * This means the value could have errors, but the errors will be disappeared - * for every @len_window aligned calls. For example, if @len_window is 10, the - * pseudo moving sum with 11th value to 19th value would have an error. But - * the sum with 20th value will not have the error. - * - * Return: Pseudo-moving average after getting the @new_value. - */ -static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum, - unsigned int len_window, unsigned int new_value) -{ - return mvsum - nomvsum / len_window + new_value; -} - /** * damon_update_region_access_rate() - Update the access rate of a region. * @r: The DAMON region to update for its access check result. * @accessed: Whether the region has accessed during last sampling interval. - * @attrs: The damon_attrs of the DAMON context. * * Update the access rate of a region with the region's last sampling interval * access check result. * * Usually this will be called by &damon_operations->check_accesses callback. */ -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs) +void damon_update_region_access_rate(struct damon_region *r, bool accessed) { - unsigned int len_window = 1; - - /* - * sample_interval can be zero, but cannot be larger than - * aggr_interval, owing to validation of damon_set_attrs(). - */ - if (attrs->sample_interval) - len_window = damon_max_nr_accesses(attrs); - r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp, - r->last_nr_accesses * 10000, len_window, - accessed ? 10000 : 0); - if (accessed) r->nr_accesses++; } diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 8298c6001fd0..32f41491b726 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON-based LRU-lists Sorting - * - * Author: SeongJae Park */ #define pr_fmt(fmt) "damon-lru-sort: " fmt diff --git a/mm/damon/modules-common.c b/mm/damon/modules-common.c index 86d58f8c4f63..f87fa46a95a0 100644 --- a/mm/damon/modules-common.c +++ b/mm/damon/modules-common.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Common Code for DAMON Modules - * - * Author: SeongJae Park */ #include diff --git a/mm/damon/modules-common.h b/mm/damon/modules-common.h index f103ad556368..6fd45490e45b 100644 --- a/mm/damon/modules-common.h +++ b/mm/damon/modules-common.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Common Code for DAMON Modules - * - * Author: SeongJae Park */ #include diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 5c93ef2bb8a9..6bdd1cfd3863 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Common Code for Data Access Monitoring - * - * Author: SeongJae Park */ #include @@ -143,6 +141,7 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, * Transform it to fit in [0, DAMOS_MAX_SCORE] */ hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + hotness = max(min(hotness, DAMOS_MAX_SCORE), 0); return hotness; } diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index 5efa5b5970de..38d295488fa1 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Common Code for Data Access Monitoring - * - * Author: SeongJae Park */ #include diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index d0598f5f2688..617246498173 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON Code for The Physical Address Space - * - * Author: SeongJae Park */ #define pr_fmt(fmt) "damon-pa: " fmt @@ -82,7 +80,7 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) } static void __damon_pa_check_access(struct damon_region *r, - struct damon_attrs *attrs, unsigned long addr_unit) + unsigned long addr_unit) { static phys_addr_t last_addr; static unsigned long last_folio_sz = PAGE_SIZE; @@ -93,12 +91,12 @@ static void __damon_pa_check_access(struct damon_region *r, /* If the region is in the last checked page, reuse the result */ if (ALIGN_DOWN(last_addr, last_folio_sz) == ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); return; } last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); last_addr = sampling_addr; } @@ -111,8 +109,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access( - r, &ctx->attrs, ctx->addr_unit); + __damon_pa_check_access(r, ctx->addr_unit); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index ce4499cf4b8b..11b70d0a9a6f 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON-based page reclamation - * - * Author: SeongJae Park */ #define pr_fmt(fmt) "damon-reclaim: " fmt diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 0e14f5bb8f75..b05b68f73e10 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -138,7 +138,7 @@ static int damon_stat_damon_call_fn(void *data) /* avoid unnecessarily frequent stat update */ if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies + - msecs_to_jiffies(5 * MSEC_PER_SEC))) + secs_to_jiffies(5))) return 0; damon_stat_last_refresh_jiffies = jiffies; diff --git a/mm/damon/sysfs-common.c b/mm/damon/sysfs-common.c index bdc6ae2639e4..c59d7bf7a73a 100644 --- a/mm/damon/sysfs-common.c +++ b/mm/damon/sysfs-common.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * Common Code for DAMON Sysfs Interface - * - * Author: SeongJae Park */ #include diff --git a/mm/damon/sysfs-common.h b/mm/damon/sysfs-common.h index 3079306966a9..733764716e8d 100644 --- a/mm/damon/sysfs-common.h +++ b/mm/damon/sysfs-common.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Common Code for DAMON Sysfs Interface - * - * Author: SeongJae Park */ #include diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 329cfd0bbe9f..dbf2b0515d58 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON sysfs Interface - * - * Copyright (c) 2022 SeongJae Park */ #include @@ -159,7 +157,7 @@ struct damon_sysfs_scheme_region { }; static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( - struct damon_region *region) + struct damon_region *region, struct damon_ctx *ctx) { struct damon_sysfs_scheme_region *sysfs_region = kmalloc_obj(*sysfs_region); @@ -167,7 +165,7 @@ static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc( return NULL; sysfs_region->kobj = (struct kobject){}; sysfs_region->ar = region->ar; - sysfs_region->nr_accesses = region->nr_accesses_bp / 10000; + sysfs_region->nr_accesses = damon_nr_accesses_mvsum(region, ctx); sysfs_region->age = region->age; sysfs_region->probes = NULL; INIT_LIST_HEAD(&sysfs_region->list); @@ -334,6 +332,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { damos_sysfs_region_rm_dirs(r); list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -913,8 +912,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1462,8 +1463,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -1993,22 +1996,19 @@ static int damon_sysfs_access_pattern_add_dirs( err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->sz, "sz"); if (err) - goto put_sz_out; + return err; err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->nr_accesses, "nr_accesses"); if (err) - goto put_nr_accesses_sz_out; + goto put_sz_out; err = damon_sysfs_access_pattern_add_range_dir(access_pattern, &access_pattern->age, "age"); if (err) - goto put_age_nr_accesses_sz_out; + goto put_nr_accesses_sz_out; return 0; -put_age_nr_accesses_sz_out: - kobject_put(&access_pattern->age->kobj); - access_pattern->age = NULL; put_nr_accesses_sz_out: kobject_put(&access_pattern->nr_accesses->kobj); access_pattern->nr_accesses = NULL; @@ -2143,8 +2143,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2516,12 +2518,12 @@ static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme) goto put_filters_watermarks_quotas_access_pattern_out; err = damon_sysfs_scheme_set_tried_regions(scheme); if (err) - goto put_tried_regions_out; + goto put_stats_out; return 0; -put_tried_regions_out: - kobject_put(&scheme->tried_regions->kobj); - scheme->tried_regions = NULL; +put_stats_out: + kobject_put(&scheme->stats->kobj); + scheme->stats = NULL; put_filters_watermarks_quotas_access_pattern_out: kobject_put(&scheme->ops_filters->kobj); scheme->ops_filters = NULL; @@ -2686,6 +2688,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2727,13 +2730,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); @@ -3117,7 +3122,7 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, if (total_bytes_only) return; - region = damon_sysfs_scheme_region_alloc(r); + region = damon_sysfs_scheme_region_alloc(r, ctx); if (!region) return; region->sz_filter_passed = sz_filter_passed; @@ -3127,12 +3132,14 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes, sysfs_regions->nr_regions)) goto out; if (damos_sysfs_region_add_dirs(region, ctx, r)) - goto out; + goto del_out; list_add_tail(®ion->list, &sysfs_regions->regions_list); sysfs_regions->nr_regions++; return; +del_out: + kobject_del(®ion->kobj); out: kobject_put(®ion->kobj); } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 2e95e3bac774..6710b6d019bf 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON sysfs Interface - * - * Copyright (c) 2022 SeongJae Park */ #include @@ -107,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -333,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -371,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -957,8 +960,10 @@ static void damon_sysfs_filters_rm_dirs(struct damon_sysfs_filters *filters) struct damon_sysfs_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1068,7 +1073,7 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void) return kzalloc_obj(struct damon_sysfs_probe); } -static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) +static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe) { struct damon_sysfs_filters *filters; int err; @@ -1076,22 +1081,22 @@ static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr) filters = damon_sysfs_filters_alloc(); if (!filters) return -ENOMEM; - attr->filters = filters; + probe->filters = filters; err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype, - &attr->kobj, "filters"); + &probe->kobj, "filters"); if (err) { kobject_put(&filters->kobj); - attr->filters = NULL; + probe->filters = NULL; } return err; } -static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *attr) +static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe) { - if (attr->filters) { - damon_sysfs_filters_rm_dirs(attr->filters); - kobject_put(&attr->filters->kobj); + if (probe->filters) { + damon_sysfs_filters_rm_dirs(probe->filters); + kobject_put(&probe->filters->kobj); } } @@ -1134,6 +1139,7 @@ static void damon_sysfs_probes_rm_dirs( for (i = 0; i < probes->nr; i++) { damon_sysfs_probe_rm_dirs(probes_arr[i]); + kobject_del(&probes_arr[i]->kobj); kobject_put(&probes_arr[i]->kobj); } probes->nr = 0; @@ -1175,6 +1181,7 @@ static int damon_sysfs_probes_add_dirs( err = damon_sysfs_probe_add_dirs(probe); if (err) { + kobject_del(&probe->kobj); kobject_put(&probe->kobj); damon_sysfs_probes_rm_dirs(probes); return err; @@ -1642,6 +1649,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1680,13 +1688,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -1899,47 +1909,65 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, return damon_set_attrs(ctx, &attrs); } -static int damon_sysfs_set_probes(struct damon_ctx *ctx, - struct damon_sysfs_probes *sys_probes) +static int damon_sysfs_set_filters(struct damon_probe *probe, + struct damon_sysfs_filters *sys_filters) { int i; - for (i = 0; i < sys_probes->nr; i++) { - struct damon_sysfs_filters *sys_filters = - sys_probes->probes_arr[i]->filters; - struct damon_probe *c; - int j; + for (i = 0; i < sys_filters->nr; i++) { + struct damon_sysfs_filter *sys_filter = + sys_filters->filters_arr[i]; + struct damon_filter *filter; - if (!sys_filters) - continue; - c = damon_new_probe(); - if (!c) + filter = damon_new_filter(sys_filter->type, + sys_filter->matching, + sys_filter->allow); + if (!filter) return -ENOMEM; - damon_add_probe(ctx, c); - - for (j = 0; j < sys_filters->nr; j++) { - struct damon_sysfs_filter *sys_filter = - sys_filters->filters_arr[j]; - struct damon_filter *filter; - - filter = damon_new_filter(sys_filter->type, - sys_filter->matching, - sys_filter->allow); - if (!filter) - return -ENOMEM; - if (filter->type == DAMON_FILTER_TYPE_MEMCG) { - int err; - - err = damon_sysfs_memcg_path_to_id( - sys_filter->path, - &filter->memcg_id); - if (err) { - damon_destroy_filter(filter); - return err; - } + if (filter->type == DAMON_FILTER_TYPE_MEMCG) { + int err; + + err = damon_sysfs_memcg_path_to_id( + sys_filter->path, + &filter->memcg_id); + if (err) { + damon_destroy_filter(filter); + return err; } - damon_add_filter(c, filter); } + damon_add_filter(probe, filter); + } + return 0; +} + +static int damon_sysfs_set_probe(struct damon_probe *probe, + struct damon_sysfs_probe *sys_probe) +{ + struct damon_sysfs_filters *sys_filters; + + sys_filters = sys_probe->filters; + if (!sys_filters) + return 0; + return damon_sysfs_set_filters(probe, sys_filters); +} + +static int damon_sysfs_set_probes(struct damon_ctx *ctx, + struct damon_sysfs_probes *sys_probes) +{ + int i, err; + + for (i = 0; i < sys_probes->nr; i++) { + struct damon_sysfs_probe *sys_probe; + struct damon_probe *p; + + p = damon_new_probe(); + if (!p) + return -ENOMEM; + damon_add_probe(ctx, p); + sys_probe = sys_probes->probes_arr[i]; + err = damon_sysfs_set_probe(p, sys_probe); + if (err) + return err; } return 0; } @@ -2501,6 +2529,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -2555,13 +2584,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 1cfb8c176b87..0124f83b39b8 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -1,10 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Data Access Monitor Unit Tests - * - * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. - * - * Author: SeongJae Park */ #ifdef CONFIG_DAMON_KUNIT_TEST @@ -118,7 +114,6 @@ static void damon_test_aggregate(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = accesses[it][ir]; - r->nr_accesses_bp = accesses[it][ir] * 10000; damon_add_region(r, t); } it++; @@ -155,7 +150,6 @@ static void damon_test_split_at(struct kunit *test) damon_free_target(t); kunit_skip(test, "region alloc fail"); } - r->nr_accesses_bp = 420000; r->nr_accesses = 42; r->last_nr_accesses = 15; r->age = 10; @@ -168,7 +162,6 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul); KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp); KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); KUNIT_EXPECT_EQ(test, r->age, r_new->age); @@ -191,7 +184,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = 10; - r->nr_accesses_bp = 100000; r->age = 9; damon_add_region(r, t); r2 = damon_new_region(100, 300); @@ -200,7 +192,6 @@ static void damon_test_merge_two(struct kunit *test) kunit_skip(test, "second region alloc fail"); } r2->nr_accesses = 20; - r2->nr_accesses_bp = 200000; r2->age = 21; damon_add_region(r2, t); @@ -208,7 +199,6 @@ static void damon_test_merge_two(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); - KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, 160000u); KUNIT_EXPECT_EQ(test, r->age, 17u); i = 0; @@ -256,7 +246,6 @@ static void damon_test_merge_regions_of(struct kunit *test) kunit_skip(test, "region alloc fail"); } r->nr_accesses = nrs[i]; - r->nr_accesses_bp = nrs[i] * 10000; damon_add_region(r, t); } @@ -339,6 +328,69 @@ static void damon_test_split_regions_of(struct kunit *test) damon_destroy_ctx(c); } +/* + * When the total region count is already above max_nr_regions / 2, + * kdamond_split_regions() must keep refining the resolution by splitting a + * fraction of the regions (making progress), without exceeding + * max_nr_regions. + */ +static void damon_test_split_above_half_progresses(struct kunit *test) +{ + struct damon_ctx *c; + struct damon_target *t; + struct damon_region *r; + unsigned long start; + unsigned int nr_before, nr_after, i; + const unsigned int nr_init = 760; + const unsigned long region_sz = 100; + + c = damon_new_ctx(); + if (!c) + kunit_skip(test, "ctx alloc fail"); + + /* Keep the split arithmetic independent of the page size */ + c->min_region_sz = 1; + c->attrs.min_nr_regions = 10; + c->attrs.max_nr_regions = 1500; + + t = damon_new_target(); + if (!t) { + damon_destroy_ctx(c); + kunit_skip(test, "target alloc fail"); + } + + for (i = 0; i < nr_init; i++) { + start = i * region_sz; + r = damon_new_region(start, start + region_sz); + if (!r) { + damon_free_target(t); + damon_destroy_ctx(c); + kunit_skip(test, "region alloc fail"); + } + r->nr_accesses = (i & 1) ? 0 : 100; + r->age = 5; + damon_add_region(r, t); + } + + damon_add_target(c, t); + + nr_before = damon_nr_regions(t); + /* Above max_nr_regions / 2, so the blanket-split path is skipped */ + KUNIT_EXPECT_GT(test, (unsigned long)nr_before, + c->attrs.max_nr_regions / 2); + + kdamond_split_regions(c); + + nr_after = damon_nr_regions(t); + /* Still made progress ... */ + KUNIT_EXPECT_GT(test, nr_after, nr_before); + /* ... but did not overshoot the configured maximum */ + KUNIT_EXPECT_LE(test, (unsigned long)nr_after, + c->attrs.max_nr_regions); + + damon_destroy_ctx(c); +} + static void damon_test_ops_registration(struct kunit *test) { struct damon_ctx *c = damon_new_ctx(); @@ -556,7 +608,6 @@ static void damon_test_update_monitoring_result(struct kunit *test) kunit_skip(test, "region alloc fail"); r->nr_accesses = 15; - r->nr_accesses_bp = 150000; r->age = 20; new_attrs = (struct damon_attrs){ @@ -608,18 +659,27 @@ static void damon_test_set_attrs(struct kunit *test) damon_destroy_ctx(c); } -static void damon_test_moving_sum(struct kunit *test) +static void damon_test_mvsum(struct kunit *test) { - unsigned int mvsum = 50000, nomvsum = 50000, len_window = 10; - unsigned int new_values[] = {10000, 0, 10000, 0, 0, 0, 10000, 0, 0, 0}; - unsigned int expects[] = {55000, 50000, 55000, 50000, 45000, 40000, - 45000, 40000, 35000, 30000}; + unsigned long input_expects[] = { + /* current value, last value, remaining window (bp) */ + 0, 49, 10000, 49, /* 0 + 49 * 1 */ + 3, 10, 7000, 10, /* 3 + 10 * 0.7 */ + 3, 10, 5000, 8, /* 3 + 10 * 0.5 */ + 32, 100, 1000, 42, /* 32 + 100 * 0.1 */ + 42, 49, 0, 42, /* 42 + 49 * 0 */ + }; + int i; - for (i = 0; i < ARRAY_SIZE(new_values); i++) { - mvsum = damon_moving_sum(mvsum, nomvsum, len_window, - new_values[i]); - KUNIT_EXPECT_EQ(test, mvsum, expects[i]); + for (i = 0; i < ARRAY_SIZE(input_expects); i += 4) { + unsigned long current_nr = input_expects[i]; + unsigned long last_nr = input_expects[i + 1]; + unsigned long left_window_bp = input_expects[i + 2]; + unsigned long expect = input_expects[i + 3]; + + KUNIT_EXPECT_EQ(test, damon_mvsum(current_nr, last_nr, + left_window_bp), expect); } } @@ -1460,6 +1520,47 @@ static void damon_test_is_last_region(struct kunit *test) damon_free_target(t); } +/* + * Verify that damos_walk() rejects new requests when + * walk_control_obsolete is set. + * + * This tests the invariant introduced by: + * commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race") + */ +static void damon_test_walk_control_obsolete(struct kunit *test) +{ + struct damon_ctx *ctx; + struct damos_walk_control control = {}; + int ret; + + ctx = damon_new_ctx(); + if (!ctx) + kunit_skip(test, "ctx alloc fail"); + + /* Simulate shutdown phase */ + ctx->walk_control_obsolete = true; + + ret = damos_walk(ctx, &control); + + KUNIT_EXPECT_EQ(test, ret, -ECANCELED); + + damon_destroy_ctx(ctx); +} + +static void damon_test_rand(struct kunit *test) +{ + struct damon_ctx ctx; + int i; + + prandom_seed_state(&ctx.rnd_state, get_random_u64()); + for (i = 0; i < 10000; i++) { + unsigned long rnd = damon_rand(&ctx, 0, 10); + + KUNIT_EXPECT_GE(test, rnd, 0); + KUNIT_EXPECT_LE(test, rnd, 9); + } +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -1468,12 +1569,13 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_merge_two), KUNIT_CASE(damon_test_merge_regions_of), KUNIT_CASE(damon_test_split_regions_of), + KUNIT_CASE(damon_test_split_above_half_progresses), KUNIT_CASE(damon_test_ops_registration), KUNIT_CASE(damon_test_set_regions), KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp), KUNIT_CASE(damon_test_update_monitoring_result), KUNIT_CASE(damon_test_set_attrs), - KUNIT_CASE(damon_test_moving_sum), + KUNIT_CASE(damon_test_mvsum), KUNIT_CASE(damos_test_new_filter), KUNIT_CASE(damos_test_commit_quota_goal), KUNIT_CASE(damos_test_commit_quota_goals), @@ -1489,6 +1591,8 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_set_filters_default_reject), KUNIT_CASE(damon_test_apply_min_nr_regions), KUNIT_CASE(damon_test_is_last_region), + KUNIT_CASE(damon_test_walk_control_obsolete), + KUNIT_CASE(damon_test_rand), {}, }; diff --git a/mm/damon/tests/sysfs-kunit.h b/mm/damon/tests/sysfs-kunit.h index f9ec5e795b34..138a4b8d14e7 100644 --- a/mm/damon/tests/sysfs-kunit.h +++ b/mm/damon/tests/sysfs-kunit.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Data Access Monitor Unit Tests - * - * Author: SeongJae Park */ #ifdef CONFIG_DAMON_SYSFS_KUNIT_TEST diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 563fbc7e3f44..61f844336ffb 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -1,10 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Data Access Monitor Unit Tests - * - * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. - * - * Author: SeongJae Park */ #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index d27147603564..2058db9c01d5 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* * DAMON Code for Virtual Address Spaces - * - * Author: SeongJae Park */ #define pr_fmt(fmt) "damon-va: " fmt @@ -495,27 +493,26 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, - struct damon_attrs *attrs) + struct damon_region *r, bool same_target) { static unsigned long last_addr; static unsigned long last_folio_sz = PAGE_SIZE; static bool last_accessed; if (!mm) { - damon_update_region_access_rate(r, false, attrs); + damon_update_region_access_rate(r, false); return; } /* If the region is in the last checked page, reuse the result */ if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); return; } last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); + damon_update_region_access_rate(r, last_accessed); last_addr = r->sampling_addr; } @@ -532,8 +529,7 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) mm = damon_get_mm(t); same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r, same_target); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); same_target = true; } diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 23dc3ee09561..507fbd1ae7e5 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -751,14 +751,14 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args) pmd_t pmd; if (!pgtable_supports_soft_dirty() || - !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) + !IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF)) return; if (!has_transparent_hugepage()) return; pr_debug("Validating PMD swap soft dirty\n"); - pmd = swp_entry_to_pmd(args->leaf_entry); + pmd = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd)); WARN_ON(!pmd_is_valid_softleaf(pmd)); @@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args) WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1))); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { swp_entry_t arch_entry; @@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) return; pr_debug("Validating PMD swap\n"); - pmd1 = swp_entry_to_pmd(args->leaf_entry); + pmd1 = softleaf_to_pmd(args->leaf_entry); WARN_ON(!pmd_is_huge(pmd1)); WARN_ON(!pmd_is_valid_softleaf(pmd1)); @@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) pmd2 = __swp_entry_to_pmd(arch_entry); WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1))); } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static void __init swap_migration_tests(struct pgtable_debug_args *args) { diff --git a/mm/filemap.c b/mm/filemap.c index 7e467c81d213..b39111abdc4b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -189,7 +189,6 @@ static void filemap_unaccount_folio(struct address_space *mapping, lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); } else if (folio_test_pmd_mappable(folio)) { lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); - filemap_nr_thps_dec(mapping); } if (test_bit(AS_KERNEL_FILE, &folio->mapping->flags)) mod_node_page_state(folio_pgdat(folio), @@ -2271,10 +2270,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, unsigned long nr; struct folio *folio; - rcu_read_lock(); + if (*start > end) + return 0; - for (folio = xas_load(&xas); folio && xas.xa_index <= end; - folio = xas_next(&xas)) { + rcu_read_lock(); + for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; /* @@ -2282,11 +2282,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, * No current caller is looking for DAX entries. */ if (xa_is_value(folio)) - goto update_start; + break; /* If we landed in the middle of a THP, continue at its end. */ if (xa_is_sibling(folio)) - goto update_start; + break; if (!folio_try_get(folio)) goto retry; @@ -2294,29 +2294,27 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, if (unlikely(folio != xas_reload(&xas))) goto put_folio; - if (!folio_batch_add(fbatch, folio)) { - *start = folio_next_index(folio); - goto out; - } + if (!folio_batch_add(fbatch, folio)) + break; + xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= end) + break; continue; + put_folio: folio_put(folio); - retry: xas_reset(&xas); } + rcu_read_unlock(); -update_start: nr = folio_batch_count(fbatch); - if (nr) { folio = fbatch->folios[nr - 1]; *start = folio_next_index(folio); } -out: - rcu_read_unlock(); - return folio_batch_count(fbatch); + return nr; } EXPORT_SYMBOL(filemap_get_folios_contig); @@ -2468,11 +2466,14 @@ static void filemap_get_read_batch(struct address_space *mapping, XA_STATE(xas, &mapping->i_pages, index); struct folio *folio; + if (index > max) + return; + rcu_read_lock(); for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; - if (xas.xa_index > max || xa_is_value(folio)) + if (xa_is_value(folio)) break; if (xa_is_sibling(folio)) break; @@ -2489,6 +2490,8 @@ static void filemap_get_read_batch(struct address_space *mapping, if (folio_test_readahead(folio)) break; xas_advance(&xas, folio_next_index(folio) - 1); + if (xas.xa_index >= max) + break; continue; put_folio: folio_put(folio); @@ -3226,6 +3229,7 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; size_t seek_size; + u64 next; if (start < pos) { if (!seek_data) @@ -3234,7 +3238,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, } seek_size = seek_folio_size(&xas, folio); - pos = round_up((u64)pos + 1, seek_size); + next = round_up((u64)pos + 1, seek_size); + if (next > (u64)end) + pos = end; + else + pos = next; start = folio_seek_hole_data(&xas, mapping, folio, start, pos, seek_data); if (start < pos) @@ -3324,8 +3332,6 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) unsigned int thp_order = 0; unsigned short mmap_miss; - ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1; - /* Use the readahead code, even if readahead is disabled */ if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (vm_flags & VM_HUGEPAGE)) { /* @@ -3421,7 +3427,6 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) * mmap read-around */ ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); - ra->start = max(ra->start, vmf->vma->vm_pgoff); ra->size = ra->ra_pages; ra->async_size = ra->ra_pages / 4; ra->order = 0; @@ -3469,7 +3474,6 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, } if (folio_test_readahead(folio)) { - ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1; fpin = maybe_unlock_mmap_for_io(vmf, fpin); page_cache_async_ra(&ractl, folio, ra->ra_pages); } @@ -4709,7 +4713,7 @@ static inline bool can_do_cachestat(struct file *f) { if (f->f_mode & FMODE_WRITE) return true; - if (inode_owner_or_capable(file_mnt_idmap(f), file_inode(f))) + if (file_owner_or_capable(f)) return true; return file_permission(f, MAY_WRITE) == 0; } diff --git a/mm/gup_test.c b/mm/gup_test.c index 9dd48db897b9..eb4c9cda16ed 100644 --- a/mm/gup_test.c +++ b/mm/gup_test.c @@ -105,11 +105,15 @@ static int __gup_test_ioctl(unsigned int cmd, unsigned long i, nr_pages, addr, next; long nr; struct page **pages; + unsigned long end; int ret = 0; bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; - if (gup->size > ULONG_MAX) + if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX) + return -EINVAL; + if (check_add_overflow((unsigned long)gup->addr, + (unsigned long)gup->size, &end)) return -EINVAL; nr_pages = gup->size / PAGE_SIZE; @@ -125,13 +129,13 @@ static int __gup_test_ioctl(unsigned int cmd, i = 0; nr = gup->nr_pages_per_call; start_time = ktime_get(); - for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { + for (addr = gup->addr; addr < end; addr = next) { if (nr != gup->nr_pages_per_call) break; next = addr + nr * PAGE_SIZE; - if (next > gup->addr + gup->size) { - next = gup->addr + gup->size; + if (next > end) { + next = end; nr = (next - addr) / PAGE_SIZE; } diff --git a/mm/hmm.c b/mm/hmm.c index c72c9ddfdb2f..4f3f627d2b47 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -331,7 +331,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, return hmm_vma_fault(addr, end, required_fault, walk); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, unsigned long end, unsigned long *hmm_pfns, pmd_t pmd) @@ -391,7 +391,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start, return -EFAULT; return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static int hmm_vma_walk_pmd(pmd_t *pmdp, unsigned long start, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 64492dcb9d1e..c0892cc533a9 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -89,9 +89,6 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) { struct inode *inode; - if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) - return false; - if (!vma->vm_file) return false; @@ -100,7 +97,10 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) if (IS_ANON_FILE(inode)) return false; - return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); + if (!mapping_pmd_folio_support(vma->vm_file->f_mapping)) + return false; + + return S_ISREG(inode->i_mode); } /* If returns true, we are unable to access the VMA's folios. */ @@ -702,6 +702,8 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name) DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); +DEFINE_MTHP_STAT_ATTR(collapse_alloc, MTHP_STAT_COLLAPSE_ALLOC); +DEFINE_MTHP_STAT_ATTR(collapse_alloc_failed, MTHP_STAT_COLLAPSE_ALLOC_FAILED); DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); @@ -718,6 +720,10 @@ DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_swap_pte, MTHP_STAT_COLLAPSE_EXCEED_SWAP); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_none_pte, MTHP_STAT_COLLAPSE_EXCEED_NONE); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_shared_pte, MTHP_STAT_COLLAPSE_EXCEED_SHARED); + static struct attribute *anon_stats_attrs[] = { &anon_fault_alloc_attr.attr, @@ -734,6 +740,9 @@ static struct attribute *anon_stats_attrs[] = { &split_deferred_attr.attr, &nr_anon_attr.attr, &nr_anon_partially_mapped_attr.attr, + &collapse_exceed_swap_pte_attr.attr, + &collapse_exceed_none_pte_attr.attr, + &collapse_exceed_shared_pte_attr.attr, NULL, }; @@ -767,6 +776,8 @@ static struct attribute *any_stats_attrs[] = { #endif &split_attr.attr, &split_failed_attr.attr, + &collapse_alloc_attr.attr, + &collapse_alloc_failed_attr.attr, NULL, }; @@ -1808,7 +1819,7 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); if (pmd_swp_uffd_wp(*src_pmd)) @@ -1821,7 +1832,7 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); @@ -2546,11 +2557,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, bool uffd_wp_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write @@ -2559,12 +2571,12 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_uffd_wp(*pmd)) newpmd = pmd_swp_mkuffd_wp(newpmd); } else { @@ -2763,7 +2775,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm if (!pmd_trans_huge(src_pmdval)) { spin_unlock(src_ptl); if (pmd_is_migration_entry(src_pmdval)) { - pmd_migration_entry_wait(mm, &src_pmdval); + pmd_migration_entry_wait(mm, src_pmd); return -EAGAIN; } return -ENOENT; @@ -3576,10 +3588,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order, (1L << PG_dropbehind) | LRU_GEN_MASK | LRU_REFS_MASK)); - if (handle_hwpoison && - page_range_has_hwpoisoned(new_head, new_nr_pages)) - folio_set_has_hwpoisoned(new_folio); - new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; @@ -3601,6 +3609,14 @@ static void __split_folio_to_order(struct folio *folio, int old_order, folio_set_large_rmappable(new_folio); } + /* + * PG_has_hwpoisoned is on the 2nd page, so set it after + * the compound head is prepped. + */ + if (handle_hwpoison && + page_range_has_hwpoisoned(new_head, new_nr_pages)) + folio_set_has_hwpoisoned(new_folio); + if (folio_test_young(folio)) folio_set_young(new_folio); if (folio_test_idle(folio)) @@ -3753,33 +3769,9 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, if (!folio->mapping && !folio_test_anon(folio)) return -EBUSY; - if (folio_test_anon(folio)) { - /* order-1 is not supported for anonymous THP. */ - if (new_order == 1) - return -EINVAL; - } else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) { - if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && - !mapping_large_folio_support(folio->mapping)) { - /* - * We can always split a folio down to a single page - * (new_order == 0) uniformly. - * - * For any other scenario - * a) uniform split targeting a large folio - * (new_order > 0) - * b) any non-uniform split - * we must confirm that the file system supports large - * folios. - * - * Note that we might still have THPs in such - * mappings, which is created from khugepaged when - * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that - * case, the mapping does not actually support large - * folios properly. - */ - return -EINVAL; - } - } + /* order-1 is not supported for anonymous THP. */ + if (folio_test_anon(folio) && new_order == 1) + return -EINVAL; /* * swapcache folio could only be split to order 0 @@ -3867,7 +3859,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n } else { lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); - filemap_nr_thps_dec(mapping); } } } @@ -4877,7 +4868,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -4937,7 +4928,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } /* Set PMD. */ - pmdswp = swp_entry_to_pmd(entry); + pmdswp = softleaf_to_pmd(entry); if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); if (uffd_wp) @@ -4990,7 +4981,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) else entry = make_readable_device_private_entry( page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 571212b80835..c1b413bc487f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -58,7 +58,6 @@ struct hstate hstates[HUGE_MAX_HSTATE]; __initdata nodemask_t hugetlb_bootmem_nodes; __initdata struct list_head huge_boot_pages[MAX_NUMNODES]; -static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata; /* * Due to ordering constraints across the init code for various @@ -1317,43 +1316,27 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; } -static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, - struct vm_area_struct *vma, - unsigned long address, long gbl_chg) +static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, + struct mempolicy_interpreted *mpoli) { + nodemask_t *nodemask = mpoli->nodemask; struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask; - nodemask_t *nodemask; - int nid; - - /* - * gbl_chg==1 means the allocation requires a new page that was not - * reserved before. Making sure there's at least one free page. - */ - if (gbl_chg && !available_huge_pages(h)) - goto err; - gfp_mask = htlb_alloc_mask(h); - nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); - - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); + mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) + if (!folio) { folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); - - mpol_cond_put(mpol); + mpoli->nid, + nodemask); + } return folio; - -err: - return NULL; } #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC) @@ -1864,6 +1847,18 @@ static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h, gfp_t gfp_mask, int nid, nodemask_t *nmask) { struct folio *folio; + nodemask_t local_node_mask; + + if (!nmask) { + unsigned int cpuset_mems_cookie; + + do { + cpuset_mems_cookie = read_mems_allowed_begin(); + local_node_mask = cpuset_current_mems_allowed; + } while (read_mems_allowed_retry(cpuset_mems_cookie)); + + nmask = &local_node_mask; + } folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL); if (folio) @@ -2148,32 +2143,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas return folio; } -/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. - */ static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, + gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) { struct folio *folio = NULL; - struct mempolicy *mpol; - gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + nodemask_t *nodemask = mpoli->nodemask; - nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); - if (mpol_is_preferred_many(mpol)) { + if (mpoli->mode == MPOL_PREFERRED_MANY) { gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, + nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } - if (!folio) - folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + if (!folio) { + folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, + nodemask); + } + return folio; } @@ -2826,6 +2817,104 @@ void wait_for_freed_hugetlb_folios(void) flush_work(&free_hpage_work); } +/** + * hugetlb_alloc_folio - Allocate a hugetlb folio. + * @h: Hugetlb state control block. + * @mpoli: Interpreted memory policy to use for allocation. + * @alloc_flags: Flags controlling the allocation behavior. + * + * Allocates a hugetlb folio and handles cgroup charging and global hstate + * reservations. + * + * Return: A pointer to the allocated folio, or an ERR_PTR on failure. + * -ENOSPC if cgroup charging fails or no folio is available. + * -ENOMEM if mem cgroup charging fails. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags) +{ + bool charge_hugetlb_cgroup_rsvd = alloc_flags & + HUGETLB_ALLOC_CHARG_CGROUP_RSVD; + bool use_global_reservation = alloc_flags & + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; + size_t nr_pages = pages_per_huge_page(h); + struct hugetlb_cgroup *h_cg_rsvd = NULL; + struct hugetlb_cgroup *h_cg = NULL; + gfp_t gfp = htlb_alloc_mask(h); + int idx = hstate_index(h); + struct folio *folio; + int ret; + + if (charge_hugetlb_cgroup_rsvd && + hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd)) + return ERR_PTR(-ENOSPC); + + if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup_rsvd; + } + + spin_lock_irq(&hugetlb_lock); + + folio = NULL; + if (use_global_reservation || available_huge_pages(h)) + folio = dequeue_hugetlb_folio(h, gfp, mpoli); + + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli); + if (!folio) { + ret = -ENOSPC; + goto err_uncharge_hugetlb_cgroup; + } + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + } + + if (use_global_reservation) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + + hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio); + + if (charge_hugetlb_cgroup_rsvd) { + hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd, + folio); + } + + spin_unlock_irq(&hugetlb_lock); + + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); + /* + * Unconditionally increment NR_HUGETLB here because if + * mem_cgroup_charge_hugetlb failed, freeing the page will + * decrement NR_HUGETLB. + */ + lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages); + + if (ret == -ENOMEM) { + free_huge_folio(folio); + /* + * Skip uncharging hugetlb_cgroup since the charges + * were committed to the folio and freeing the folio + * would have cleared those up. + */ + return ERR_PTR(ret); + } + + return folio; + + err_uncharge_hugetlb_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg); + err_uncharge_hugetlb_cgroup_rsvd: + if (charge_hugetlb_cgroup_rsvd) + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd); + + return ERR_PTR(ret); +} + typedef enum { /* * For either 0/1: we checked the per-vma resv map, and one resv @@ -2860,12 +2949,13 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long retval, gbl_chg, gbl_reserve; map_chg_state map_chg; - int ret, idx; - struct hugetlb_cgroup *h_cg = NULL; - struct hugetlb_cgroup *h_cg_rsvd = NULL; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; - - idx = hstate_index(h); + struct mempolicy_interpreted mpoli; + gfp_t gfp = htlb_alloc_mask(h); + struct mempolicy *mpol; + nodemask_t *nodemask; + u8 alloc_flags = 0; + int nid; + int ret; /* Whether we need a separate per-vma reservation? */ if (cow_from_owner) { @@ -2897,8 +2987,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, */ if (map_chg) { gbl_chg = hugepage_subpool_get_pages(spool, 1); - if (gbl_chg < 0) + if (gbl_chg < 0) { + ret = -ENOSPC; goto out_end_reservation; + } } else { /* * If we have the vma reservation ready, no need for extra @@ -2908,57 +3000,39 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } /* - * If this allocation is not consuming a per-vma reservation, - * charge the hugetlb cgroup now. + * If allocation doesn't reuse a reservation in the resv_map, + * charge for the reservation. */ - if (map_chg) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg_rsvd); - if (ret) - goto out_subpool_put; - } - - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_uncharge_cgroup_reservation; + if (map_chg != MAP_CHG_REUSE) + alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD; - spin_lock_irq(&hugetlb_lock); /* - * glb_chg is passed to indicate whether or not a page must be taken - * from the global free pool (global change). gbl_chg == 0 indicates - * a reservation exists for the allocation. + * gbl_chg == 0 indicates a reservation exists for this + * allocation, so try to use it. */ - folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg); - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); - if (!folio) - goto out_uncharge_cgroup; - spin_lock_irq(&hugetlb_lock); - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + if (gbl_chg == 0) + alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS; - /* - * Either dequeued or buddy-allocated folio needs to add special - * mark to the folio when it consumes a global reservation. - */ - if (!gbl_chg) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } + /* Takes reference on mpol. */ + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); + mpoli = (struct mempolicy_interpreted){ + .nid = nid, +#ifdef CONFIG_NUMA + .mode = mpol ? mpol->mode : MPOL_DEFAULT, +#else + .mode = MPOL_DEFAULT, +#endif + .nodemask = nodemask, + }; - hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (map_chg) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd, folio); - } + folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags); - spin_unlock_irq(&hugetlb_lock); + mpol_cond_put(mpol); + + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_subpool_put; + } hugetlb_set_folio_subpool(folio, spool); @@ -2987,27 +3061,8 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, } } - ret = mem_cgroup_charge_hugetlb(folio, gfp); - /* - * Unconditionally increment NR_HUGETLB here. If it turns out that - * mem_cgroup_charge_hugetlb failed, then immediately free the page and - * decrement NR_HUGETLB. - */ - lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h)); - - if (ret == -ENOMEM) { - free_huge_folio(folio); - return ERR_PTR(-ENOMEM); - } - return folio; -out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (map_chg) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg_rsvd); out_subpool_put: /* * put page to subpool iff the quota of subpool's rsv_hpages is used @@ -3018,100 +3073,94 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, hugetlb_acct_memory(h, -gbl_reserve); } - out_end_reservation: if (map_chg != MAP_CHG_ENFORCED) vma_end_reservation(h, vma, addr); - return ERR_PTR(-ENOSPC); + return ERR_PTR(ret); } static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} + +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + unsigned long pfn; + unsigned int nid_request = nid; + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + pfn = PHYS_PFN(__pa(m)); + nid = early_pfn_to_nid(pfn); /* - * Only initialize the head struct page in memmap_init_reserved_pages, - * rest of the struct pages will be initialized by the HugeTLB - * subsystem itself. - * The head struct page is used to get folio information by the HugeTLB - * subsystem like zone id and node id. + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. */ - memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), - huge_page_size(h) - PAGE_SIZE); + INIT_LIST_HEAD(&m->list); + m->hstate = h; + m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0; - return 1; + /* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */ + if (!hugetlb_early_cma(h) && + pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) { + /* + * If the allocated page is on a different node than requested + * (e.g., on PowerPC LPARs), put it on the requested node's list, + * because hugetlb_free_cross_zone_pages() only frees cross-zone + * pages belonging to the requested node. + */ + if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request)) + list_add(&m->list, &huge_boot_pages[nid_request]); + else + list_add(&m->list, &huge_boot_pages[nid]); + } else { + list_add_tail(&m->list, &huge_boot_pages[nid]); + m->flags |= HUGE_BOOTMEM_ZONES_VALID; + /* + * Only initialize the head struct page in memmap_init_reserved_pages, + * rest of the struct pages will be initialized by the HugeTLB + * subsystem itself. + * The head struct page is used to get folio information by the HugeTLB + * subsystem like zone id and node id. + */ + memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), + huge_page_size(h) - PAGE_SIZE); + } + + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ @@ -3223,57 +3272,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h, } } -bool __init hugetlb_bootmem_page_zones_valid(int nid, - struct huge_bootmem_page *m) -{ - unsigned long start_pfn; - bool valid; - - if (m->flags & HUGE_BOOTMEM_ZONES_VALID) { - /* - * Already validated, skip check. - */ - return true; - } - - if (hugetlb_bootmem_page_earlycma(m)) { - valid = cma_validate_zones(m->cma); - goto out; - } - - start_pfn = virt_to_phys(m) >> PAGE_SHIFT; - - valid = !pfn_range_intersects_zones(nid, start_pfn, - pages_per_huge_page(m->hstate)); -out: - if (!valid) - hstate_boot_nrinvalid[hstate_index(m->hstate)]++; - - return valid; -} - -/* - * Free a bootmem page that was found to be invalid (intersecting with - * multiple zones). - * - * Since it intersects with multiple zones, we can't just do a free - * operation on all pages at once, but instead have to walk all - * pages, freeing them one by one. - */ -static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page, - struct hstate *h) -{ - unsigned long npages = pages_per_huge_page(h); - unsigned long pfn; - - while (npages--) { - pfn = page_to_pfn(page); - __init_page_from_nid(pfn, nid); - free_reserved_page(page); - page++; - } -} - /* * Put bootmem huge pages into the standard lists after mem_map is up. * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages. @@ -3289,17 +3287,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid) struct folio *folio = (void *)page; h = m->hstate; - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Can't use this page. Initialize the - * page structures if that hasn't already - * been done, and give them to the page - * allocator. - */ - hugetlb_bootmem_free_invalid_page(nid, page, h); - continue; - } - /* * It is possible to have multiple huge page sizes (hstates) * in this list. If so, process each size separately. @@ -3353,7 +3340,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3365,10 +3352,63 @@ static void __init gather_bootmem_prealloc(void) .max_threads = num_node_state(N_MEMORY), .numa_aware = true, }; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct zone *zone; + + for_each_zone(zone) { + for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { + struct page *tail, *p; + unsigned int order; + + tail = zone->vmemmap_tails[i]; + if (!tail) + continue; + + order = i + VMEMMAP_TAIL_MIN_ORDER; + p = page_to_virt(tail); + /* + * prep_and_add_bootmem_folios() can access pageblock + * flags on bootmem HugeTLB pages, so initialize the + * shared tail struct pages here before bootmem folios + * start using them. + */ + for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) + init_compound_tail(p + j, NULL, order, zone); + } + } +#endif padata_do_multithreaded(&job); } +static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid) +{ + unsigned long freed = 0; + struct huge_bootmem_page *m, *tmp; + + if (!hstate_is_gigantic(h)) + return freed; + + list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) { + if (m->flags & HUGE_BOOTMEM_ZONES_VALID) + break; + + list_del(&m->list); + memblock_free(m, huge_page_size(h)); + freed++; + } + + if (freed) { + char buf[32]; + + string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf)); + pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n", + freed, buf, nid); + } + + return freed; +} + static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) { unsigned long i; @@ -3399,6 +3439,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) cond_resched(); } + i -= hugetlb_free_cross_zone_pages(h, nid); + if (!list_empty(&folio_list)) prep_and_add_allocated_folios(h, &folio_list); @@ -3472,6 +3514,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) { + int nid; unsigned long i; for (i = 0; i < h->max_huge_pages; ++i) { @@ -3480,6 +3523,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h) cond_resched(); } + for_each_node(nid) + i -= hugetlb_free_cross_zone_pages(h, nid); + return i; } @@ -3557,7 +3603,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -3635,20 +3681,13 @@ static void __init hugetlb_init_hstates(void) static void __init report_hugepages(void) { struct hstate *h; - unsigned long nrinvalid; for_each_hstate(h) { char buf[32]; - nrinvalid = hstate_boot_nrinvalid[hstate_index(h)]; - h->max_huge_pages -= nrinvalid; - string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n", buf, h->nr_huge_pages); - if (nrinvalid) - pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n", - buf, nrinvalid, str_plural(nrinvalid)); pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n", hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf); } @@ -4127,7 +4166,6 @@ static int __init hugetlb_init(void) } hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); @@ -4444,15 +4482,12 @@ hugetlb_early_param("default_hugepagesz", default_hugepagesz_setup); void __init hugetlb_bootmem_set_nodes(void) { int i, nid; - unsigned long start_pfn, end_pfn; if (!nodes_empty(hugetlb_bootmem_nodes)) return; - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { - if (end_pfn > start_pfn) - node_set(nid, hugetlb_bootmem_nodes); - } + for_each_mem_pfn_range(i, MAX_NUMNODES, NULL, NULL, &nid) + node_set(nid, hugetlb_bootmem_nodes); } void __init hugetlb_bootmem_alloc(void) @@ -4918,7 +4953,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, softleaf = softleaf_from_pte(entry); if (unlikely(softleaf_is_hwpoison(softleaf))) { if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + entry = pte_swp_clear_uffd_wp(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(softleaf_is_migration(softleaf))) { bool uffd_wp = pte_swp_uffd_wp(entry); @@ -4936,7 +4971,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, set_huge_pte_at(src, addr, src_pte, entry, sz); } if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + entry = pte_swp_clear_uffd_wp(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(pte_is_marker(entry))) { const pte_marker marker = copy_pte_marker(softleaf, dst_vma); @@ -7182,7 +7217,8 @@ void folio_putback_hugetlb(struct folio *folio) folio_put(folio); } -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason) +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason) { struct hstate *h = folio_hstate(old_folio); diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 39344d6c78d8..07faf625675b 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -9,6 +9,9 @@ #include #include +#include +#include +#include #include "internal.h" #include "hugetlb_cma.h" @@ -18,6 +21,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; static bool hugetlb_cma_only __ro_after_init; static unsigned long hugetlb_cma_size __ro_after_init; +static unsigned int hugetlb_cma_percent __initdata; +static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata; + +#ifdef CONFIG_NUMA +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + struct memblock_region *reg; + phys_addr_t size = 0; + + for_each_mem_region(reg) { + if (reg->nid == nid) + size += reg->size; + } + return size; +} +#else +static phys_addr_t __init memblock_node_memory_size(int nid) +{ + return memblock_phys_mem_size(); +} +#endif + void hugetlb_cma_free_frozen_folio(struct folio *folio) { WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)], @@ -34,7 +59,7 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, if (!hugetlb_cma_size) return NULL; - if (hugetlb_cma[nid]) + if (hugetlb_cma[nid] && node_isset(nid, *nodemask)) page = cma_alloc_frozen_compound(hugetlb_cma[nid], order); if (!page && !(gfp_mask & __GFP_THISNODE)) { @@ -56,37 +81,27 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; - struct huge_bootmem_page *m; - int node = *nid; + void *m; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); - if (!m) { - if (node_exact) - return NULL; - - for_each_node_mask(node, hugetlb_bootmem_nodes) { - cma = hugetlb_cma[node]; - if (!cma || node == *nid) - continue; - m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; - break; - } - } - } + if (m || node_exact) + return m; - if (m) { - m->flags = HUGE_BOOTMEM_CMA; - m->cma = cma; + for_each_node_mask(node, hugetlb_bootmem_nodes) { + cma = hugetlb_cma[node]; + if (!cma || node == nid) + continue; + m = cma_reserve_early(cma, huge_page_size(h)); + if (m) + return m; } - return m; + return NULL; } static int __init cmdline_parse_hugetlb_cma(char *p) @@ -100,14 +115,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) break; if (s[count] == ':') { + char *next; + if (tmp >= MAX_NUMNODES) break; nid = array_index_nospec(tmp, MAX_NUMNODES); s += count + 1; - tmp = memparse(s, &s); - hugetlb_cma_size_in_node[nid] = tmp; - hugetlb_cma_size += tmp; + tmp = memparse(s, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n", + tmp, nid); + break; + } + hugetlb_cma_percent_in_node[nid] = tmp; + hugetlb_cma_size_in_node[nid] = 0; + s = next + 1; + } else { + hugetlb_cma_size_in_node[nid] = tmp; + hugetlb_cma_percent_in_node[nid] = 0; + s = next; + } /* * Skip the separator if have one, otherwise @@ -118,7 +147,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) else break; } else { - hugetlb_cma_size = memparse(p, &p); + char *next; + + tmp = memparse(p, &next); + if (*next == '%') { + if (tmp > 100) { + pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp); + } else { + hugetlb_cma_percent = tmp; + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } + } else { + hugetlb_cma_size = tmp; + hugetlb_cma_percent = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + hugetlb_cma_size_in_node[nid] = 0; + hugetlb_cma_percent_in_node[nid] = 0; + } + } break; } } @@ -144,8 +194,36 @@ void __init hugetlb_cma_reserve(void) { unsigned long size, reserved, per_node, order, gigantic_page_size; bool node_specific_cma_alloc = false; + bool has_node_specific_param = false; int nid; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) { + has_node_specific_param = true; + break; + } + } + + if (has_node_specific_param) { + hugetlb_cma_size = 0; + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (hugetlb_cma_percent_in_node[nid]) { + phys_addr_t node_gfp_mem = memblock_node_memory_size(nid); + u64 s; + + s = mul_u64_u32_div((u64)node_gfp_mem, + hugetlb_cma_percent_in_node[nid], + 100); + + hugetlb_cma_size_in_node[nid] = s; + } + hugetlb_cma_size += hugetlb_cma_size_in_node[nid]; + } + } else if (hugetlb_cma_percent) { + hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(), + hugetlb_cma_percent, 100); + } + if (!hugetlb_cma_size) return; @@ -231,9 +309,11 @@ void __init hugetlb_cma_reserve(void) res = cma_declare_contiguous_multi(size, gigantic_page_size, HUGETLB_PAGE_ORDER, name, &hugetlb_cma[nid], nid); - if (res) { - pr_warn("hugetlb_cma: reservation failed: err %d, node %d", + if (res || !cma_validate_zones(hugetlb_cma[nid])) { + pr_warn("hugetlb_cma: %s: err %d, node %d\n", + res ? "reservation failed" : "reserved area spans zones", res, nid); + hugetlb_cma[nid] = NULL; continue; } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 133b46dfb09f..1430a5aa2de5 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -624,6 +624,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, LIST_HEAD(vmemmap_pages); unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; + if (list_empty(folio_list)) + return; + nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { int ret; @@ -635,12 +638,12 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), + register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)), &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); + HUGETLB_VMEMMAP_RESERVE_PAGES); continue; } @@ -745,6 +748,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m) return true; } +static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) +{ + struct zone *zone; + enum zone_type zone_type; + + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + zone = &NODE_DATA(nid)->node_zones[zone_type]; + if (zone_spans_pfn(zone, pfn)) + return zone; + } + + return NULL; +} + /* * Initialize memmap section for a gigantic page, HVO-style. */ @@ -752,6 +769,7 @@ void __init hugetlb_vmemmap_init_early(int nid) { unsigned long psize, paddr, section_size; unsigned long ns, i, pnum, pfn, nr_pages; + unsigned long start, end; struct huge_bootmem_page *m = NULL; void *map; @@ -761,6 +779,8 @@ void __init hugetlb_vmemmap_init_early(int nid) section_size = (1UL << PA_SECTION_SHIFT); list_for_each_entry(m, &huge_boot_pages[nid], list) { + struct zone *zone; + if (!vmemmap_should_optimize_bootmem_page(m)) continue; @@ -769,6 +789,14 @@ void __init hugetlb_vmemmap_init_early(int nid) paddr = virt_to_phys(m); pfn = PHYS_PFN(paddr); map = pfn_to_page(pfn); + start = (unsigned long)map; + end = start + hugetlb_vmemmap_size(m->hstate); + zone = pfn_to_zone(nid, pfn); + + if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate), + zone, HUGETLB_VMEMMAP_RESERVE_SIZE)) + panic("Failed to allocate memmap for HugeTLB page\n"); + memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE)); pnum = pfn_to_section_nr(pfn); ns = psize / section_size; @@ -783,78 +811,6 @@ void __init hugetlb_vmemmap_init_early(int nid) m->flags |= HUGE_BOOTMEM_HVO; } } - -static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn) -{ - struct zone *zone; - enum zone_type zone_type; - - for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { - zone = &NODE_DATA(nid)->node_zones[zone_type]; - if (zone_spans_pfn(zone, pfn)) - return zone; - } - - return NULL; -} - -void __init hugetlb_vmemmap_init_late(int nid) -{ - struct huge_bootmem_page *m, *tm; - unsigned long phys, nr_pages, start, end; - unsigned long pfn, nr_mmap; - struct zone *zone = NULL; - struct hstate *h; - void *map; - - if (!READ_ONCE(vmemmap_optimize_enabled)) - return; - - list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) { - if (!(m->flags & HUGE_BOOTMEM_HVO)) - continue; - - phys = virt_to_phys(m); - h = m->hstate; - pfn = PHYS_PFN(phys); - nr_pages = pages_per_huge_page(h); - map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - if (!hugetlb_bootmem_page_zones_valid(nid, m)) { - /* - * Oops, the hugetlb page spans multiple zones. - * Remove it from the list, and populate it normally. - */ - list_del(&m->list); - - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - - memblock_phys_free(phys, huge_page_size(h)); - continue; - } - - if (!zone || !zone_spans_pfn(zone, pfn)) - zone = pfn_to_zone(nid, pfn); - if (WARN_ON_ONCE(!zone)) - continue; - - if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone, - HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) { - /* Fallback if HVO population fails */ - vmemmap_populate(start, end, nid, NULL); - nr_mmap = end - start; - } else { - m->flags |= HUGE_BOOTMEM_ZONES_VALID; - nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE; - } - - memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); - } -} #endif static const struct ctl_table hugetlb_vmemmap_sysctls[] = { @@ -870,27 +826,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = { static int __init hugetlb_vmemmap_init(void) { const struct hstate *h; - struct zone *zone; /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */ BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES); - for_each_zone(zone) { - for (int i = 0; i < NR_VMEMMAP_TAILS; i++) { - struct page *tail, *p; - unsigned int order; - - tail = zone->vmemmap_tails[i]; - if (!tail) - continue; - - order = i + VMEMMAP_TAIL_MIN_ORDER; - p = page_to_virt(tail); - for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++) - init_compound_tail(p + j, NULL, order, zone); - } - } - for_each_hstate(h) { if (hugetlb_vmemmap_optimizable(h)) { register_sysctl_init("vm", hugetlb_vmemmap_sysctls); diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 18b490825215..7ac49c52457d 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list); #ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT void hugetlb_vmemmap_init_early(int nid); -void hugetlb_vmemmap_init_late(int nid); #endif @@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid) { } -static inline void hugetlb_vmemmap_init_late(int nid) -{ -} - static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h) { return 0; diff --git a/mm/internal.h b/mm/internal.h index 181e79f1d6a2..fa4fb69444ec 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,13 @@ #include "vma.h" struct folio_batch; +struct hstate; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; +}; /* * Maintains state across a page table move. The operation assumes both source @@ -668,7 +675,7 @@ void page_alloc_sysctl_init(void); */ struct alloc_context { struct zonelist *zonelist; - nodemask_t *nodemask; + const nodemask_t *nodemask; struct zoneref *preferred_zoneref; int migratetype; @@ -994,10 +1001,9 @@ static inline void sparse_init(void) {} * mm/sparse-vmemmap.c */ #ifdef CONFIG_SPARSEMEM_VMEMMAP -void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages); +void sparse_init_subsection_map(void); #else -static inline void sparse_init_subsection_map(unsigned long pfn, - unsigned long nr_pages) +static inline void sparse_init_subsection_map(void) { } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ @@ -1436,7 +1442,6 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); -extern void set_pageblock_order(void); unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, struct list_head *folio_list); @@ -1755,7 +1760,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte void __meminit __init_single_page(struct page *page, unsigned long pfn, unsigned long zone, int nid); -void __meminit __init_page_from_nid(unsigned long pfn, int nid); /* shrinker related functions */ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, @@ -1951,4 +1955,17 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +/* + * Ensure @mm is on the init_mm.mmlist so swapoff can find it. + */ +static inline void mm_prepare_for_swap_entries(struct mm_struct *mm) +{ + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } +} + #endif /* __MM_INTERNAL_H */ diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index cbef5e450954..a848eb2f9910 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -61,7 +61,7 @@ DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc); EXPORT_SYMBOL_GPL(kasan_flag_vmalloc); /* Whether to check write accesses only. */ -static bool kasan_flag_write_only = false; +static bool kasan_flag_write_only; #define PAGE_ALLOC_SAMPLE_DEFAULT 1 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3 diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 655dc5ce3240..6577bd76954e 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -77,6 +77,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_param WRITE_ONCE(kfence_enabled, false); } + if (num && kasan_hw_tags_enabled()) { + pr_info("disabled as KASAN HW tags are enabled\n"); + return -EINVAL; + } + *((unsigned long *)kp->arg) = num; if (num && !READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING) @@ -500,7 +505,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g /* * We check slab_want_init_on_alloc() ourselves, rather than letting - * SL*B do the initialization, as otherwise we might overwrite KFENCE's + * slab do the initialization, as otherwise it might overwrite KFENCE's * redzone. */ if (unlikely(slab_want_init_on_alloc(gfp, cache))) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 73e262cb30dd..617bca76db49 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -99,6 +99,8 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); static struct kmem_cache *mm_slot_cache __ro_after_init; +#define KHUGEPAGED_MIN_MTHP_ORDER 2 + struct collapse_control { bool is_khugepaged; @@ -110,6 +112,9 @@ struct collapse_control { /* nodemask for allocation fallback */ nodemask_t alloc_nmask; + + /* Each bit represents a single occupied (!none/zero) page. */ + DECLARE_BITMAP(mthp_present_ptes, MAX_PTRS_PER_PTE); }; /** @@ -348,6 +353,98 @@ static bool pte_none_or_zero(pte_t pte) return pte_present(pte) && is_zero_pfn(pte_pfn(pte)); } +/** + * collapse_max_ptes_none - Calculate maximum allowed empty PTEs or PTEs mapping + * the shared zeropage for the given collapse operation. + * @cc: The collapse control struct + * @vma: The vma to check for userfaultfd + * @order: The folio order being collapsed to + * + * Return: Maximum number of empty/shared zeropage PTEs for the collapse operation + */ +static unsigned int collapse_max_ptes_none(struct collapse_control *cc, + struct vm_area_struct *vma, unsigned int order) +{ + const unsigned int max_ptes_none = khugepaged_max_ptes_none; + + if (vma && userfaultfd_armed(vma)) + return 0; + /* for MADV_COLLAPSE, allow any empty/shared zeropage PTEs */ + if (!cc->is_khugepaged) + return HPAGE_PMD_NR; + /* for PMD collapse, respect the user defined maximum */ + if (is_pmd_order(order)) + return max_ptes_none; + /* + * for mTHP collapse with the sysctl value set to KHUGEPAGED_MAX_PTES_LIMIT, + * scale the maximum number of PTEs to the order of the collapse. + */ + if (max_ptes_none == KHUGEPAGED_MAX_PTES_LIMIT) + return (1 << order) - 1; + /* + * For mTHP collapse of values other than 0 or KHUGEPAGED_MAX_PTES_LIMIT, + * emit a warning and return 0. + */ + if (max_ptes_none) + pr_warn_once("mTHP collapse does not support max_ptes_none" + " values other than 0 or %u, defaulting to 0.\n", + KHUGEPAGED_MAX_PTES_LIMIT); + return 0; +} + +/** + * collapse_max_ptes_shared - Calculate maximum allowed PTEs that map shared + * anonymous pages for the given collapse operation. + * @cc: The collapse control struct + * @order: The folio order being collapsed to + * + * Return: Maximum number of PTEs that map shared anonymous pages for the + * collapse operation + */ +static unsigned int collapse_max_ptes_shared(struct collapse_control *cc, + unsigned int order) +{ + /* + * For MADV_COLLAPSE, do not restrict the number of PTEs that map shared + * anonymous pages. + */ + if (!cc->is_khugepaged) + return HPAGE_PMD_NR; + /* + * for mTHP collapse do not allow collapsing anonymous memory pages that + * are shared between processes. + */ + if (!is_pmd_order(order)) + return 0; + /* for PMD collapse, respect the user defined maximum */ + return khugepaged_max_ptes_shared; +} + +/** + * collapse_max_ptes_swap - Calculate the maximum allowed non-present PTEs or the + * maximum allowed non-present pagecache entries for the given collapse operation. + * @cc: The collapse control struct + * @order: The folio order being collapsed to + * + * Return: Maximum number of non-present PTEs or the maximum allowed non-present + * pagecache entries for the collapse operation. + */ +static unsigned int collapse_max_ptes_swap(struct collapse_control *cc, + unsigned int order) +{ + /* + * For MADV_COLLAPSE, do not restrict the number PTEs entries or + * pagecache entries that are non-present. + */ + if (!cc->is_khugepaged) + return HPAGE_PMD_NR; + /* for mTHP collapse do not allow any non-present PTEs or pagecache entries */ + if (!is_pmd_order(order)) + return 0; + /* for PMD collapse, respect the user defined maximum */ + return khugepaged_max_ptes_swap; +} + int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags, int advice) { @@ -406,26 +503,32 @@ static inline int collapse_test_exit_or_disable(struct mm_struct *mm) mm_flags_test(MMF_DISABLE_THP_COMPLETELY, mm); } -static bool hugepage_pmd_enabled(void) +static inline bool anon_hpage_enabled(void) +{ + if (READ_ONCE(huge_anon_orders_always)) + return true; + if (READ_ONCE(huge_anon_orders_madvise)) + return true; + if (READ_ONCE(huge_anon_orders_inherit) && + hugepage_global_enabled()) + return true; + return false; +} + +static bool hugepage_enabled(void) { /* * We cover the anon, shmem and the file-backed case here; file-backed - * hugepages, when configured in, are determined by the global control. - * Anon pmd-sized hugepages are determined by the pmd-size control. + * hugepages are determined by the global control. + * Anon hugepages are determined by its per-size mTHP control. * Shmem pmd-sized hugepages are also determined by its pmd-size control, * except when the global shmem_huge is set to SHMEM_HUGE_DENY. */ - if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && - hugepage_global_enabled()) + if (hugepage_global_enabled()) return true; - if (test_bit(PMD_ORDER, &huge_anon_orders_always)) - return true; - if (test_bit(PMD_ORDER, &huge_anon_orders_madvise)) - return true; - if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) && - hugepage_global_enabled()) + if (anon_hpage_enabled()) return true; - if (IS_ENABLED(CONFIG_SHMEM) && shmem_hpage_pmd_enabled()) + if (shmem_hpage_pmd_enabled()) return true; return false; } @@ -462,14 +565,36 @@ void __khugepaged_enter(struct mm_struct *mm) wake_up_interruptible(&khugepaged_wait); } +/* + * Check what orders are possible based on the vma and collapse type. + * This is used to determine if mTHP collapse is a viable option. + */ +static unsigned long collapse_possible_orders(struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type tva_flags) +{ + unsigned long orders; + + /* If khugepaged is scanning an anonymous vma, allow mTHP collapse */ + if ((tva_flags == TVA_KHUGEPAGED) && vma_is_anonymous(vma)) + orders = THP_ORDERS_ALL_ANON; + else + orders = BIT(HPAGE_PMD_ORDER); + + return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders); +} + +static bool collapse_possible(struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type tva_flags) +{ + return collapse_possible_orders(vma, vm_flags, tva_flags); +} + void khugepaged_enter_vma(struct vm_area_struct *vma, vm_flags_t vm_flags) { - if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && - hugepage_pmd_enabled()) { - if (thp_vma_allowable_order(vma, vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) - __khugepaged_enter(vma->vm_mm); - } + if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && hugepage_enabled() + && collapse_possible(vma, vm_flags, TVA_KHUGEPAGED)) + __khugepaged_enter(vma->vm_mm); } void __khugepaged_exit(struct mm_struct *mm) @@ -541,8 +666,11 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte, static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, unsigned long start_addr, pte_t *pte, struct collapse_control *cc, - struct list_head *compound_pagelist) + unsigned int order, struct list_head *compound_pagelist) { + const unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, order); + const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, order); + const unsigned long nr_pages = 1UL << order; struct page *page = NULL; struct folio *folio = NULL; unsigned long addr = start_addr; @@ -550,20 +678,18 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, int none_or_zero = 0, shared = 0, referenced = 0; enum scan_result result = SCAN_FAIL; - for (_pte = pte; _pte < pte + HPAGE_PMD_NR; + for (_pte = pte; _pte < pte + nr_pages; _pte++, addr += PAGE_SIZE) { pte_t pteval = ptep_get(_pte); if (pte_none_or_zero(pteval)) { - ++none_or_zero; - if (!userfaultfd_armed(vma) && - (!cc->is_khugepaged || - none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { + if (++none_or_zero > max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; - count_vm_event(THP_SCAN_EXCEED_NONE_PTE); + if (is_pmd_order(order)) + count_vm_event(THP_SCAN_EXCEED_NONE_PTE); + count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_NONE); goto out; } + continue; } if (!pte_present(pteval)) { result = SCAN_PTE_NON_PRESENT; @@ -594,14 +720,28 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, /* See collapse_scan_pmd(). */ if (folio_maybe_mapped_shared(folio)) { - ++shared; - if (cc->is_khugepaged && - shared > khugepaged_max_ptes_shared) { + /* + * TODO: Support shared pages without leading to further + * mTHP collapses. Currently bringing in new pages via + * shared may cause a future higher order collapse on a + * rescan of the same range. + */ + if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; - count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); + if (is_pmd_order(order)) + count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); + count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out; } } + /* + * TODO: In some cases of partially-mapped folios, we'd actually + * want to collapse. + */ + if (!is_pmd_order(order) && folio_order(folio) >= order) { + result = SCAN_PTE_MAPPED_HUGEPAGE; + goto out; + } if (folio_test_large(folio)) { struct folio *f; @@ -678,29 +818,29 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma, } else { result = SCAN_SUCCEED; trace_mm_collapse_huge_page_isolate(folio, none_or_zero, - referenced, result); + referenced, result, order); return result; } out: release_pte_pages(pte, _pte, compound_pagelist); trace_mm_collapse_huge_page_isolate(folio, none_or_zero, - referenced, result); + referenced, result, order); return result; } static void __collapse_huge_page_copy_succeeded(pte_t *pte, - struct vm_area_struct *vma, - unsigned long address, - spinlock_t *ptl, - struct list_head *compound_pagelist) + struct vm_area_struct *vma, unsigned long address, + spinlock_t *ptl, unsigned int order, + struct list_head *compound_pagelist) { - unsigned long end = address + HPAGE_PMD_SIZE; + const unsigned long nr_pages = 1UL << order; + unsigned long end = address + (PAGE_SIZE * nr_pages); struct folio *src, *tmp; pte_t pteval; pte_t *_pte; unsigned int nr_ptes; - for (_pte = pte; _pte < pte + HPAGE_PMD_NR; _pte += nr_ptes, + for (_pte = pte; _pte < pte + nr_pages; _pte += nr_ptes, address += nr_ptes * PAGE_SIZE) { nr_ptes = 1; pteval = ptep_get(_pte); @@ -753,11 +893,10 @@ static void __collapse_huge_page_copy_succeeded(pte_t *pte, } static void __collapse_huge_page_copy_failed(pte_t *pte, - pmd_t *pmd, - pmd_t orig_pmd, - struct vm_area_struct *vma, - struct list_head *compound_pagelist) + pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma, + unsigned int order, struct list_head *compound_pagelist) { + const unsigned long nr_pages = 1UL << order; spinlock_t *pmd_ptl; /* @@ -773,7 +912,7 @@ static void __collapse_huge_page_copy_failed(pte_t *pte, * Release both raw and compound pages isolated * in __collapse_huge_page_isolate. */ - release_pte_pages(pte, pte + HPAGE_PMD_NR, compound_pagelist); + release_pte_pages(pte, pte + nr_pages, compound_pagelist); } /* @@ -793,16 +932,17 @@ static void __collapse_huge_page_copy_failed(pte_t *pte, */ static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *folio, pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma, - unsigned long address, spinlock_t *ptl, + unsigned long address, spinlock_t *ptl, unsigned int order, struct list_head *compound_pagelist) { + const unsigned long nr_pages = 1UL << order; unsigned int i; enum scan_result result = SCAN_SUCCEED; /* * Copying pages' contents is subject to memory poison at any iteration. */ - for (i = 0; i < HPAGE_PMD_NR; i++) { + for (i = 0; i < nr_pages; i++) { pte_t pteval = ptep_get(pte + i); struct page *page = folio_page(folio, i); unsigned long src_addr = address + i * PAGE_SIZE; @@ -821,10 +961,10 @@ static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *foli if (likely(result == SCAN_SUCCEED)) __collapse_huge_page_copy_succeeded(pte, vma, address, ptl, - compound_pagelist); + order, compound_pagelist); else __collapse_huge_page_copy_failed(pte, pmd, orig_pmd, vma, - compound_pagelist); + order, compound_pagelist); return result; } @@ -905,12 +1045,13 @@ static int collapse_find_target_node(struct collapse_control *cc) /* * If mmap_lock temporarily dropped, revalidate vma - * before taking mmap_lock. + * after taking the mmap_lock again. * Returns enum scan_result value. */ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - bool expect_anon, struct vm_area_struct **vmap, struct collapse_control *cc) + bool expect_anon, struct vm_area_struct **vmap, + struct collapse_control *cc, unsigned int order) { struct vm_area_struct *vma; enum tva_type type = cc->is_khugepaged ? TVA_KHUGEPAGED : @@ -923,15 +1064,22 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l if (!vma) return SCAN_VMA_NULL; + /* + * We cannot collapse VMA regions that do not span the full PMD. This is + * due to the potential of the PMD being shared by another VMA leaving + * us vulnerable to a race condition. Always check the PMD order here to + * ensure its not shared by another VMA. We'd need to lock all VMAs in + * the PMD range to support this. + */ if (!thp_vma_suitable_order(vma, address, PMD_ORDER)) return SCAN_ADDRESS_RANGE; - if (!thp_vma_allowable_order(vma, vma->vm_flags, type, PMD_ORDER)) + if (!thp_vma_allowable_orders(vma, vma->vm_flags, type, BIT(order))) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then * remapped to file after khugepaged reaquired the mmap_lock. * - * thp_vma_allowable_order may return true for qualified file + * thp_vma_allowable_orders may return true for qualified file * vmas. */ if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap))) @@ -989,16 +1137,20 @@ static enum scan_result check_pmd_still_valid(struct mm_struct *mm, * Bring missing pages in from swap, to complete THP collapse. * Only done if khugepaged_scan_pmd believes it is worthwhile. * + * For mTHP orders the function bails on the first swap entry, because + * faulting pages back in during collapse could re-populate PTEs that + * push a later scan over the threshold for a higher-order collapse. + * * Called and returns without pte mapped or spinlocks held. * Returns result: if not SCAN_SUCCEED, mmap_lock has been released. */ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long start_addr, pmd_t *pmd, - int referenced) + struct vm_area_struct *vma, unsigned long start_addr, + pmd_t *pmd, int referenced, unsigned int order) { int swapped_in = 0; vm_fault_t ret = 0; - unsigned long addr, end = start_addr + (HPAGE_PMD_NR * PAGE_SIZE); + unsigned long addr, end = start_addr + (PAGE_SIZE << order); enum scan_result result; pte_t *pte = NULL; spinlock_t *ptl; @@ -1030,6 +1182,20 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, pte_present(vmf.orig_pte)) continue; + /* + * TODO: Support swapin without leading to further mTHP + * collapses. Currently bringing in new pages via swapin may + * cause a future higher order collapse on a rescan of the same + * range. + */ + if (!is_pmd_order(order)) { + count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SWAP); + pte_unmap(pte); + mmap_read_unlock(mm); + result = SCAN_EXCEED_SWAP_PTE; + goto out; + } + vmf.pte = pte; vmf.ptl = ptl; ret = do_swap_page(&vmf); @@ -1064,62 +1230,69 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, result = SCAN_SUCCEED; out: - trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result); + trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result, + order); return result; } static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, - struct collapse_control *cc) + struct collapse_control *cc, unsigned int order) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : GFP_TRANSHUGE); int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - count_vm_event(THP_COLLAPSE_ALLOC); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC); + count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC); + if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } - count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); + if (is_pmd_order(order)) + count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); *foliop = folio; return SCAN_SUCCEED; } -static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long address, - int referenced, int unmapped, struct collapse_control *cc) +/* + * collapse_huge_page expects the mmap_lock to be unlocked before entering and + * will always return with the lock unlocked, to avoid holding the mmap_lock + * while allocating a THP, as that could trigger direct reclaim/compaction. + * Note that the VMA must be rechecked after grabbing the mmap_lock again. + */ +static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long start_addr, + int referenced, int unmapped, struct collapse_control *cc, + unsigned int order) { + const unsigned long pmd_addr = start_addr & HPAGE_PMD_MASK; + const unsigned long end_addr = start_addr + (PAGE_SIZE << order); LIST_HEAD(compound_pagelist); pmd_t *pmd, _pmd; - pte_t *pte; + pte_t *pte = NULL; pgtable_t pgtable; struct folio *folio; spinlock_t *pmd_ptl, *pte_ptl; enum scan_result result = SCAN_FAIL; struct vm_area_struct *vma; struct mmu_notifier_range range; + bool anon_vma_locked = false; - VM_BUG_ON(address & ~HPAGE_PMD_MASK); - - /* - * Before allocating the hugepage, release the mmap_lock read lock. - * The allocation can take potentially a long time if it involves - * sync compaction, and we do not need to hold the mmap_lock during - * that. We will recheck the vma after taking it again in write mode. - */ - mmap_read_unlock(mm); - - result = alloc_charge_folio(&folio, mm, cc); + result = alloc_charge_folio(&folio, mm, cc, order); if (result != SCAN_SUCCEED) goto out_nolock; @@ -1129,13 +1302,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a } mmap_read_lock(mm); - result = hugepage_vma_revalidate(mm, address, true, &vma, cc); + result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true, + &vma, cc, order); if (result != SCAN_SUCCEED) { mmap_read_unlock(mm); goto out_nolock; } - result = find_pmd_or_thp_or_none(mm, address, &pmd); + result = find_pmd_or_thp_or_none(mm, pmd_addr, &pmd); if (result != SCAN_SUCCEED) { mmap_read_unlock(mm); goto out_nolock; @@ -1147,8 +1321,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a * released when it fails. So we jump out_nolock directly in * that case. Continuing to collapse causes inconsistency. */ - result = __collapse_huge_page_swapin(mm, vma, address, pmd, - referenced); + result = __collapse_huge_page_swapin(mm, vma, start_addr, pmd, + referenced, order); if (result != SCAN_SUCCEED) goto out_nolock; } @@ -1163,19 +1337,28 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a * mmap_lock. */ mmap_write_lock(mm); - result = hugepage_vma_revalidate(mm, address, true, &vma, cc); + result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true, + &vma, cc, order); if (result != SCAN_SUCCEED) goto out_up_write; /* check if the pmd is still valid */ vma_start_write(vma); - result = check_pmd_still_valid(mm, address, pmd); + result = check_pmd_still_valid(mm, pmd_addr, pmd); if (result != SCAN_SUCCEED) goto out_up_write; anon_vma_lock_write(vma->anon_vma); + anon_vma_locked = true; - mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address, - address + HPAGE_PMD_SIZE); + /* + * Only notify about the PTE range we will actually modify. While we + * temporary unmap the whole PTE table for mTHP collapse, we'll remap + * it later, leaving other PTEs effectively unmodified. The locks we + * hold prevent anybody from stumbling over such temporarily unmapped + * PTE tables. + */ + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, start_addr, + end_addr); mmu_notifier_invalidate_range_start(&range); pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ @@ -1187,25 +1370,23 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a * Parallel GUP-fast is fine since GUP-fast will back off when * it detects PMD is changed. */ - _pmd = pmdp_collapse_flush(vma, address, pmd); + _pmd = pmdp_collapse_flush(vma, pmd_addr, pmd); spin_unlock(pmd_ptl); mmu_notifier_invalidate_range_end(&range); tlb_remove_table_sync_one(); - pte = pte_offset_map_lock(mm, &_pmd, address, &pte_ptl); + pte = pte_offset_map_lock(mm, &_pmd, start_addr, &pte_ptl); if (pte) { - result = __collapse_huge_page_isolate(vma, address, pte, cc, - &compound_pagelist); + result = __collapse_huge_page_isolate(vma, start_addr, pte, cc, + order, &compound_pagelist); spin_unlock(pte_ptl); } else { result = SCAN_NO_PTE_TABLE; } if (unlikely(result != SCAN_SUCCEED)) { - if (pte) - pte_unmap(pte); spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); + VM_WARN_ON_ONCE(!pmd_none(*pmd)); /* * We can only use set_pmd_at when establishing * hugepmds and never for establishing regular pmds that @@ -1213,20 +1394,24 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a */ pmd_populate(mm, pmd, pmd_pgtable(_pmd)); spin_unlock(pmd_ptl); - anon_vma_unlock_write(vma->anon_vma); goto out_up_write; } /* - * All pages are isolated and locked so anon_vma rmap - * can't run anymore. + * For PMD collapse all pages are isolated and locked so anon_vma + * rmap can't run anymore. For mTHP collapse the PMD entry has been + * removed and not all pages are isolated and locked, so we must hold + * the lock to prevent neighboring folios from attempting to access + * this PMD until its reinstalled. */ - anon_vma_unlock_write(vma->anon_vma); + if (is_pmd_order(order)) { + anon_vma_unlock_write(vma->anon_vma); + anon_vma_locked = false; + } result = __collapse_huge_page_copy(pte, folio, pmd, _pmd, - vma, address, pte_ptl, - &compound_pagelist); - pte_unmap(pte); + vma, start_addr, pte_ptl, + order, &compound_pagelist); if (unlikely(result != SCAN_SUCCEED)) goto out_up_write; @@ -1236,37 +1421,190 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a * write. */ __folio_mark_uptodate(folio); - pgtable = pmd_pgtable(_pmd); - spin_lock(pmd_ptl); - BUG_ON(!pmd_none(*pmd)); - pgtable_trans_huge_deposit(mm, pmd, pgtable); - map_anon_folio_pmd_nopf(folio, pmd, vma, address); + VM_WARN_ON_ONCE(!pmd_none(*pmd)); + if (is_pmd_order(order)) { + pgtable = pmd_pgtable(_pmd); + pgtable_trans_huge_deposit(mm, pmd, pgtable); + map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr); + } else { + /* + * Some architectures (e.g. MIPS) walk the live page table in + * their implementation. update_mmu_cache_range() must be called + * with a valid page table hierarchy and the PTE lock held. + * Acquire it nested inside pmd_ptl when they are distinct locks. + */ + if (pte_ptl != pmd_ptl) + spin_lock_nested(pte_ptl, SINGLE_DEPTH_NESTING); + pmd_populate(mm, pmd, pmd_pgtable(_pmd)); + map_anon_folio_pte_nopf(folio, pte, vma, start_addr, + /*uffd_wp=*/ false); + if (pte_ptl != pmd_ptl) + spin_unlock(pte_ptl); + } spin_unlock(pmd_ptl); folio = NULL; result = SCAN_SUCCEED; out_up_write: + if (anon_vma_locked) + anon_vma_unlock_write(vma->anon_vma); + if (pte) + pte_unmap(pte); mmap_write_unlock(mm); out_nolock: if (folio) folio_put(folio); - trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); + trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order); return result; } +/* Return the highest naturally aligned order that fits at @offset within a PMD. */ +static unsigned int max_order_from_offset(unsigned int offset) +{ + if (offset == 0) + return HPAGE_PMD_ORDER; + + return min_t(unsigned int, __ffs(offset), HPAGE_PMD_ORDER); +} + +/* + * mthp_collapse() consumes the bitmap that is generated during + * collapse_scan_pmd() to determine what regions and mTHP orders fit best. + * + * Each bit in cc->mthp_present_ptes represents a single occupied (!none/zero) + * page. We start at the PMD order and check if it is eligible for collapse; + * if not, we check the left and right halves of the PTE page table we are + * examining at a lower order. + * + * For each of these, we determine how many PTE entries are occupied in the + * range of PTE entries we propose to collapse, then we compare this to a + * threshold number of PTE entries which would need to be occupied for a + * collapse to be permitted at that order (accounting for max_ptes_none). + * + * If a collapse is permitted, we attempt to collapse the PTE range into a + * mTHP. + */ +static enum scan_result mthp_collapse(struct mm_struct *mm, + unsigned long address, int referenced, int unmapped, + struct collapse_control *cc, unsigned long enabled_orders) +{ + unsigned int nr_occupied_ptes, nr_ptes, max_ptes_none; + enum scan_result last_result = SCAN_FAIL; + int collapsed = 0; + bool alloc_failed = false; + unsigned long collapse_address; + unsigned int offset = 0; + unsigned int order = HPAGE_PMD_ORDER; + + while (offset < HPAGE_PMD_NR) { + nr_ptes = 1UL << order; + + if (!test_bit(order, &enabled_orders)) + goto next_order; + + max_ptes_none = collapse_max_ptes_none(cc, NULL, order); + nr_occupied_ptes = bitmap_weight_from(cc->mthp_present_ptes, offset, + offset + nr_ptes); + + /* + * Swap PTEs accepted during the scan are counted in @unmapped, + * not in the present-PTE bitmap. Account them for the PMD-order + * candidate. + */ + if (is_pmd_order(order)) + nr_occupied_ptes += unmapped; + + if (nr_occupied_ptes >= nr_ptes - max_ptes_none) { + enum scan_result ret; + + collapse_address = address + offset * PAGE_SIZE; + ret = collapse_huge_page(mm, collapse_address, referenced, + unmapped, cc, order); + + switch (ret) { + /* Cases where we continue to next collapse candidate */ + case SCAN_SUCCEED: + collapsed += nr_ptes; + fallthrough; + case SCAN_PTE_MAPPED_HUGEPAGE: + goto next_offset; + /* Cases where lower orders might still succeed */ + case SCAN_ALLOC_HUGE_PAGE_FAIL: + alloc_failed = true; + fallthrough; + case SCAN_LACK_REFERENCED_PAGE: + case SCAN_EXCEED_NONE_PTE: + case SCAN_EXCEED_SWAP_PTE: + case SCAN_EXCEED_SHARED_PTE: + case SCAN_PAGE_LOCK: + case SCAN_PAGE_COUNT: + case SCAN_PAGE_NULL: + case SCAN_DEL_PAGE_LRU: + case SCAN_PTE_NON_PRESENT: + case SCAN_PTE_UFFD_WP: + case SCAN_PAGE_LAZYFREE: + last_result = ret; + goto next_order; + /* Cases where no further collapse is possible */ + case SCAN_PMD_MAPPED: + fallthrough; + default: + last_result = ret; + goto done; + } + } + +next_order: + /* + * Continue with the next smaller order if there is still + * any smaller order enabled. When at the smallest order + * we must always move to the next offset. + */ + if (order > KHUGEPAGED_MIN_MTHP_ORDER && + (enabled_orders & GENMASK(order - 1, 0))) { + order--; + continue; + } +next_offset: + /* + * Advance past the region we just processed and determine the + * highest order we can attempt next. Since huge pages must be + * naturally aligned, the max order we can attempt next is + * limited by the alignment of the new offset. + * E.g. if we collapsed a order-2 mTHP at offset 0, offset + * becomes 4 and __ffs(4) == 2, so the next attempt starts at + * order 2. + */ + offset += nr_ptes; + order = max_order_from_offset(offset); + } +done: + if (collapsed) + return SCAN_SUCCEED; + if (alloc_failed) + return SCAN_ALLOC_HUGE_PAGE_FAIL; + return last_result; +} + static enum scan_result collapse_scan_pmd(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long start_addr, bool *lock_dropped, struct collapse_control *cc) { + const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER); + const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER); + unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER); + enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE; pmd_t *pmd; - pte_t *pte, *_pte; + pte_t *pte, *_pte, pteval; + int i; int none_or_zero = 0, shared = 0, referenced = 0; enum scan_result result = SCAN_FAIL; struct page *page = NULL; struct folio *folio = NULL; unsigned long addr; + unsigned long enabled_orders; spinlock_t *ptl; int node = NUMA_NO_NODE, unmapped = 0; @@ -1278,8 +1616,19 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, goto out; } + bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE); memset(cc->node_load, 0, sizeof(cc->node_load)); nodes_clear(cc->alloc_nmask); + + enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags); + + /* + * If PMD is the only enabled order, enforce max_ptes_none, otherwise + * scan all pages to populate the bitmap for mTHP collapse. + */ + if (enabled_orders != BIT(HPAGE_PMD_ORDER)) + max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT; + pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl); if (!pte) { cc->progress++; @@ -1287,42 +1636,41 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, goto out; } - for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR; - _pte++, addr += PAGE_SIZE) { + for (i = 0; i < HPAGE_PMD_NR; i++) { + _pte = pte + i; + addr = start_addr + i * PAGE_SIZE; + pteval = ptep_get(_pte); + cc->progress++; - pte_t pteval = ptep_get(_pte); if (pte_none_or_zero(pteval)) { - ++none_or_zero; - if (!userfaultfd_armed(vma) && - (!cc->is_khugepaged || - none_or_zero <= khugepaged_max_ptes_none)) { - continue; - } else { + if (++none_or_zero > max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); + count_mthp_stat(HPAGE_PMD_ORDER, + MTHP_STAT_COLLAPSE_EXCEED_NONE); goto out_unmap; } + continue; } if (!pte_present(pteval)) { - ++unmapped; - if (!cc->is_khugepaged || - unmapped <= khugepaged_max_ptes_swap) { - /* - * Always be strict with uffd-wp - * enabled swap entries. Please see - * comment below for pte_uffd_wp(). - */ - if (pte_swp_uffd_wp_any(pteval)) { - result = SCAN_PTE_UFFD_WP; - goto out_unmap; - } - continue; - } else { + if (++unmapped > max_ptes_swap) { result = SCAN_EXCEED_SWAP_PTE; count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); + count_mthp_stat(HPAGE_PMD_ORDER, + MTHP_STAT_COLLAPSE_EXCEED_SWAP); + goto out_unmap; + } + /* + * Always be strict with uffd-wp + * enabled swap entries. Please see + * comment below for pte_uffd_wp(). + */ + if (pte_swp_uffd_wp_any(pteval)) { + result = SCAN_PTE_UFFD_WP; goto out_unmap; } + continue; } if (pte_uffd_wp(pteval)) { /* @@ -1365,15 +1713,17 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, * is shared. */ if (folio_maybe_mapped_shared(folio)) { - ++shared; - if (cc->is_khugepaged && - shared > khugepaged_max_ptes_shared) { + if (++shared > max_ptes_shared) { result = SCAN_EXCEED_SHARED_PTE; count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); + count_mthp_stat(HPAGE_PMD_ORDER, + MTHP_STAT_COLLAPSE_EXCEED_SHARED); goto out_unmap; } } + /* Set bit for occupied pages */ + __set_bit(i, cc->mthp_present_ptes); /* * Record which node the original page is from and save this * information to cc->node_load[]. @@ -1430,9 +1780,11 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm, out_unmap: pte_unmap_unlock(pte, ptl); if (result == SCAN_SUCCEED) { - result = collapse_huge_page(mm, start_addr, referenced, - unmapped, cc); - /* collapse_huge_page will return with the mmap_lock released */ + /* collapse_huge_page expects the lock to be dropped before calling */ + mmap_read_unlock(mm); + result = mthp_collapse(mm, start_addr, referenced, + unmapped, cc, enabled_orders); + /* mmap_lock was released above, set lock_dropped */ *lock_dropped = true; } out: @@ -1900,10 +2252,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, int nr_none = 0; bool is_shmem = shmem_file(file); - VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + /* + * MADV_COLLAPSE ignores shmem huge config, so do not check shmem + * + * TODO: once shmem always calls mapping_set_large_folios() on its + * mapping, the shmem check can be removed. + */ + VM_WARN_ON_ONCE(!is_shmem && !mapping_pmd_folio_support(mapping)); + VM_WARN_ON_ONCE(start & (HPAGE_PMD_NR - 1)); - result = alloc_charge_folio(&new_folio, mm, cc); + result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out; @@ -1986,20 +2344,22 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } } else if (folio_test_dirty(folio)) { /* - * khugepaged only works on read-only fd, - * so this page is dirty because it hasn't - * been flushed since first write. There - * won't be new dirty pages. + * This page is dirty because it hasn't + * been flushed since first write. * - * Trigger async flush here and hope the - * writeback is done when khugepaged - * revisits this page. + * Trigger async flush for read-only files and + * hope the writeback is done when khugepaged + * revisits this page. Writable files can have + * their folios dirty at any time; blindly + * flushing them would cause undesirable + * system-wide writeback. * * This is a one-off situation. We are not * forcing writeback in loop. */ xas_unlock_irq(&xas); - filemap_flush(mapping); + if (!inode_is_open_for_write(mapping->host)) + filemap_flush(mapping); result = SCAN_PAGE_DIRTY_OR_WRITEBACK; goto xa_unlocked; } else if (folio_test_writeback(folio)) { @@ -2045,8 +2405,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, if (!is_shmem && (folio_test_dirty(folio) || folio_test_writeback(folio))) { /* - * khugepaged only works on read-only fd, so this - * folio is dirty because it hasn't been flushed + * khugepaged only works on clean file-backed folios, + * so this folio is dirty because it hasn't been flushed * since first write. */ result = SCAN_PAGE_DIRTY_OR_WRITEBACK; @@ -2090,6 +2450,27 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, goto out_unlock; } + /* + * At this point, the folio is locked and unmapped. If the PTE + * was dirty, try_to_unmap() has transferred the dirty bit to + * the folio and we must not collapse it into a clean + * file-backed folio. + * + * If the folio is clean here, no one can write it until we + * drop the folio lock. A write through a stale TLB entry came + * from a clean PTE and must fault because the PTE has been + * cleared; the fault path has to take the folio lock before + * installing a writable mapping. Buffered write paths also + * have to take the folio lock before modifying file contents + * without a mapping, typically via write_begin_get_folio(). + */ + if (!is_shmem && folio_test_dirty(folio)) { + result = SCAN_PAGE_DIRTY_OR_WRITEBACK; + xas_unlock_irq(&xas); + folio_putback_lru(folio); + goto out_unlock; + } + /* * Accumulate the folios that are being collapsed. */ @@ -2102,21 +2483,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, goto xa_unlocked; } - if (!is_shmem) { - filemap_nr_thps_inc(mapping); - /* - * Paired with the fence in do_dentry_open() -> get_write_access() - * to ensure i_writecount is up to date and the update to nr_thps - * is visible. Ensures the page cache will be truncated if the - * file is opened writable. - */ - smp_mb(); - if (inode_is_open_for_write(mapping->host)) { - result = SCAN_FAIL; - filemap_nr_thps_dec(mapping); - } - } - xa_locked: xas_unlock_irq(&xas); xa_unlocked: @@ -2294,19 +2660,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_putback_lru(folio); folio_put(folio); } - /* - * Undo the updates of filemap_nr_thps_inc for non-SHMEM - * file only. This undo is not needed unless failure is - * due to SCAN_COPY_MC. - */ - if (!is_shmem && result == SCAN_COPY_MC) { - filemap_nr_thps_dec(mapping); - /* - * Paired with the fence in do_dentry_open() -> get_write_access() - * to ensure the update to nr_thps is visible. - */ - smp_mb(); - } new_folio->mapping = NULL; @@ -2322,6 +2675,8 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, unsigned long addr, struct file *file, pgoff_t start, struct collapse_control *cc) { + const unsigned int max_ptes_none = collapse_max_ptes_none(cc, NULL, HPAGE_PMD_ORDER); + const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER); struct folio *folio = NULL; struct address_space *mapping = file->f_mapping; XA_STATE(xas, &mapping->i_pages, start); @@ -2340,8 +2695,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, if (xa_is_value(folio)) { swap += 1 << xas_get_order(&xas); - if (cc->is_khugepaged && - swap > khugepaged_max_ptes_swap) { + if (swap > max_ptes_swap) { result = SCAN_EXCEED_SWAP_PTE; count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); break; @@ -2412,8 +2766,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, cc->progress += HPAGE_PMD_NR; if (result == SCAN_SUCCEED) { - if (cc->is_khugepaged && - present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { + if (present < HPAGE_PMD_NR - max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { @@ -2532,7 +2885,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max, cc->progress++; break; } - if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) { + if (!collapse_possible(vma, vma->vm_flags, TVA_KHUGEPAGED)) { cc->progress++; continue; } @@ -2607,7 +2960,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max, static int khugepaged_has_work(void) { - return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled(); + return !list_empty(&khugepaged_scan.mm_head) && hugepage_enabled(); } static int khugepaged_wait_event(void) @@ -2680,7 +3033,7 @@ static void khugepaged_wait_work(void) return; } - if (hugepage_pmd_enabled()) + if (hugepage_enabled()) wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); } @@ -2711,7 +3064,7 @@ void set_recommended_min_free_kbytes(void) int nr_zones = 0; unsigned long recommended_min; - if (!hugepage_pmd_enabled()) { + if (!hugepage_enabled()) { calculate_min_free_kbytes(); goto update_wmarks; } @@ -2761,7 +3114,7 @@ int start_stop_khugepaged(void) int err = 0; mutex_lock(&khugepaged_mutex); - if (hugepage_pmd_enabled()) { + if (hugepage_enabled()) { if (!khugepaged_thread) khugepaged_thread = kthread_run(khugepaged, NULL, "khugepaged"); @@ -2787,7 +3140,7 @@ int start_stop_khugepaged(void) void khugepaged_min_free_kbytes_update(void) { mutex_lock(&khugepaged_mutex); - if (hugepage_pmd_enabled() && khugepaged_thread) + if (hugepage_enabled() && khugepaged_thread) set_recommended_min_free_kbytes(); mutex_unlock(&khugepaged_mutex); } @@ -2842,9 +3195,15 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, BUG_ON(vma->vm_start > start); BUG_ON(vma->vm_end < end); - if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER)) + if (!collapse_possible(vma, vma->vm_flags, TVA_FORCED_COLLAPSE)) return -EINVAL; + hstart = ALIGN(start, HPAGE_PMD_SIZE); + hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE); + + if (hstart >= hend) + return 0; + cc = kmalloc_obj(*cc); if (!cc) return -ENOMEM; @@ -2854,9 +3213,6 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, mmgrab(mm); lru_add_drain_all(); - hstart = ALIGN(start, HPAGE_PMD_SIZE); - hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE); - for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { enum scan_result result = SCAN_FAIL; @@ -2866,8 +3222,8 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start, mmap_unlocked = false; *lock_dropped = true; result = hugepage_vma_revalidate(mm, addr, false, &vma, - cc); - if (result != SCAN_SUCCEED) { + cc, HPAGE_PMD_ORDER); + if (result != SCAN_SUCCEED) { last_fail = result; goto out_nolock; } diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 7c7ba17ce7af..e96e9efd19b0 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -687,7 +687,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) atomic_set(&object->use_count, 1); object->excess_ref = 0; object->count = 0; /* white color initially */ - object->checksum = 0; + object->checksum = ~0; object->del_state = 0; /* task information */ @@ -981,7 +981,7 @@ static void reset_checksum(unsigned long ptr) } raw_spin_lock_irqsave(&object->lock, flags); - object->checksum = 0; + object->checksum = ~0; raw_spin_unlock_irqrestore(&object->lock, flags); put_object(object); } @@ -1410,7 +1410,8 @@ static bool update_checksum(struct kmemleak_object *object) for_each_possible_cpu(cpu) { void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); - object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); + object->checksum = crc32(object->checksum, + kasan_reset_tag((void *)ptr), object->size); } } else { object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); @@ -1524,22 +1525,25 @@ static int scan_should_stop(void) /* * Scan a memory block (exclusive range) for valid pointers and add those - * found to the gray list. + * found to the gray list. Return non-zero if the scan was interrupted. */ -static void scan_block(void *_start, void *_end, - struct kmemleak_object *scanned) +static int scan_block(void *_start, void *_end, + struct kmemleak_object *scanned) { unsigned long *ptr; unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER); unsigned long *end = _end - (BYTES_PER_POINTER - 1); unsigned long flags; + int stop = 0; raw_spin_lock_irqsave(&kmemleak_lock, flags); for (ptr = start; ptr < end; ptr++) { unsigned long pointer; - if (scan_should_stop()) + if (scan_should_stop()) { + stop = 1; break; + } kasan_disable_current(); pointer = *(unsigned long *)kasan_reset_tag((void *)ptr); @@ -1549,22 +1553,28 @@ static void scan_block(void *_start, void *_end, pointer_update_refs(scanned, pointer, OBJECT_PERCPU); } raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return stop; } /* * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency. + * Return non-zero if the scan was interrupted. */ #ifdef CONFIG_SMP -static void scan_large_block(void *start, void *end) +static int scan_large_block(void *start, void *end) { void *next; while (start < end) { next = min(start + MAX_SCAN_SIZE, end); - scan_block(start, next, NULL); + if (scan_block(start, next, NULL)) + return 1; start = next; cond_resched(); } + + return 0; } #endif @@ -1695,6 +1705,43 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) put_object(object); } +/* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + int stop = 0; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + stop = scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched(); + } while (pid && !stop); +} + /* * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it * does not touch user memory that was freed concurrently; the rest of the @@ -1804,6 +1851,7 @@ static void kmemleak_scan(void) int __maybe_unused i; struct xarray dedup; int new_leaks = 0; + int stop = 0; jiffies_last_scan = jiffies; @@ -1847,9 +1895,11 @@ static void kmemleak_scan(void) #ifdef CONFIG_SMP /* per-cpu sections scanning */ - for_each_possible_cpu(i) - scan_large_block(__per_cpu_start + per_cpu_offset(i), - __per_cpu_end + per_cpu_offset(i)); + for_each_possible_cpu(i) { + if (scan_large_block(__per_cpu_start + per_cpu_offset(i), + __per_cpu_end + per_cpu_offset(i))) + goto scan_gray; + } #endif /* @@ -1876,32 +1926,28 @@ static void kmemleak_scan(void) /* only scan if page is in use */ if (page_count(page) == 0) continue; - scan_block(page, page + 1, NULL); + stop = scan_block(page, page + 1, NULL); + if (stop) + break; } + if (stop) + break; } put_online_mems(); + if (stop) + goto scan_gray; /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned * above. */ +scan_gray: scan_gray_list(); /* diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b..41ab25aa2a82 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2327,23 +2327,24 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite tree_rmap_item = unstable_tree_search_insert(rmap_item, page, &tree_page); if (tree_rmap_item) { + struct folio *tree_folio; bool split; kfolio = try_to_merge_two_pages(rmap_item, page, tree_rmap_item, tree_page); + tree_folio = page_folio(tree_page); /* - * If both pages we tried to merge belong to the same compound - * page, then we actually ended up increasing the reference - * count of the same compound page twice, and split_huge_page - * failed. + * If both pages we tried to merge belong to the same (large) + * folio, then we actually ended up increasing the reference + * count of the same folio twice, and split_huge_page failed. + * * Here we set a flag if that happened, and we use it later to - * try split_huge_page again. Since we call put_page right + * try split_huge_page again. Since we call folio_put() right * afterwards, the reference count will be correct and * split_huge_page should succeed. */ - split = PageTransCompound(page) - && compound_head(page) == compound_head(tree_page); - put_page(tree_page); + split = folio == tree_folio; + folio_put(tree_folio); if (kfolio) { /* * The pages were successfully merged: insert new diff --git a/mm/madvise.c b/mm/madvise.c index cd9bb077072c..9292f60b19aa 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -336,8 +336,7 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) * otherwise we'd be including shared non-exclusive mappings, which * opens a side channel. */ - return inode_owner_or_capable(&nop_mnt_idmap, - file_inode(vma->vm_file)) || + return file_owner_or_capable(vma->vm_file) || file_permission(vma->vm_file, MAY_WRITE) == 0; } @@ -695,10 +694,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, nr = swap_pte_batch(pte, max_nr, ptent); nr_swap -= nr; swap_put_entries_direct(entry, nr); - clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(mm, addr, pte, nr); } else if (softleaf_is_hwpoison(entry) || softleaf_is_poison_marker(entry)) { - pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); + pte_clear(mm, addr, pte); } continue; } @@ -1234,7 +1233,7 @@ static int guard_remove_pte_entry(pte_t *pte, unsigned long addr, if (is_guard_pte_marker(ptent)) { /* Simply clear the PTE marker. */ - pte_clear_not_present_full(walk->mm, addr, pte, false); + pte_clear(walk->mm, addr, pte); update_mmu_cache(walk->vma, addr, pte); } diff --git a/mm/memblock.c b/mm/memblock.c index ccd43f3abb82..43e036f3e11a 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1028,40 +1028,6 @@ int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) } #endif -#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH -__init void memblock_set_kho_scratch_only(void) -{ - kho_scratch_only = true; -} - -__init void memblock_clear_kho_scratch_only(void) -{ - kho_scratch_only = false; -} - -__init void memmap_init_kho_scratch_pages(void) -{ - phys_addr_t start, end; - unsigned long pfn; - int nid; - u64 i; - - if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) - return; - - /* - * Initialize struct pages for free scratch memory. - * The struct pages for reserved scratch memory will be set up in - * memmap_init_reserved_pages() - */ - __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, - MEMBLOCK_KHO_SCRATCH, &start, &end, &nid) { - for (pfn = PFN_UP(start); pfn < PFN_DOWN(end); pfn++) - init_deferred_page(pfn, nid); - } -} -#endif - /** * memblock_setclr_flag - set or clear flag for a memory region * @type: memblock type to set/clear flag for @@ -2258,10 +2224,8 @@ static void __init free_unused_memmap(void) } #ifdef CONFIG_SPARSEMEM - if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { - prev_end = pageblock_align(end); + if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); - } #endif } @@ -2535,6 +2499,28 @@ int reserve_mem_release_by_name(const char *name) return 1; } +#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH +__init void memblock_set_kho_scratch_only(void) +{ + kho_scratch_only = true; +} + +__init void memblock_clear_kho_scratch_only(void) +{ + kho_scratch_only = false; +} + +bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr) +{ + int i = memblock_search(&memblock.memory, addr); + + if (i == -1) + return false; + + return memblock_is_kho_scratch(&memblock.memory.regions[i]); +} +#endif + #ifdef CONFIG_KEXEC_HANDOVER static int __init reserved_mem_preserve(void) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 765069211567..e8b6e1560278 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1476,6 +1477,297 @@ void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) mem_cgroup_oom_unlock(memcg); } +/* + * cgroup v1 userspace vmpressure interface (memory.pressure_level / + * cgroup.event_control). Kept here so v2-only kernels (CONFIG_MEMCG_V1=n) + * drop the whole eventfd accumulator, its work item, and the per-memcg + * state it requires. + * + * When there are too little pages left to scan, vmpressure() may miss the + * critical pressure as number of pages will be less than "window size". + * However, in that case the vmscan priority will raise fast as the + * reclaimer will try to scan LRUs more deeply. + * + * The vmscan logic considers these special priorities: + * + * prio == DEF_PRIORITY (12): reclaimer starts with that value + * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed + * prio == 0 : close to OOM, kernel scans every page in an lru + * + * Any value in this range is acceptable for this tunable (i.e. from 12 to + * 0). Current value for the vmpressure_level_critical_prio is chosen + * empirically, but the number, in essence, means that we consider + * critical level when scanning depth is ~10% of the lru size (vmscan + * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one + * eights). + */ +static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); + +enum vmpressure_modes { + VMPRESSURE_NO_PASSTHROUGH = 0, + VMPRESSURE_HIERARCHY, + VMPRESSURE_LOCAL, + VMPRESSURE_NUM_MODES, +}; + +static const char * const vmpressure_str_levels[] = { + [VMPRESSURE_LOW] = "low", + [VMPRESSURE_MEDIUM] = "medium", + [VMPRESSURE_CRITICAL] = "critical", +}; + +static const char * const vmpressure_str_modes[] = { + [VMPRESSURE_NO_PASSTHROUGH] = "default", + [VMPRESSURE_HIERARCHY] = "hierarchy", + [VMPRESSURE_LOCAL] = "local", +}; + +struct vmpressure_event { + struct eventfd_ctx *efd; + enum vmpressure_levels level; + enum vmpressure_modes mode; + struct list_head node; +}; + +static struct vmpressure *work_to_vmpressure(struct work_struct *work) +{ + return container_of(work, struct vmpressure, work); +} + +static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) +{ + struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); + + memcg = parent_mem_cgroup(memcg); + if (!memcg) + return NULL; + return memcg_to_vmpressure(memcg); +} + +static bool vmpressure_event(struct vmpressure *vmpr, + const enum vmpressure_levels level, + bool ancestor, bool signalled) +{ + struct vmpressure_event *ev; + bool ret = false; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ancestor && ev->mode == VMPRESSURE_LOCAL) + continue; + if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) + continue; + if (level < ev->level) + continue; + eventfd_signal(ev->efd); + ret = true; + } + mutex_unlock(&vmpr->events_lock); + + return ret; +} + +static void vmpressure_work_fn(struct work_struct *work) +{ + struct vmpressure *vmpr = work_to_vmpressure(work); + unsigned long scanned; + unsigned long reclaimed; + enum vmpressure_levels level; + bool ancestor = false; + bool signalled = false; + + spin_lock(&vmpr->sr_lock); + /* + * Several contexts might be calling vmpressure(), so it is + * possible that the work was rescheduled again before the old + * work context cleared the counters. In that case we will run + * just after the old work returns, but then scanned might be zero + * here. No need for any locks here since we don't care if + * vmpr->reclaimed is in sync. + */ + scanned = vmpr->tree_scanned; + if (!scanned) { + spin_unlock(&vmpr->sr_lock); + return; + } + + reclaimed = vmpr->tree_reclaimed; + vmpr->tree_scanned = 0; + vmpr->tree_reclaimed = 0; + spin_unlock(&vmpr->sr_lock); + + level = vmpressure_calc_level(scanned, reclaimed); + + do { + if (vmpressure_event(vmpr, level, ancestor, signalled)) + signalled = true; + ancestor = true; + } while ((vmpr = vmpressure_parent(vmpr))); +} + +/* + * Tree-mode accumulator: accumulate per-memcg scanned/reclaimed and + * schedule the work that walks the parent chain and signals registered + * eventfd listeners once we cross the window threshold. + */ +void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) +{ + spin_lock(&vmpr->sr_lock); + scanned = vmpr->tree_scanned += scanned; + vmpr->tree_reclaimed += reclaimed; + spin_unlock(&vmpr->sr_lock); + + if (scanned < vmpressure_win) + return; + schedule_work(&vmpr->work); +} + +void vmpressure_v1_init(struct vmpressure *vmpr) +{ + mutex_init(&vmpr->events_lock); + INIT_LIST_HEAD(&vmpr->events); + INIT_WORK(&vmpr->work, vmpressure_work_fn); +} + +void vmpressure_v1_cleanup(struct vmpressure *vmpr) +{ + /* + * Make sure there is no pending work before eventfd infrastructure + * goes away. + */ + flush_work(&vmpr->work); +} + +/** + * vmpressure_prio() - Account memory pressure through reclaimer priority level + * @gfp: reclaimer's gfp mask + * @memcg: cgroup memory controller handle + * @prio: reclaimer's priority + * + * This function should be called from the reclaim path every time when + * the vmscan's reclaiming priority (scanning depth) changes. + * + * This function does not return any value. + */ +void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) +{ + /* + * We only use prio for accounting critical level. For more info + * see comment for vmpressure_level_critical_prio variable above. + */ + if (prio > vmpressure_level_critical_prio) + return; + + /* + * OK, the prio is below the threshold, updating vmpressure + * information before shrinker dives into long shrinking of long + * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 + * to the vmpressure() basically means that we signal 'critical' + * level. + */ + vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); +} + +#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) + +/** + * vmpressure_register_event() - Bind vmpressure notifications to an eventfd + * @memcg: memcg that is interested in vmpressure notifications + * @eventfd: eventfd context to link notifications with + * @args: event arguments (pressure level threshold, optional mode) + * + * This function associates eventfd context with the vmpressure + * infrastructure, so that the notifications will be delivered to the + * @eventfd. The @args parameter is a comma-delimited string that denotes a + * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", + * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. + * "hierarchy" or "local"). + * + * To be used as memcg event method. + * + * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could + * not be parsed. + */ +int vmpressure_register_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd, const char *args) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; + enum vmpressure_levels level; + char *spec, *spec_orig; + char *token; + int ret = 0; + + spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); + if (!spec) + return -ENOMEM; + + /* Find required level */ + token = strsep(&spec, ","); + ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); + if (ret < 0) + goto out; + level = ret; + + /* Find optional mode */ + token = strsep(&spec, ","); + if (token) { + ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); + if (ret < 0) + goto out; + mode = ret; + } + + ev = kzalloc_obj(*ev); + if (!ev) { + ret = -ENOMEM; + goto out; + } + + ev->efd = eventfd; + ev->level = level; + ev->mode = mode; + + mutex_lock(&vmpr->events_lock); + list_add(&ev->node, &vmpr->events); + mutex_unlock(&vmpr->events_lock); + ret = 0; +out: + kfree(spec_orig); + return ret; +} + +/** + * vmpressure_unregister_event() - Unbind eventfd from vmpressure + * @memcg: memcg handle + * @eventfd: eventfd context that was used to link vmpressure with the @cg + * + * This function does internal manipulations to detach the @eventfd from + * the vmpressure notifications, and then frees internal resources + * associated with the @eventfd (but the @eventfd itself is not freed). + * + * To be used as memcg event method. + */ +void vmpressure_unregister_event(struct mem_cgroup *memcg, + struct eventfd_ctx *eventfd) +{ + struct vmpressure *vmpr = memcg_to_vmpressure(memcg); + struct vmpressure_event *ev; + + mutex_lock(&vmpr->events_lock); + list_for_each_entry(ev, &vmpr->events, node) { + if (ev->efd != eventfd) + continue; + list_del(&ev->node); + kfree(ev); + break; + } + mutex_unlock(&vmpr->events_lock); +} + static DEFINE_MUTEX(memcg_max_mutex); static int mem_cgroup_resize_max(struct mem_cgroup *memcg, @@ -1513,6 +1805,10 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, if (!ret) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -1551,6 +1847,10 @@ static int mem_cgroup_force_empty(struct mem_cgroup *memcg) if (signal_pending(current)) return -EINTR; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL)) nr_retries--; diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index f92f81108d5e..d3ed5b93290f 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -17,14 +17,8 @@ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) -#define for_each_mem_cgroup(iter) \ - for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(NULL, iter, NULL)) - void drain_all_stock(struct mem_cgroup *root_memcg); -unsigned long memcg_events(struct mem_cgroup *memcg, int event); int memory_stat_show(struct seq_file *m, void *v); struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 56cd4af08232..938f190a98fe 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2039,7 +2039,7 @@ struct obj_stock_pcp { /* * On rare archs with 256KiB base page size (hexagon and powerpc 44x) * keep nr_bytes to unsigned int as uint16_t cannot represent the full -e patches/memcg-uint16_t-for-nr_bytes-in-obj_stock_pcp.patch * sub-page remainder. Such archs are not cacheline optimization target. + * sub-page remainder. Such archs are not cacheline optimization targets. */ unsigned int nr_bytes[NR_OBJ_STOCK]; #else @@ -3535,7 +3535,8 @@ static inline size_t obj_full_size(struct kmem_cache *s) } bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, - gfp_t flags, size_t size, void **p) + gfp_t flags, unsigned int slab_alloc_flags, + size_t size, void **p) { size_t obj_size = obj_full_size(s); struct obj_cgroup *objcg; @@ -3583,7 +3584,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, slab = virt_to_slab(p[i]); if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, false)) { + alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) { continue; } @@ -4216,7 +4217,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) /* * A memcg must be visible for expand_shrinker_info() * by the time the maps are allocated. So, we allocate maps - * here, when for_each_mem_cgroup() can't skip it. + * here, when mem_cgroup_iter() can't skip it. */ if (alloc_shrinker_info(memcg)) goto offline_kmem; @@ -4793,6 +4794,10 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; @@ -4844,6 +4849,10 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, if (signal_pending(current)) break; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg_is_dying(memcg)) + break; + if (!drained) { drain_all_stock(memcg); drained = true; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51508a55c405..4916ab145325 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -74,9 +74,11 @@ static int sysctl_memory_failure_recovery __read_mostly = 1; static int sysctl_enable_soft_offline __read_mostly = 1; +static int sysctl_panic_on_unrecoverable_mf __read_mostly; + atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); -static bool hw_memory_failure __read_mostly = false; +static bool hw_memory_failure __read_mostly; static DEFINE_MUTEX(mf_mutex); @@ -155,6 +157,15 @@ static const struct ctl_table memory_failure_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, + }, + { + .procname = "panic_on_unrecoverable_memory_failure", + .data = &sysctl_panic_on_unrecoverable_mf, + .maxlen = sizeof(sysctl_panic_on_unrecoverable_mf), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, } }; @@ -980,17 +991,6 @@ static bool has_extra_refcount(struct page_state *ps, struct page *p, return false; } -/* - * Error hit kernel page. - * Do nothing, try to be lucky and not touch this instead. For a few cases we - * could be more sophisticated. - */ -static int me_kernel(struct page_state *ps, struct page *p) -{ - unlock_page(p); - return MF_IGNORED; -} - /* * Page in unknown state. Do nothing. * This is a catch-all in case we fail to make sense of the page state. @@ -1199,10 +1199,8 @@ static int me_huge_page(struct page_state *ps, struct page *p) #define mlock (1UL << PG_mlocked) #define lru (1UL << PG_lru) #define head (1UL << PG_head) -#define reserved (1UL << PG_reserved) static struct page_state error_states[] = { - { reserved, reserved, MF_MSG_KERNEL, me_kernel }, /* * free pages are specially detected outside this table: * PG_buddy pages only make a small fraction of all free pages. @@ -1234,7 +1232,6 @@ static struct page_state error_states[] = { #undef mlock #undef lru #undef head -#undef reserved static void update_per_node_mf_stats(unsigned long pfn, enum mf_result result) @@ -1269,6 +1266,15 @@ static void update_per_node_mf_stats(unsigned long pfn, ++mf_stats->total; } +static bool panic_on_unrecoverable_mf(enum mf_action_page_type type, + enum mf_result result) +{ + if (!sysctl_panic_on_unrecoverable_mf) + return false; + + return type == MF_MSG_KERNEL && result == MF_IGNORED; +} + /* * "Dirty/Clean" indication is not 100% accurate due to the possibility of * setting PG_dirty outside page lock. See also comment above set_page_dirty(). @@ -1286,6 +1292,9 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type, pr_err("%#lx: recovery action for %s: %s\n", pfn, action_page_types[type], action_name[result]); + if (panic_on_unrecoverable_mf(type, result)) + panic("Memory failure: %#lx: unrecoverable page", pfn); + return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; } @@ -1339,6 +1348,38 @@ static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) return PageLRU(page) || is_free_buddy_page(page); } +/* + * Positive identification of pages the hwpoison handler cannot recover: + * pages owned by kernel internals with no userspace mapping to unmap, no + * file mapping to invalidate, and no migration target. + */ +static inline bool is_kernel_owned_page(struct page *page) +{ + struct page *head; + bool kernel_owned; + + /* PG_reserved is a per-page flag, never set on a compound page. */ + if (PageReserved(page)) + return true; + + /* + * Page-type bits live only on the head page, so resolve any tail + * first. The check takes no refcount; recheck the head afterwards + * so a concurrent split or compound free cannot leave us trusting + * a stale view. A residual free->alloc->free cannot be closed here + * (frozen slab and large-kmalloc pages cannot be pinned), but is + * harmless: where a wrong verdict could panic, memory_failure() has + * already set PageHWPoison, which bars the page from the allocator. + */ +retry: + head = compound_head(page); + kernel_owned = PageSlab(head) || PageTable(head) || + PageLargeKmalloc(head); + if (head != compound_head(page)) + goto retry; + return kernel_owned; +} + static int __get_hwpoison_page(struct page *page, unsigned long flags) { struct folio *folio = page_folio(page); @@ -1385,6 +1426,19 @@ static int get_any_page(struct page *p, unsigned long flags) if (flags & MF_COUNT_INCREASED) count_increased = true; + /* + * Page types we know are kernel-owned and cannot be recovered. + * Short-circuit before the shake_page() / retry loop, which + * cannot turn any of these into something HWPoisonHandlable(). + * Drop the caller's reference if MF_COUNT_INCREASED took one. + */ + if (is_kernel_owned_page(p)) { + if (count_increased) + put_page(p); + ret = -ENOTRECOVERABLE; + goto out; + } + try_again: if (!count_increased) { ret = __get_hwpoison_page(p, flags); @@ -1432,7 +1486,7 @@ static int get_any_page(struct page *p, unsigned long flags) ret = -EIO; } out: - if (ret == -EIO) + if (ret == -EIO || ret == -ENOTRECOVERABLE) pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); return ret; @@ -1489,7 +1543,10 @@ static int __get_unpoison_page(struct page *page) * -EIO for pages on which we can not handle memory errors, * -EBUSY when get_hwpoison_page() has raced with page lifecycle * operations like allocation and free, - * -EHWPOISON when the page is hwpoisoned and taken off from buddy. + * -EHWPOISON when the page is hwpoisoned and taken off from buddy, + * -ENOTRECOVERABLE for kernel-owned pages identified by + * is_kernel_owned_page() (PG_reserved, slab, + * page-table, large-kmalloc) that the handler cannot recover. */ static int get_hwpoison_page(struct page *p, unsigned long flags) { @@ -2402,7 +2459,8 @@ int memory_failure(unsigned long pfn, int flags) * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. */ res = get_hwpoison_page(p, flags); - if (!res) { + switch (res) { + case 0: if (is_free_buddy_page(p)) { if (take_page_off_buddy(p)) { page_ref_inc(p); @@ -2421,7 +2479,19 @@ int memory_failure(unsigned long pfn, int flags) res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); } goto unlock_mutex; - } else if (res < 0) { + case 1: + /* Got a refcount on a handlable page. */ + break; + case -ENOTRECOVERABLE: + /* + * Stable unhandlable kernel-owned page (PG_reserved, + * slab, page tables, large-kmalloc). + * No recovery possible. + */ + res = action_result(pfn, MF_MSG_KERNEL, MF_IGNORED); + goto unlock_mutex; + default: + /* Transient lifecycle race with the page allocator. */ res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); goto unlock_mutex; } diff --git a/mm/memory.c b/mm/memory.c index 56be920c56d7..6637c5b13c9b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -953,14 +953,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (swap_dup_entry_direct(entry) < 0) return -EIO; - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(dst_mm); /* Mark the swap entry as shared. */ if (pte_swp_exclusive(orig_pte)) { pte = pte_swp_clear_exclusive(orig_pte); @@ -1797,7 +1790,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } - clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm); + clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr); *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); return nr; @@ -4386,7 +4379,7 @@ void unmap_mapping_folio(struct folio *folio) details.zap_flags = ZAP_FLAG_DROP_MARKER; i_mmap_lock_read(mapping); - if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))) + if (unlikely(mapping_mapped(mapping))) unmap_mapping_range_tree(&mapping->i_mmap, first_index, last_index, &details); i_mmap_unlock_read(mapping); @@ -4416,7 +4409,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, last_index = ULONG_MAX; i_mmap_lock_read(mapping); - if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))) + if (unlikely(mapping_mapped(mapping))) unmap_mapping_range_tree(&mapping->i_mmap, first_index, last_index, &details); i_mmap_unlock_read(mapping); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 7ac19fab2263..8b137328dcf0 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1263,7 +1263,8 @@ static pg_data_t *hotadd_init_pgdat(int nid) pgdat = NODE_DATA(nid); /* init node's zones as empty zones, we don't have any present pages.*/ - free_area_init_core_hotplug(pgdat); + if (free_area_init_core_hotplug(pgdat)) + return NULL; /* * The node we allocated has no zone fallback lists. For avoiding diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 36699fabd3c2..bba65898aee1 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -2057,24 +2057,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) diff --git a/mm/mempool.c b/mm/mempool.c index 473a029fa31f..cb74e718b2c6 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -16,11 +16,28 @@ #include #include #include +#include +#include #include "slab.h" static DECLARE_FAULT_ATTR(fail_mempool_alloc); static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk); +/* + * Debugging support for mempool using static key. + * + * This allows enabling mempool debug at boot time via: + * mempool_debug + */ +static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled); + +static int __init mempool_debug_setup(char *str) +{ + static_branch_enable(&mempool_debug_enabled); + return 1; +} +__setup("mempool_debug", mempool_debug_setup); + static int __init mempool_faul_inject_init(void) { int error; @@ -37,7 +54,6 @@ static int __init mempool_faul_inject_init(void) } late_initcall(mempool_faul_inject_init); -#ifdef CONFIG_SLUB_DEBUG_ON static void poison_error(struct mempool *pool, void *element, size_t size, size_t byte) { @@ -140,14 +156,6 @@ static void poison_element(struct mempool *pool, void *element) #endif } } -#else /* CONFIG_SLUB_DEBUG_ON */ -static inline void check_element(struct mempool *pool, void *element) -{ -} -static inline void poison_element(struct mempool *pool, void *element) -{ -} -#endif /* CONFIG_SLUB_DEBUG_ON */ static __always_inline bool kasan_poison_element(struct mempool *pool, void *element) @@ -175,7 +183,10 @@ static void kasan_unpoison_element(struct mempool *pool, void *element) static __always_inline void add_element(struct mempool *pool, void *element) { BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr); - poison_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + poison_element(pool, element); + if (kasan_poison_element(pool, element)) pool->elements[pool->curr_nr++] = element; } @@ -186,7 +197,9 @@ static void *remove_element(struct mempool *pool) BUG_ON(pool->curr_nr < 0); kasan_unpoison_element(pool, element); - check_element(pool, element); + + if (static_branch_unlikely(&mempool_debug_enabled)) + check_element(pool, element); return element; } diff --git a/mm/memremap.c b/mm/memremap.c index 81766d822400..accba23aef28 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -425,6 +425,7 @@ void free_zone_device_folio(struct folio *folio) mem_cgroup_uncharge(folio); if (folio_test_anon(folio)) { + mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, -1); for (i = 0; i < nr; i++) __ClearPageAnonExclusive(folio_page(folio, i)); } diff --git a/mm/migrate.c b/mm/migrate.c index d9b23909d716..a786549551e3 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -362,7 +362,7 @@ static bool remove_migration_pte(struct folio *folio, idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff; new = folio_page(folio, idx); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /* PMD-mapped THP migration entry */ if (!pvmw.pte) { VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) || @@ -545,7 +545,7 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, p } #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) { spinlock_t *ptl; @@ -590,7 +590,8 @@ static int __folio_migrate_mapping(struct address_space *mapping, /* No turning back from here */ newfolio->index = folio->index; newfolio->mapping = folio->mapping; - if (folio_test_anon(folio) && folio_test_large(folio)) + if (folio_test_anon(folio) && folio_test_large(folio) && + !folio_test_hugetlb(folio)) mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); if (folio_test_swapbacked(folio)) __folio_set_swapbacked(newfolio); @@ -623,7 +624,8 @@ static int __folio_migrate_mapping(struct address_space *mapping, */ newfolio->index = folio->index; newfolio->mapping = folio->mapping; - if (folio_test_anon(folio) && folio_test_large(folio)) + if (folio_test_anon(folio) && folio_test_large(folio) && + !folio_test_hugetlb(folio)) mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); folio_ref_add(newfolio, nr); /* add cache reference */ if (folio_test_swapbacked(folio)) @@ -884,7 +886,7 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst, * @mapping: The address_space containing the folio. * @dst: The folio to migrate the data to. * @src: The folio containing the current data. - * @mode: How to migrate the page. + * @mode: How to migrate the folio. * * Common logic to directly migrate a single LRU folio suitable for * folios that do not have private data. @@ -1130,7 +1132,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src, } /* - * To record some information during migration, we use unused private + * To record some information during migration, we use the migrate_info * field of struct folio of the newly allocated destination folio. * This is safe because nobody is using it except us. */ @@ -1143,27 +1145,24 @@ enum { static void __migrate_folio_record(struct folio *dst, int old_folio_state, struct anon_vma *anon_vma) { - dst->private = (void *)anon_vma + old_folio_state; + dst->migrate_info = (unsigned long)anon_vma | old_folio_state; } static void __migrate_folio_extract(struct folio *dst, int *old_folio_state, struct anon_vma **anon_vmap) { - unsigned long private = (unsigned long)dst->private; + unsigned long info = dst->migrate_info; - *anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES); - *old_folio_state = private & FOLIO_OLD_STATES; - dst->private = NULL; + *anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES); + *old_folio_state = info & FOLIO_OLD_STATES; + dst->migrate_info = 0; } /* Restore the source folio to the original state upon failure */ -static void migrate_folio_undo_src(struct folio *src, - int page_was_mapped, - struct anon_vma *anon_vma, - bool locked, - struct list_head *ret) +static void migrate_folio_undo_src(struct folio *src, int was_mapped, + struct anon_vma *anon_vma, bool locked, struct list_head *ret) { - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, src, 0); /* Drop an anon_vma reference if we took one */ if (anon_vma) @@ -1217,7 +1216,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return -ENOMEM; *dstp = dst; - dst->private = NULL; + dst->migrate_info = 0; if (!folio_trylock(src)) { if (mode == MIGRATE_ASYNC) @@ -1449,7 +1448,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, } /* - * Counterpart of unmap_and_move_page() for hugepage migration. + * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb + * folio migration. * * This function doesn't wait the completion of hugepage I/O * because there is no race between I/O and migration for hugepage. @@ -1466,20 +1466,20 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, * because then pte is replaced with migration swap entry and direct I/O code * will wait in the page fault for migration to complete. */ -static int unmap_and_move_huge_page(new_folio_t get_new_folio, +static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, struct folio *src, int force, enum migrate_mode mode, - int reason, struct list_head *ret) + enum migrate_reason reason, struct list_head *ret) { struct folio *dst; int rc = -EAGAIN; - int page_was_mapped = 0; + int was_mapped = 0; struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; enum ttu_flags ttu = 0; if (folio_ref_count(src) == 1) { - /* page was freed from under us. So we are done. */ + /* folio was freed from under us. So we are done. */ folio_putback_hugetlb(src); return 0; } @@ -1501,8 +1501,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } /* - * Check for pages which are in the process of being freed. Without - * folio_mapping() set, hugetlbfs specific move page routine will not + * Check for folios which are in the process of being freed. Without + * folio_mapping() set, hugetlbfs specific move folio routine will not * be called and we could leak usage counts for subpools. */ if (hugetlb_folio_subpool(src) && !folio_mapping(src)) { @@ -1532,13 +1532,13 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, } try_to_migrate(src, ttu); - page_was_mapped = 1; + was_mapped = 1; } if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); - if (page_was_mapped) + if (was_mapped) remove_migration_ptes(src, !rc ? dst : src, ttu); if (ttu & TTU_RMAP_LOCKED) @@ -1626,7 +1626,7 @@ struct migrate_pages_stats { */ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct migrate_pages_stats *stats, struct list_head *ret_folios) { @@ -1663,10 +1663,10 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, continue; } - rc = unmap_and_move_huge_page(get_new_folio, - put_new_folio, private, - folio, pass > 2, mode, - reason, ret_folios); + rc = unmap_and_move_hugetlb_folio(get_new_folio, + put_new_folio, private, + folio, pass > 2, mode, + reason, ret_folios); /* * The rules are: * 0: hugetlb folio will be put back @@ -1716,7 +1716,7 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio, static void migrate_folios_move(struct list_head *src_folios, struct list_head *dst_folios, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct migrate_pages_stats *stats, int *retry, int *thp_retry, int *nr_failed, @@ -1741,7 +1741,7 @@ static void migrate_folios_move(struct list_head *src_folios, /* * The rules are: * 0: folio will be freed - * -EAGAIN: stay on the unmap_folios list + * -EAGAIN: stay on the src_folios list * Other errno: put on ret_folios list */ switch (rc) { @@ -1799,7 +1799,7 @@ static void migrate_folios_undo(struct list_head *src_folios, */ static int migrate_pages_batch(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, - unsigned long private, enum migrate_mode mode, int reason, + unsigned long private, enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats, int nr_pass) { @@ -2011,7 +2011,7 @@ static int migrate_pages_batch(struct list_head *from, static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, + enum migrate_mode mode, enum migrate_reason reason, struct list_head *ret_folios, struct list_head *split_folios, struct migrate_pages_stats *stats) { @@ -2088,7 +2088,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio, */ int migrate_pages(struct list_head *from, new_folio_t get_new_folio, free_folio_t put_new_folio, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded) { int rc, rc_gather; int nr_pages; diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 554754eb26ff..052167f9ad54 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -77,6 +77,9 @@ static int migrate_vma_collect_hole(unsigned long start, * @folio: the folio to split * @fault_page: struct page associated with the fault if any * + * If @folio is not the folio containing @fault_page, the caller must hold a + * reference on @folio. The helper consumes that reference. + * * Returns 0 on success */ static int migrate_vma_split_folio(struct folio *folio, @@ -86,10 +89,8 @@ static int migrate_vma_split_folio(struct folio *folio, struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; struct folio *new_fault_folio = NULL; - if (folio != fault_folio) { - folio_get(folio); + if (folio != fault_folio) folio_lock(folio); - } ret = split_folio(folio); if (ret) { @@ -166,11 +167,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, } else if (!pmd_present(*pmdp)) { const softleaf_t entry = softleaf_from_pmd(*pmdp); - folio = softleaf_to_folio(entry); - if (!softleaf_is_device_private(entry) || - !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) || - (folio->pgmap->owner != migrate->pgmap_owner)) { + !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { + spin_unlock(ptl); + return migrate_vma_collect_skip(start, end, walk); + } + + folio = softleaf_to_folio(entry); + if (folio->pgmap->owner != migrate->pgmap_owner) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } @@ -307,6 +311,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -350,6 +357,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (folio && folio_test_large(folio)) { int ret; + /* migrate_vma_split_folio() consumes this reference */ + if (folio != fault_folio) + folio_get(folio); lazy_mmu_mode_disable(); pte_unmap_unlock(ptep, ptl); ret = migrate_vma_split_folio(folio, @@ -513,7 +523,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate) migrate->pgmap_owner); mmu_notifier_invalidate_range_start(&range); - walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end, + walk_page_range_vma(migrate->vma, migrate->start, migrate->end, &migrate_vma_walk_ops, migrate); mmu_notifier_invalidate_range_end(&range); @@ -768,7 +778,7 @@ int migrate_vma_setup(struct migrate_vma *args) } EXPORT_SYMBOL(migrate_vma_setup); -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF /** * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm * at @addr. folio is already allocated as a part of the migration process with @@ -835,7 +845,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, else swp_entry = make_readable_device_private_entry( page_to_pfn(page)); - entry = swp_entry_to_pmd(swp_entry); + entry = softleaf_to_pmd(swp_entry); } else { if (folio_is_zone_device(folio) && !folio_is_device_coherent(folio)) { @@ -923,7 +933,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, migrate->src[i+idx] = migrate_pfn(pfn + i) | flags; return ret; } -#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* !CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF */ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, unsigned long addr, struct page *page, @@ -944,7 +954,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate, static unsigned long migrate_vma_nr_pages(unsigned long *src) { unsigned long nr = 1; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF if (*src & MIGRATE_PFN_COMPOUND) nr = HPAGE_PMD_NR; #else diff --git a/mm/mincore.c b/mm/mincore.c index 296f2e3922b5..53b982803771 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -227,8 +227,7 @@ static inline bool can_do_mincore(struct vm_area_struct *vma) * for writing; otherwise we'd be including shared non-exclusive * mappings, which opens a side channel. */ - return inode_owner_or_capable(&nop_mnt_idmap, - file_inode(vma->vm_file)) || + return file_owner_or_capable(vma->vm_file) || file_permission(vma->vm_file, MAY_WRITE) == 0; } @@ -259,7 +258,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v memset(vec, 1, pages); return pages; } - err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); + + /* + * mincore historically reports PFNMAP mappings as non-resident. + */ + if (vma->vm_flags & VM_PFNMAP) { + __mincore_unmapped_range(addr, end, vma, vec); + return (end - addr) >> PAGE_SHIFT; + } + + err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; return (end - addr) >> PAGE_SHIFT; diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df..97e49038d8d3 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -446,7 +446,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma, vma_flags_reset_once(vma, new_vma_flags); lru_add_drain(); - walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL); + walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL); lru_add_drain(); if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) { diff --git a/mm/mm_init.c b/mm/mm_init.c index 65623f95bec3..bb05ec756e06 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -332,8 +332,7 @@ static unsigned long __init early_calculate_totalpages(void) unsigned long pages = end_pfn - start_pfn; totalpages += pages; - if (pages) - node_set_state(nid, N_MEMORY); + node_set_state(nid, N_MEMORY); } return totalpages; } @@ -674,43 +673,20 @@ static inline void fixup_hashdist(void) static inline void fixup_hashdist(void) {} #endif /* CONFIG_NUMA */ -#ifdef CONFIG_ZONE_DEVICE +#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) static __meminit void pageblock_migratetype_init_range(unsigned long pfn, - unsigned long nr_pages, int migratetype) + unsigned long nr_pages, int migratetype, bool atomic) { const unsigned long end = pfn + nr_pages; for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) { init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false); - if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) + if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); } } #endif -/* - * Initialize a reserved page unconditionally, finding its zone first. - */ -void __meminit __init_page_from_nid(unsigned long pfn, int nid) -{ - pg_data_t *pgdat; - int zid; - - pgdat = NODE_DATA(nid); - - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE, - false); -} - #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static inline void pgdat_set_deferred_range(pg_data_t *pgdat) { @@ -769,10 +745,25 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) static void __meminit __init_deferred_page(unsigned long pfn, int nid) { + pg_data_t *pgdat = NODE_DATA(nid); + int zid; + if (early_page_initialised(pfn, nid)) return; - __init_page_from_nid(pfn, nid); + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (zone_spans_pfn(zone, pfn)) + break; + } + __init_single_page(pfn_to_page(pfn), pfn, zid, nid); + + if (pageblock_aligned(pfn)) { + enum migratetype mt = + kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + } } #else static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} @@ -797,28 +788,6 @@ void __meminit init_deferred_page(unsigned long pfn, int nid) __init_deferred_page(pfn, nid); } -/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ -static bool __meminit -overlap_memmap_init(unsigned long zone, unsigned long *pfn) -{ - static struct memblock_region *r __meminitdata; - - if (mirrored_kernelcore && zone == ZONE_MOVABLE) { - if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { - for_each_mem_region(r) { - if (*pfn < memblock_region_memory_end_pfn(r)) - break; - } - } - if (*pfn >= memblock_region_memory_base_pfn(r) && - memblock_is_mirror(r)) { - *pfn = memblock_region_memory_end_pfn(r); - return true; - } - } - return false; -} - /* * Only struct pages that correspond to ranges defined by memblock.memory * are zeroed and initialized by going through __init_single_page() during @@ -905,8 +874,6 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone * function. They do not exist on hotplugged memory. */ if (context == MEMINIT_EARLY) { - if (overlap_memmap_init(zone, &pfn)) - continue; if (defer_init(nid, pfn, zone_end_pfn)) { deferred_struct_pages = true; break; @@ -941,7 +908,8 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone static void __init memmap_init_zone_range(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn, - unsigned long *hole_pfn) + unsigned long *hole_pfn, + enum migratetype mt) { unsigned long zone_start_pfn = zone->zone_start_pfn; unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; @@ -954,8 +922,7 @@ static void __init memmap_init_zone_range(struct zone *zone, return; memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, - zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, - false); + zone_end_pfn, MEMINIT_EARLY, NULL, mt, false); if (*hole_pfn < start_pfn) init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); @@ -971,6 +938,8 @@ static void __init memmap_init(void) for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { struct pglist_data *node = NODE_DATA(nid); + enum migratetype mt = + kho_scratch_migratetype(start_pfn, MIGRATE_MOVABLE); for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = node->node_zones + j; @@ -979,7 +948,7 @@ static void __init memmap_init(void) continue; memmap_init_zone_range(zone, start_pfn, end_pfn, - &hole_pfn); + &hole_pfn, mt); zone_id = j; } } @@ -1138,7 +1107,7 @@ void __ref memmap_init_zone_device(struct zone *zone, compound_nr_pages(pfn, altmap, pgmap)); } - pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); + pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false); pr_debug("%s initialised %lu pages in %ums\n", __func__, nr_pages, jiffies_to_msecs(jiffies - start)); @@ -1170,9 +1139,8 @@ static void __init adjust_zone_range_for_zone_movable(int nid, arch_zone_highest_possible_pfn[movable_zone]); /* Adjust for ZONE_MOVABLE starting within this range */ - } else if (!mirrored_kernelcore && - *zone_start_pfn < zone_movable_pfn[nid] && - *zone_end_pfn > zone_movable_pfn[nid]) { + } else if (*zone_start_pfn < zone_movable_pfn[nid] && + *zone_end_pfn > zone_movable_pfn[nid]) { *zone_end_pfn = zone_movable_pfn[nid]; /* Check if this whole range is within ZONE_MOVABLE */ @@ -1220,40 +1188,11 @@ static unsigned long __init zone_absent_pages_in_node(int nid, unsigned long zone_start_pfn, unsigned long zone_end_pfn) { - unsigned long nr_absent; - /* zone is empty, we don't have any absent pages */ if (zone_start_pfn == zone_end_pfn) return 0; - nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); - - /* - * ZONE_MOVABLE handling. - * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages - * and vice versa. - */ - if (mirrored_kernelcore && zone_movable_pfn[nid]) { - unsigned long start_pfn, end_pfn; - struct memblock_region *r; - - for_each_mem_region(r) { - start_pfn = clamp(memblock_region_memory_base_pfn(r), - zone_start_pfn, zone_end_pfn); - end_pfn = clamp(memblock_region_memory_end_pfn(r), - zone_start_pfn, zone_end_pfn); - - if (zone_type == ZONE_MOVABLE && - memblock_is_mirror(r)) - nr_absent += end_pfn - start_pfn; - - if (zone_type == ZONE_NORMAL && - !memblock_is_mirror(r)) - nr_absent += end_pfn - start_pfn; - } - } - - return nr_absent; + return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); } /* @@ -1334,7 +1273,7 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) { - unsigned long realtotalpages = 0, totalpages = 0; + unsigned long realtotalpages = 0; enum zone_type i; for (i = 0; i < MAX_NR_ZONES; i++) { @@ -1364,11 +1303,10 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, zone->present_early_pages = real_size; #endif - totalpages += spanned; realtotalpages += real_size; } - pgdat->node_spanned_pages = totalpages; + pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; pgdat->node_present_pages = realtotalpages; pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); } @@ -1492,7 +1430,7 @@ static inline void setup_usemap(struct zone *zone) {} #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) +static void __init set_pageblock_order(void) { unsigned int order = PAGE_BLOCK_MAX_ORDER; @@ -1518,7 +1456,7 @@ void __init set_pageblock_order(void) * include/linux/pageblock-flags.h for the values of pageblock_order based on * the kernel config */ -void __init set_pageblock_order(void) +static inline void __init set_pageblock_order(void) { } @@ -1532,16 +1470,22 @@ void __init set_pageblock_order(void) * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { int nid = pgdat->node_id; enum zone_type z; - int cpu; + int cpu, i; pgdat_init_internals(pgdat); - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + if (pgdat->per_cpu_nodestats == &boot_nodestats) { + struct per_cpu_nodestat __percpu *p; + + p = alloc_percpu(struct per_cpu_nodestat); + if (!p) + return -ENOMEM; + pgdat->per_cpu_nodestats = p; + } /* * Reset the nr_zones, order and highest_zoneidx before reuse. @@ -1554,10 +1498,17 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) pgdat->node_start_pfn = 0; pgdat->node_present_pages = 0; - for_each_online_cpu(cpu) { - struct per_cpu_nodestat *p; + /* + * Hot-unplug can leave per-cpu vmstat deltas unfolded (folders skip + * offline nodes) - reconcile this at online. Foreign access to counters + * is safe: the node is not online yet and we hold the hotplug lock. + */ + for_each_possible_cpu(cpu) { + struct per_cpu_nodestat *p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); - p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) + if (p->vm_node_stat_diff[i]) + node_page_state_add(p->vm_node_stat_diff[i], pgdat, i); memset(p, 0, sizeof(*p)); } @@ -1572,6 +1523,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) zone->present_pages = 0; zone_init_internals(zone, z, nid, 0); } + + return 0; } #endif @@ -1810,7 +1763,6 @@ static void __init free_area_init(void) bool descending; arch_zone_limits_init(max_zone_pfn); - sparse_init(); start_pfn = PHYS_PFN(memblock_start_of_DRAM()); descending = arch_has_descending_max_zone_pfns(); @@ -1860,18 +1812,12 @@ static void __init free_area_init(void) (u64)zone_movable_pfn[i] << PAGE_SHIFT); } - /* - * Print out the early node map, and initialize the - * subsection-map relative to active online memory ranges to - * enable future "sub-section" extensions of the memory map. - */ + /* Print out the early node map. */ pr_info("Early memory node ranges\n"); - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); - sparse_init_subsection_map(start_pfn, end_pfn - start_pfn); - } /* Initialise every node */ mminit_verify_pageflags_layout(); @@ -1905,11 +1851,7 @@ static void __init free_area_init(void) } } - for_each_node_state(nid, N_MEMORY) - sparse_vmemmap_init_nid_late(nid); - calc_nr_kernel_pages(); - memmap_init(); /* disable hash distribution for systems with a single node */ fixup_hashdist(); @@ -1969,7 +1911,7 @@ unsigned long __init node_map_pfn_alignment(void) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static void __init deferred_free_pages(unsigned long pfn, - unsigned long nr_pages) + unsigned long nr_pages, enum migratetype mt) { struct page *page; unsigned long i; @@ -1977,13 +1919,12 @@ static void __init deferred_free_pages(unsigned long pfn, if (!nr_pages) return; + pageblock_migratetype_init_range(pfn, nr_pages, mt, true); + page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { - for (i = 0; i < nr_pages; i += pageblock_nr_pages) - init_pageblock_migratetype(page + i, MIGRATE_MOVABLE, - false); __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); return; } @@ -1991,12 +1932,8 @@ static void __init deferred_free_pages(unsigned long pfn, /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); - for (i = 0; i < nr_pages; i++, page++, pfn++) { - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(page, MIGRATE_MOVABLE, - false); - __free_pages_core(page, 0, MEMINIT_EARLY); - } + for (i = 0; i < nr_pages; i++) + __free_pages_core(page + i, 0, MEMINIT_EARLY); } /* Completion tracking for deferred_init_memmap() threads */ @@ -2052,6 +1989,8 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { unsigned long spfn = PFN_UP(start); unsigned long epfn = PFN_DOWN(end); + enum migratetype mt = + kho_scratch_migratetype(spfn, MIGRATE_MOVABLE); if (spfn >= end_pfn) break; @@ -2064,7 +2003,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, unsigned long chunk_end = min(mo_pfn, epfn); nr_pages += deferred_init_pages(zone, spfn, chunk_end); - deferred_free_pages(spfn, chunk_end - spfn); + deferred_free_pages(spfn, chunk_end - spfn, mt); spfn = chunk_end; @@ -2319,6 +2258,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); @@ -2683,10 +2623,13 @@ void __init __weak mem_init(void) void __init mm_core_init_early(void) { + free_area_init(); + hugetlb_cma_reserve(); hugetlb_bootmem_alloc(); - free_area_init(); + sparse_init(); + memmap_init(); } /* diff --git a/mm/mmzone.c b/mm/mmzone.c index 0c8f181d9d50..59dc3f2076a6 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -43,7 +43,8 @@ struct zone *next_zone(struct zone *zone) return zone; } -static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) +static inline int zref_in_nodemask(struct zoneref *zref, + const nodemask_t *nodes) { #ifdef CONFIG_NUMA return node_isset(zonelist_node_idx(zref), *nodes); @@ -55,7 +56,7 @@ static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes) /* Returns the next zone at or below highest_zoneidx in a zonelist */ struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { /* * Find the next suitable zone to use for the allocation. diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028c..8665a23f38d3 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -143,7 +143,7 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce * that the ptes point to consecutive pages of the same anon large folio. */ -static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len, +static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len, struct page *first_page, bool expected_anon_exclusive) { int idx; @@ -174,16 +174,16 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb) { bool expected_anon_exclusive; - int sub_batch_idx = 0; + int batch_idx = 0; int len; while (nr_ptes) { - expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); - len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, + expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx); + len = page_anon_exclusive_batch(batch_idx, nr_ptes, first_page, expected_anon_exclusive); prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, - sub_batch_idx, expected_anon_exclusive, tlb); - sub_batch_idx += len; + batch_idx, expected_anon_exclusive, tlb); + batch_idx += len; nr_ptes -= len; } } @@ -708,16 +708,9 @@ static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, 0 : -EACCES; } -static int prot_none_test(unsigned long addr, unsigned long next, - struct mm_walk *walk) -{ - return 0; -} - static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, - .test_walk = prot_none_test, .walk_lock = PGWALK_WRLOCK, }; @@ -753,7 +746,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) { pgprot_t new_pgprot = vm_get_page_prot(newflags); - error = walk_page_range(current->mm, start, end, + error = walk_page_range_vma(vma, start, end, &prot_none_walk_ops, &new_pgprot); if (error) return error; diff --git a/mm/numa_memblks.c b/mm/numa_memblks.c index 3c3c4eac3514..be6e7346f6c7 100644 --- a/mm/numa_memblks.c +++ b/mm/numa_memblks.c @@ -17,20 +17,6 @@ nodemask_t numa_nodes_parsed __initdata; static struct numa_meminfo numa_meminfo __initdata_or_meminfo; static struct numa_meminfo numa_reserved_meminfo __initdata_or_meminfo; -/* - * Set nodes, which have memory in @mi, in *@nodemask. - */ -static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask, - const struct numa_meminfo *mi) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(mi->blk); i++) - if (mi->blk[i].start != mi->blk[i].end && - mi->blk[i].nid != NUMA_NO_NODE) - node_set(mi->blk[i].nid, *nodemask); -} - /** * numa_reset_distance - Reset NUMA distance table * @@ -56,7 +42,6 @@ static int __init numa_alloc_distance(void) /* size the new table and allocate it */ nodes_parsed = numa_nodes_parsed; - numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo); for_each_node_mask(i, nodes_parsed) cnt = i; @@ -135,13 +120,20 @@ EXPORT_SYMBOL(__node_distance); static int __init numa_add_memblk_to(int nid, u64 start, u64 end, struct numa_meminfo *mi) { + /* whine about and ignore invalid nid */ + if (nid < 0 || nid >= MAX_NUMNODES) { + pr_warn("Warning: invalid memblk node id %d [mem %#010Lx-%#010Lx]\n", + nid, start, end - 1); + return -EINVAL; + } + /* ignore zero length blks */ if (start == end) return 0; - /* whine about and ignore invalid blks */ - if (start > end || nid < 0 || nid >= MAX_NUMNODES) { - pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", + /* whine about and ignore invalid ranges */ + if (start > end) { + pr_warn("Warning: invalid memblk range for node %d [mem %#010Lx-%#010Lx]\n", nid, start, end - 1); return 0; } @@ -193,13 +185,20 @@ static void __init numa_move_tail_memblk(struct numa_meminfo *dst, int idx, * @end: End address of the new memblk * * Add a new memblk to the default numa_meminfo. + * On success @nid is also set in numa_nodes_parsed. * * RETURNS: * 0 on success, -errno on failure. */ int __init numa_add_memblk(int nid, u64 start, u64 end) { - return numa_add_memblk_to(nid, start, end, &numa_meminfo); + int ret; + + ret = numa_add_memblk_to(nid, start, end, &numa_meminfo); + if (!ret) + node_set(nid, numa_nodes_parsed); + + return ret; } /** @@ -256,8 +255,7 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi) /* preserve info for non-RAM areas above 'max_pfn': */ if (bi->end > high) { - numa_add_memblk_to(bi->nid, high, bi->end, - &numa_reserved_meminfo); + numa_add_reserved_memblk(bi->nid, high, bi->end); bi->end = high; } @@ -401,7 +399,6 @@ static int __init numa_register_meminfo(struct numa_meminfo *mi) /* Account for nodes with cpus and no memory */ node_possible_map = numa_nodes_parsed; - numa_nodemask_from_meminfo(&node_possible_map, mi); if (WARN_ON(nodes_empty(node_possible_map))) return -EINVAL; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f7db8f049bd2..762d9b6bc792 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -165,7 +165,9 @@ DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif +#ifdef CONFIG_NUMA DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); +#endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* @@ -1246,7 +1248,7 @@ void __clear_page_tag_ref(struct page *page) /* Should be called only if mem_alloc_profiling_enabled() */ static noinline void __pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr) + unsigned int nr, gfp_t gfp_flags) { union pgtag_ref_handle handle; union codetag_ref ref; @@ -1260,17 +1262,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task, * page_ext is not available yet, record the pfn so we can * clear the tag ref later when page_ext is initialized. */ - alloc_tag_add_early_pfn(page_to_pfn(page)); + alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags); if (task->alloc_tag) alloc_tag_set_inaccurate(task->alloc_tag); } } static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr) + unsigned int nr, gfp_t gfp_flags) { if (mem_alloc_profiling_enabled()) - __pgalloc_tag_add(page, task, nr); + __pgalloc_tag_add(page, task, nr, gfp_flags); } /* Should be called only if mem_alloc_profiling_enabled() */ @@ -1303,7 +1305,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void pgalloc_tag_add(struct page *page, struct task_struct *task, - unsigned int nr) {} + unsigned int nr, gfp_t gfp_flags) {} static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {} static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {} @@ -1858,7 +1860,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order, set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); - pgalloc_tag_add(page, current, 1 << order); + pgalloc_tag_add(page, current, 1 << order, gfp_flags); } static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, @@ -3247,10 +3249,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, } while (check_new_pages(page, order)); /* - * If this is a high-order atomic allocation then check - * if the pageblock should be reserved for the future + * Slowpath (precarious) high-atomic allocations may reserve + * a pageblock for future use. */ - if (unlikely(alloc_flags & ALLOC_HIGHATOMIC)) + if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) && + ((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN))) reserve_highatomic_pageblock(page, order, zone); __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); @@ -3739,13 +3742,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) static inline unsigned int alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) { - unsigned int alloc_flags; + unsigned int alloc_flags = 0; - /* - * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save a branch. - */ - alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM); + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; if (defrag_mode) { alloc_flags |= ALLOC_NOFRAGMENT; @@ -3979,7 +3979,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, return NULL; } -static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) +static void warn_alloc_show_mem(gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned int filter = SHOW_MEM_FILTER_NODES; @@ -3999,7 +3999,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) mem_cgroup_show_protected_memory(NULL); } -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...) { struct va_format vaf; va_list args; @@ -4474,17 +4474,32 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask, } static inline unsigned int -gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) +gfp_to_alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order) { - unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; + unsigned int alloc_flags = 0; + + if (gfp_mask & __GFP_DIRECT_RECLAIM) + return 0; /* - * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE - * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD - * to save two branches. + * Not worth trying to allocate harder for __GFP_NOMEMALLOC even + * if it can't schedule. */ - BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE); - BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD); + if (gfp_mask & __GFP_NOMEMALLOC) + return 0; + + alloc_flags |= ALLOC_NON_BLOCK; + + if (order > 0 && (gfp_mask & __GFP_HIGH)) + alloc_flags |= ALLOC_HIGHATOMIC; + + return alloc_flags; +} + +static inline unsigned int +gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) +{ + unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; /* * The caller may dip into page reserves a bit more if the caller @@ -4492,21 +4507,14 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH). */ - alloc_flags |= (__force int) - (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM)); - - if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { - /* - * Not worth trying to allocate harder for __GFP_NOMEMALLOC even - * if it can't schedule. - */ - if (!(gfp_mask & __GFP_NOMEMALLOC)) { - alloc_flags |= ALLOC_NON_BLOCK; + if (gfp_mask & __GFP_HIGH) + alloc_flags |= ALLOC_MIN_RESERVE; + if (gfp_mask & __GFP_KSWAPD_RECLAIM) + alloc_flags |= ALLOC_KSWAPD; - if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE)) - alloc_flags |= ALLOC_HIGHATOMIC; - } + alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp_mask, order); + if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) { /* * Ignore cpuset mems for non-blocking __GFP_HIGH (probably * GFP_ATOMIC) rather than fail, see the comment for @@ -4687,7 +4695,7 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac) return false; } -static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask, +static void check_alloc_stall_warn(gfp_t gfp_mask, const nodemask_t *nodemask, unsigned int order, unsigned long alloc_start_time) { static DEFINE_SPINLOCK(alloc_stall_lock); @@ -5299,6 +5307,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, * memory until all local zones are considered. */ alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + alloc_flags |= gfp_to_alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC; /* First allocation attempt */ page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac); @@ -6653,7 +6662,8 @@ static int sysctl_min_unmapped_ratio_sysctl_handler(const struct ctl_table *tabl if (rc) return rc; - setup_min_unmapped_ratio(); + if (write) + setup_min_unmapped_ratio(); return 0; } @@ -6680,7 +6690,8 @@ static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, i if (rc) return rc; - setup_min_slab_ratio(); + if (write) + setup_min_slab_ratio(); return 0; } @@ -6902,15 +6913,15 @@ static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages continue; } - if (start && memdesc_section(page->flags) != start_sec) { + if (start && memdesc_section(&page->flags) != start_sec) { free_prepared_contig_range(start, i - nr_start); start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } else if (!start) { start = page; nr_start = i; - start_sec = memdesc_section(page->flags); + start_sec = memdesc_section(&page->flags); } } diff --git a/mm/page_io.c b/mm/page_io.c index 60977c970cdf..b23f494fcc83 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -327,8 +327,13 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) rcu_read_lock(); memcg = folio_memcg(folio); css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); - bio_associate_blkg_from_css(bio, css); + if (!css || !css_tryget(css)) + css = NULL; rcu_read_unlock(); + + bio_associate_blkg_from_css(bio, css); + if (css) + css_put(css); } #else #define bio_associate_blkg_from_page(bio, folio) do { } while (0) diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 7a9d631945a3..32ce8a7d9df3 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -41,8 +41,14 @@ bool page_is_unmovable(struct zone *zone, struct page *page, * We need not scan over tail pages because we don't * handle each tail page individually in migration. */ - if (PageHuge(page) || PageCompound(page)) { + if (PageCompound(page)) { struct folio *folio = page_folio(page); + unsigned long nr_pages, pfn; + unsigned int order; + + order = compound_order(&folio->page); + if (order > MAX_FOLIO_ORDER) + return true; if (folio_test_hugetlb(folio)) { struct hstate *h; @@ -54,15 +60,16 @@ bool page_is_unmovable(struct zone *zone, struct page *page, * The huge page may be freed so can not * use folio_hstate() directly. */ - h = size_to_hstate(folio_size(folio)); - if (h && !hugepage_migration_supported(h)) + h = size_to_hstate(PAGE_SIZE << order); + if (!h || !hugepage_migration_supported(h)) return true; - - } else if (!folio_test_lru(folio)) { + } else if (!PageLRU(page)) { return true; } - *step = folio_nr_pages(folio) - folio_page_idx(folio, page); + nr_pages = 1UL << order; + pfn = page_to_pfn(page); + *step = (pfn | (nr_pages - 1)) + 1 - pfn; return false; } diff --git a/mm/page_owner.c b/mm/page_owner.c index 2dddcb6510aa..26d6ab6530ce 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -54,6 +54,25 @@ struct stack_print_ctx { u8 flags; }; +enum page_owner_print_mode { + PAGE_OWNER_PRINT_STACK, + PAGE_OWNER_PRINT_HANDLE, + PAGE_OWNER_PRINT_STACK_HANDLE, +}; + +static const char * const page_owner_print_mode_strings[] = { + [PAGE_OWNER_PRINT_STACK] = "stack", + [PAGE_OWNER_PRINT_HANDLE] = "handle", + [PAGE_OWNER_PRINT_STACK_HANDLE] = "stack_handle", +}; + +struct page_owner_filter_state { + enum page_owner_print_mode print_mode; + nodemask_t nid_filter; + bool nid_filter_enabled; + spinlock_t lock; +}; + static bool page_owner_enabled __initdata; DEFINE_STATIC_KEY_FALSE(page_owner_inited); @@ -339,13 +358,13 @@ noinline void __set_page_owner(struct page *page, unsigned short order, depot_stack_handle_t handle; handle = save_stack(gfp_mask); - __update_page_owner_handle(page, handle, order, gfp_mask, -1, + __update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER, ts_nsec, current->pid, current->tgid, current->comm); inc_stack_record_count(handle, gfp_mask, 1 << order); } -void __folio_set_owner_migrate_reason(struct folio *folio, int reason) +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { struct page_ext *page_ext = page_ext_get(&folio->page); struct page_owner *page_owner; @@ -422,6 +441,39 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) rcu_read_unlock(); } +/* + * Check if a page is a buddy page and advance @pfn past the entire buddy block. + * This safely reads the buddy order without the zone lock, which may cause us + * to skip less than the full buddy block, but that is acceptable for page owner + * iteration purposes. + * + * The lockless read of buddy_order_unsafe() can also return a garbage order if + * the page is concurrently allocated and PageBuddy is cleared between the check + * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so + * that a bogus order cannot carry @pfn into an unvalidated memory section, + * which would break callers that rely on boundary-aligned pfn_valid() checks. + * + * Return: true if the page was skipped (caller should continue its loop), + * false if the page is not a buddy page and should be processed normally. + */ +static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page) +{ + unsigned long order; + + if (!PageBuddy(page)) + return false; + + order = buddy_order_unsafe(page); + if (order <= MAX_PAGE_ORDER) { + unsigned long new_pfn = *pfn + (1UL << order); + unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES); + + *pfn = min(new_pfn, boundary) - 1; + } + + return true; +} + void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { @@ -461,14 +513,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, if (page_zone(page) != zone) continue; - if (PageBuddy(page)) { - unsigned long freepage_order; - - freepage_order = buddy_order_unsafe(page); - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -505,13 +551,13 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, seq_putc(m, '\n'); } +#ifdef CONFIG_MEMCG /* * Looking for memcg information and print it out */ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, struct page *page) { -#ifdef CONFIG_MEMCG unsigned long memcg_data; struct mem_cgroup *memcg; bool online; @@ -534,29 +580,42 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, cgroup_name(memcg->css.cgroup, name, sizeof(name)); ret += scnprintf(kbuf + ret, count - ret, "Charged %sto %smemcg %s\n", - PageMemcgKmem(page) ? "(via objcg) " : "", + (memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "", online ? "" : "offline ", name); out_unlock: rcu_read_unlock(); -#endif /* CONFIG_MEMCG */ return ret; } +#else +static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, + struct page *page) +{ + return ret; +} +#endif static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + depot_stack_handle_t handle, + struct page_owner_filter_state *state) { int ret, pageblock_mt, page_mt; char *kbuf; + enum page_owner_print_mode print_mode; + unsigned long flags; count = min_t(size_t, count, PAGE_SIZE); kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; + spin_lock_irqsave(&state->lock, flags); + print_mode = state->print_mode; + spin_unlock_irqrestore(&state->lock, flags); + ret = scnprintf(kbuf, count, "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n", page_owner->order, page_owner->gfp_mask, @@ -575,11 +634,20 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, migratetype_names[pageblock_mt], &page->flags.f); - ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); - if (ret >= count) - goto err; + if (print_mode != PAGE_OWNER_PRINT_HANDLE) { + ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0); + if (ret >= count) + goto err; + } - if (page_owner->last_migrate_reason != -1) { + if (print_mode != PAGE_OWNER_PRINT_STACK) { + ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n", + handle); + if (ret >= count) + goto err; + } + + if (page_owner->last_migrate_reason != MR_NEVER) { ret += scnprintf(kbuf + ret, count - ret, "Page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); @@ -630,10 +698,10 @@ void __dump_page_owner(const struct page *page) else pr_alert("page_owner tracks the page as freed\n"); - pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n", + pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n", page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, page_owner->pid, page_owner->tgid, page_owner->comm, - page_owner->ts_nsec, page_owner->free_ts_nsec); + page_owner->ts_nsec); handle = READ_ONCE(page_owner->handle); if (!handle) @@ -645,12 +713,13 @@ void __dump_page_owner(const struct page *page) if (!handle) { pr_alert("page_owner free stack trace missing\n"); } else { - pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n", + page_owner->free_pid, page_owner->free_tgid, + page_owner->free_ts_nsec); stack_depot_print(handle); } - if (page_owner->last_migrate_reason != -1) + if (page_owner->last_migrate_reason != MR_NEVER) pr_alert("page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); @@ -664,6 +733,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) struct page_ext *page_ext; struct page_owner *page_owner; depot_stack_handle_t handle; + struct page_owner_filter_state *state = file->private_data; + unsigned long flags; if (!static_branch_unlikely(&page_owner_inited)) return -EINVAL; @@ -697,13 +768,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) } page = pfn_to_page(pfn); - if (PageBuddy(page)) { - unsigned long freepage_order = buddy_order_unsafe(page); - - if (freepage_order <= MAX_PAGE_ORDER) - pfn += (1UL << freepage_order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } page_ext = page_ext_get(page); if (unlikely(!page_ext)) @@ -740,15 +806,37 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) if (!handle) goto ext_put_continue; + spin_lock_irqsave(&state->lock, flags); + if (state->nid_filter_enabled) { + int nid; + memdesc_flags_t page_flags = READ_ONCE(page->flags); + + /* + * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid + * VM_BUG_ON when accessing poisoned pages. + */ + if (page_flags.f == PAGE_POISON_PATTERN) { + spin_unlock_irqrestore(&state->lock, flags); + goto ext_put_continue; + } + nid = memdesc_nid(&page_flags); + if (!node_isset(nid, state->nid_filter)) { + spin_unlock_irqrestore(&state->lock, flags); + goto ext_put_continue; + } + } + spin_unlock_irqrestore(&state->lock, flags); + /* Record the next PFN to read in the file offset */ *ppos = pfn + 1; page_owner_tmp = *page_owner; page_ext_put(page_ext); return print_page_owner(buf, count, pfn, page, - &page_owner_tmp, handle); + &page_owner_tmp, handle, state); ext_put_continue: page_ext_put(page_ext); + cond_resched(); } return 0; @@ -798,20 +886,8 @@ static void init_pages_in_zone(struct zone *zone) if (page_zone(page) != zone) continue; - /* - * To avoid having to grab zone->lock, be a little - * careful when reading buddy page order. The only - * danger is that we skip too much and potentially miss - * some early allocated pages, which is better than - * heavy lock contention. - */ - if (PageBuddy(page)) { - unsigned long order = buddy_order_unsafe(page); - - if (order > 0 && order <= MAX_PAGE_ORDER) - pfn += (1UL << order) - 1; + if (skip_buddy_pages(&pfn, page)) continue; - } if (PageReserved(page)) continue; @@ -826,7 +902,7 @@ static void init_pages_in_zone(struct zone *zone) /* Found early allocated page */ __update_page_owner_handle(page, early_handle, 0, 0, - -1, local_clock(), current->pid, + MR_NEVER, local_clock(), current->pid, current->tgid, current->comm); count++; ext_put_continue: @@ -847,7 +923,120 @@ static void init_early_allocated_pages(void) init_pages_in_zone(zone); } +static int page_owner_open(struct inode *inode, struct file *file) +{ + struct page_owner_filter_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + spin_lock_init(&state->lock); + state->print_mode = PAGE_OWNER_PRINT_STACK; + nodes_clear(state->nid_filter); + state->nid_filter_enabled = false; + file->private_data = state; + return 0; +} + +static int page_owner_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t page_owner_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + char *kbuf; + char *orig; + char *token; + int ret; + struct page_owner_filter_state *state = file->private_data; + nodemask_t new_nid_filter; + bool new_nid_filter_enabled; + enum page_owner_print_mode new_print_mode; + unsigned long flags; + + /* + * Maximum input length for filter commands: + * - 32: print_mode command max length is 17 ("mode=stack_handle") + * with sufficient buffer + * - 6 * MAX_NUMNODES: worst case for nid list + * Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 + * bytes + */ + if (count > 32 + 6 * MAX_NUMNODES) + return -EINVAL; + + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + orig = kbuf; + + spin_lock_irqsave(&state->lock, flags); + new_print_mode = state->print_mode; + new_nid_filter = state->nid_filter; + new_nid_filter_enabled = state->nid_filter_enabled; + spin_unlock_irqrestore(&state->lock, flags); + + while ((token = strsep(&kbuf, " \t\n")) != NULL) { + if (*token == '\0') + continue; + + if (!strncmp(token, "mode=", 5)) { + ret = sysfs_match_string(page_owner_print_mode_strings, + token + 5); + if (ret < 0) + goto out_free; + new_print_mode = ret; + } else if (!strncmp(token, "nid=", 4)) { + ret = nodelist_parse(token + 4, new_nid_filter); + if (ret < 0) + goto out_free; + + if (nodes_empty(new_nid_filter)) { + ret = -EINVAL; + goto out_free; + } + + /* + * We want to filter memory allocations by numa nodes, so make sure + * that the specified nodes have memory. + */ + if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) { + ret = -EINVAL; + goto out_free; + } + + new_nid_filter_enabled = true; + } else { + ret = -EINVAL; + goto out_free; + } + } + + /* Commit all filter changes */ + spin_lock_irqsave(&state->lock, flags); + state->print_mode = new_print_mode; + state->nid_filter = new_nid_filter; + state->nid_filter_enabled = new_nid_filter_enabled; + spin_unlock_irqrestore(&state->lock, flags); + + ret = count; + +out_free: + kfree(orig); + return ret; +} + static const struct file_operations page_owner_fops = { + .owner = THIS_MODULE, + .open = page_owner_open, + .release = page_owner_release, + .write = page_owner_write, .read = read_page_owner, .llseek = lseek_page_owner, }; @@ -887,7 +1076,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned long page_owner_pages_threshold; +static unsigned long pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -904,7 +1093,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages = refcount_read(&stack_record->count) - 1; if (ctx->flags & STACK_PRINT_FLAG_PAGES && - (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold)) + (nr_base_pages < 1 || nr_base_pages < pages_threshold)) return 0; if (ctx->flags & STACK_PRINT_FLAG_STACK) { @@ -926,16 +1115,16 @@ static void stack_stop(struct seq_file *m, void *v) { } -static const struct seq_operations page_owner_stack_op = { +static const struct seq_operations stack_op = { .start = stack_start, .next = stack_next, .stop = stack_stop, .show = stack_print }; -static int page_owner_stack_open(struct inode *inode, struct file *file) +static int stack_open(struct inode *inode, struct file *file) { - int ret = seq_open_private(file, &page_owner_stack_op, + int ret = seq_open_private(file, &stack_op, sizeof(struct stack_print_ctx)); if (!ret) { @@ -948,28 +1137,26 @@ static int page_owner_stack_open(struct inode *inode, struct file *file) return ret; } -static const struct file_operations page_owner_stack_fops = { - .open = page_owner_stack_open, +static const struct file_operations stack_fops = { + .open = stack_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, }; -static int page_owner_threshold_get(void *data, u64 *val) +static int threshold_get(void *data, u64 *val) { - *val = READ_ONCE(page_owner_pages_threshold); + *val = READ_ONCE(pages_threshold); return 0; } -static int page_owner_threshold_set(void *data, u64 val) +static int threshold_set(void *data, u64 val) { - WRITE_ONCE(page_owner_pages_threshold, val); + WRITE_ONCE(pages_threshold, val); return 0; } -DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get, - &page_owner_threshold_set, "%llu"); - +DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n"); static int __init pageowner_init(void) { @@ -980,22 +1167,22 @@ static int __init pageowner_init(void) return 0; } - debugfs_create_file("page_owner", 0400, NULL, NULL, &page_owner_fops); + debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops); dir = debugfs_create_dir("page_owner_stacks", NULL); debugfs_create_file("show_stacks", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_HANDLE | STACK_PRINT_FLAG_PAGES), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("show_stacks_handles", 0400, dir, (void *)(STACK_PRINT_FLAG_STACK | STACK_PRINT_FLAG_HANDLE), - &page_owner_stack_fops); + &stack_fops); debugfs_create_file("count_threshold", 0600, dir, NULL, - &page_owner_threshold_fops); + &threshold_fops); return 0; } late_initcall(pageowner_init) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 7418f2e500bb..942e84b6908a 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -173,11 +173,8 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, * any pages that may have already been present from the previous * list processed. This should result in us reporting all pages on * an idle system in about 30 seconds. - * - * The division here should be cheap since PAGE_REPORTING_CAPACITY - * should always be a power of 2. */ - budget = DIV_ROUND_UP(area->nr_free, PAGE_REPORTING_CAPACITY * 16); + budget = DIV_ROUND_UP(area->nr_free, prdev->capacity * 16); /* loop through free list adding unreported pages to sg list */ list_for_each_entry_safe(page, next, list, lru) { @@ -222,10 +219,10 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, spin_unlock_irq(&zone->lock); /* begin processing pages in local list */ - err = prdev->report(prdev, sgl, PAGE_REPORTING_CAPACITY); + err = prdev->report(prdev, sgl, prdev->capacity); /* reset offset since the full list was reported */ - *offset = PAGE_REPORTING_CAPACITY; + *offset = prdev->capacity; /* update budget to reflect call to report function */ budget--; @@ -234,7 +231,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, spin_lock_irq(&zone->lock); /* flush reported pages from the sg list */ - page_reporting_drain(prdev, sgl, PAGE_REPORTING_CAPACITY, !err); + page_reporting_drain(prdev, sgl, prdev->capacity, !err); /* * Reset next to first entry, the old next isn't valid @@ -260,13 +257,13 @@ static int page_reporting_process_zone(struct page_reporting_dev_info *prdev, struct scatterlist *sgl, struct zone *zone) { - unsigned int order, mt, leftover, offset = PAGE_REPORTING_CAPACITY; + unsigned int order, mt, leftover, offset = prdev->capacity; unsigned long watermark; int err = 0; /* Generate minimum watermark to be able to guarantee progress */ watermark = low_wmark_pages(zone) + - (PAGE_REPORTING_CAPACITY << page_reporting_order); + (prdev->capacity << page_reporting_order); /* * Cancel request if insufficient free memory or if we failed @@ -290,7 +287,7 @@ page_reporting_process_zone(struct page_reporting_dev_info *prdev, } /* report the leftover pages before going idle */ - leftover = PAGE_REPORTING_CAPACITY - offset; + leftover = prdev->capacity - offset; if (leftover) { sgl = &sgl[offset]; err = prdev->report(prdev, sgl, leftover); @@ -322,11 +319,11 @@ static void page_reporting_process(struct work_struct *work) atomic_set(&prdev->state, state); /* allocate scatterlist to store pages being reported on */ - sgl = kmalloc_objs(*sgl, PAGE_REPORTING_CAPACITY); + sgl = kmalloc_objs(*sgl, prdev->capacity); if (!sgl) goto err_out; - sg_init_table(sgl, PAGE_REPORTING_CAPACITY); + sg_init_table(sgl, prdev->capacity); for_each_zone(zone) { err = page_reporting_process_zone(prdev, sgl, zone); @@ -377,6 +374,9 @@ int page_reporting_register(struct page_reporting_dev_info *prdev) page_reporting_order = pageblock_order; } + if (!prdev->capacity || prdev->capacity > PAGE_REPORTING_CAPACITY) + prdev->capacity = PAGE_REPORTING_CAPACITY; + /* initialize state and work structures */ atomic_set(&prdev->state, PAGE_REPORTING_IDLE); INIT_DELAYED_WORK(&prdev->work, &page_reporting_process); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 2ccbabfb2cc1..bac2eb5de63d 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -243,21 +243,31 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) */ pmde = pmdp_get_lockless(pvmw->pmd); - if (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde)) { + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && + (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || + pmd_is_device_private_entry(pmde))) { pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; - if (!pmd_present(pmde)) { + if (pmd_is_migration_entry(pmde)) { softleaf_t entry; - if (!thp_migration_supported() || - !(pvmw->flags & PVMW_MIGRATION)) + if (!(pvmw->flags & PVMW_MIGRATION)) return not_found(pvmw); entry = softleaf_from_pmd(pmde); + if (!check_pmd(softleaf_to_pfn(entry), pvmw)) + return not_found(pvmw); + return true; + } else if (pmd_is_device_private_entry(pmde)) { + softleaf_t entry; - if (!softleaf_is_migration(entry) || - !check_pmd(softleaf_to_pfn(entry), pvmw)) + if (pvmw->flags & PVMW_MIGRATION) + return not_found(pvmw); + entry = softleaf_from_pmd(pmde); + if (!check_pmd(softleaf_to_pfn(entry), pvmw)) return not_found(pvmw); return true; + } else if (!pmd_present(pmde)) { + return not_found(pvmw); } if (likely(pmd_trans_huge(pmde))) { if (pvmw->flags & PVMW_MIGRATION) @@ -266,17 +276,10 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return not_found(pvmw); return true; } - /* THP pmd was split under us: handle on pte level */ + /* THP/device-private pmd was split under us: handle on pte level */ spin_unlock(pvmw->ptl); pvmw->ptl = NULL; } else if (!pmd_present(pmde)) { - const softleaf_t entry = softleaf_from_pmd(pmde); - - if (softleaf_is_device_private(entry)) { - pvmw->ptl = pmd_lock(mm, pvmw->pmd); - return true; - } - if ((pvmw->flags & PVMW_SYNC) && thp_vma_suitable_order(vma, pvmw->address, PMD_ORDER) && diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 4f5937090590..7ed216192fc0 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -21,6 +21,8 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, /** * pcpu_get_pages - get temp pages array + * @gfp: allocation flags passed to the underlying allocator, 0 to only + * return the cached array * * Returns pointer to array of pointers to struct page which can be indexed * with pcpu_page_idx(). Note that there is only one array and accesses @@ -29,18 +31,23 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk, * RETURNS: * Pointer to temp pages array on success. */ -static struct page **pcpu_get_pages(void) +static struct page **pcpu_get_pages(gfp_t gfp) { static struct page **pages; size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]); lockdep_assert_held(&pcpu_alloc_mutex); - if (!pages) - pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL); + if (!pages && gfp) + pages = pcpu_mem_zalloc(pages_size, gfp); return pages; } +static struct page **pcpu_get_pages_cached(void) +{ + return pcpu_get_pages(0); +} + /** * pcpu_free_pages - free pages which were allocated for @chunk * @chunk: chunk pages were allocated for @@ -191,10 +198,22 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk, } static int __pcpu_map_pages(unsigned long addr, struct page **pages, - int nr_pages) + int nr_pages, gfp_t gfp) { - return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), - PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL); + unsigned int flags; + int ret; + + /* + * The vmalloc page table allocation path does not pass @gfp down + * explicitly. Apply the corresponding memalloc scope so implicit + * page table allocations preserve NOFS/NOIO constraints. + */ + flags = memalloc_apply_gfp_scope(gfp); + ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT), + PAGE_KERNEL, pages, PAGE_SHIFT, gfp); + memalloc_restore_scope(flags); + + return ret; } /** @@ -203,6 +222,7 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * @pages: pages array containing pages to be mapped * @page_start: page index of the first page to map * @page_end: page index of the last page to map + 1 + * @gfp: allocation flags passed to the underlying allocator * * For each cpu, map pages [@page_start,@page_end) into @chunk. The * caller is responsible for calling pcpu_post_map_flush() after all @@ -211,8 +231,8 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages, * This function is responsible for setting up whatever is necessary for * reverse lookup (addr -> chunk). */ -static int pcpu_map_pages(struct pcpu_chunk *chunk, - struct page **pages, int page_start, int page_end) +static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages, + int page_start, int page_end, gfp_t gfp) { unsigned int cpu, tcpu; int i, err; @@ -220,7 +240,7 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk, for_each_possible_cpu(cpu) { err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), &pages[pcpu_page_idx(cpu, page_start)], - page_end - page_start); + page_end - page_start, gfp); if (err < 0) goto err; @@ -271,21 +291,21 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk, * @chunk. * * CONTEXT: - * pcpu_alloc_mutex, does GFP_KERNEL allocation. + * pcpu_alloc_mutex, does @gfp allocation. */ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int page_start, int page_end, gfp_t gfp) { struct page **pages; - pages = pcpu_get_pages(); + pages = pcpu_get_pages(gfp); if (!pages) return -ENOMEM; if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp)) return -ENOMEM; - if (pcpu_map_pages(chunk, pages, page_start, page_end)) { + if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) { pcpu_free_pages(chunk, pages, page_start, page_end); return -ENOMEM; } @@ -319,7 +339,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, * successful population attempt so the temp pages array must * be available now. */ - pages = pcpu_get_pages(); + pages = pcpu_get_pages_cached(); BUG_ON(!pages); /* unmap and free */ @@ -340,7 +360,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) return NULL; vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes, - pcpu_nr_groups, pcpu_atom_size); + pcpu_nr_groups, pcpu_atom_size, gfp); if (!vms) { pcpu_free_chunk(chunk); return NULL; diff --git a/mm/percpu.c b/mm/percpu.c index b0676b8054ed..a802d72c116f 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1726,9 +1726,8 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s * @gfp: allocation flags * * Allocate percpu area of @size bytes aligned at @align. If @gfp doesn't - * contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN - * then no warning will be triggered on invalid or failed allocation - * requests. + * allow blocking, the allocation is atomic. If @gfp has __GFP_NOWARN then no + * warning will be triggered on invalid or failed allocation requests. * * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. @@ -1749,8 +1748,17 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, size_t bits, bit_align; gfp = current_gfp_context(gfp); - /* whitelisted flags that can be passed to the backing allocators */ - pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + /* + * Allowlisted flags that can be passed to the backing allocators. + * Backing allocations under pcpu_alloc_mutex must not recurse into + * IO/FS reclaim. Otherwise a GFP_KERNEL caller holding the mutex can + * block on reclaim while a GFP_NOIO/NOFS caller holding an IO/FS lock + * waits for the same mutex. + * + * Do not pass __GFP_NOFAIL. A small percpu allocation may need many + * backing pages, making nofail reclaim too costly under NOIO/NOFS. + */ + pcpu_gfp = gfp & (GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); is_atomic = !gfpflags_allow_blocking(gfp); do_warn = !(gfp & __GFP_NOWARN); @@ -3256,7 +3264,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t /* pte already populated, the following shouldn't fail */ rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], - unit_pages); + unit_pages, GFP_KERNEL); if (rc < 0) panic("failed to map percpu area, err=%d\n", rc); diff --git a/mm/readahead.c b/mm/readahead.c index 38ce16e3fcbd..558c92957518 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -335,8 +335,6 @@ static void do_page_cache_ra(struct readahead_control *ractl, return; end_index = (isize - 1) >> PAGE_SHIFT; - if (end_index > ractl->_max_index) - end_index = ractl->_max_index; if (index > end_index) return; /* Don't read past the page containing the last byte of the file */ @@ -487,7 +485,7 @@ void page_cache_ra_order(struct readahead_control *ractl, pgoff_t start = readahead_index(ractl); pgoff_t index = start; unsigned int min_order = mapping_min_folio_order(mapping); - pgoff_t limit; + pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT; pgoff_t mark; unsigned int nofs; int err = 0; @@ -500,8 +498,6 @@ void page_cache_ra_order(struct readahead_control *ractl, goto fallback; } - limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT; - limit = min(limit, ractl->_max_index); if (limit > index + ra->size - 1) { limit = index + ra->size - 1; mark = index + ra->size - ra->async_size; diff --git a/mm/rmap.c b/mm/rmap.c index 1c77d5dc06e9..0fb7a1b82cf3 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2304,12 +2304,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } - if (list_empty(&mm->mmlist)) { - spin_lock(&mmlist_lock); - if (list_empty(&mm->mmlist)) - list_add(&mm->mmlist, &init_mm.mmlist); - spin_unlock(&mmlist_lock); - } + mm_prepare_for_swap_entries(mm); dec_mm_counter(mm, MM_ANONPAGES); inc_mm_counter(mm, MM_SWAPENTS); swp_pte = swp_entry_to_pte(entry); @@ -2477,7 +2472,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_restart(&pvmw); continue; } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_SUPPORTS_PMD_SOFTLEAF pmdval = pmdp_get(pvmw.pmd); if (likely(pmd_present(pmdval))) pfn = pmd_pfn(pmdval); diff --git a/mm/shmem.c b/mm/shmem.c index b51f83c970bb..8c4736188b57 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1657,7 +1657,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, * reactivate the folio, and let shmem_fallocate() quit when too many. */ if (!folio_test_uptodate(folio)) { - if (inode->i_private) { + if (READ_ONCE(inode->i_private)) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; @@ -2693,7 +2693,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ - if (unlikely(inode->i_private)) { + if (unlikely(READ_ONCE(inode->i_private))) { ret = shmem_falloc_wait(vmf, inode); if (ret) return ret; @@ -3630,7 +3630,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); if ((u64)unmap_end > (u64)unmap_start) @@ -3640,7 +3640,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, /* No need to unmap again: hole-punching leaves COWed pages */ spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); wake_up_all(&shmem_falloc_waitq); WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); spin_unlock(&inode->i_lock); @@ -3672,7 +3672,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, shmem_falloc.nr_falloced = 0; shmem_falloc.nr_unswapped = 0; spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; + WRITE_ONCE(inode->i_private, &shmem_falloc); spin_unlock(&inode->i_lock); /* @@ -3747,7 +3747,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, i_size_write(inode, offset + len); undone: spin_lock(&inode->i_lock); - inode->i_private = NULL; + WRITE_ONCE(inode->i_private, NULL); spin_unlock(&inode->i_lock); out: if (!error) @@ -3864,7 +3864,7 @@ static struct dentry *shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir, } static int shmem_create(struct mnt_idmap *idmap, struct inode *dir, - struct dentry *dentry, umode_t mode, bool excl) + struct dentry *dentry, umode_t mode) { return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0); } @@ -4057,6 +4057,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, goto out_remove_offset; inode->i_op = &shmem_symlink_inode_operations; memcpy(folio_address(folio), symname, len); + folio_zero_range(folio, len, folio_size(folio) - len); folio_mark_uptodate(folio); folio_mark_dirty(folio); folio_unlock(folio); diff --git a/mm/show_mem.c b/mm/show_mem.c index 43aca5a2ac99..1b721a8ade67 100644 --- a/mm/show_mem.c +++ b/mm/show_mem.c @@ -116,7 +116,8 @@ void si_meminfo_node(struct sysinfo *val, int nid) * Determine whether the node should be displayed or not, depending on whether * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). */ -static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask) +static bool show_mem_node_skip(unsigned int flags, int nid, + const nodemask_t *nodemask) { if (!(flags & SHOW_MEM_FILTER_NODES)) return false; @@ -177,7 +178,8 @@ static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's * cpuset. */ -static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long free_pcp = 0; int cpu, nid; @@ -402,7 +404,8 @@ static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_z show_swap_cache_info(); } -void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) +void __show_mem(unsigned int filter, const nodemask_t *nodemask, + int max_zone_idx) { unsigned long total = 0, reserved = 0, highmem = 0; struct zone *zone; diff --git a/mm/shrinker.c b/mm/shrinker.c index 7082d01c8c9d..a70aab124a0e 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -59,12 +59,14 @@ static inline int shrinker_unit_alloc(struct shrinker_info *new, return 0; } -void free_shrinker_info(struct mem_cgroup *memcg) +static void __free_shrinker_info(struct mem_cgroup *memcg) { struct mem_cgroup_per_node *pn; struct shrinker_info *info; int nid; + lockdep_assert_held(&shrinker_mutex); + for_each_node(nid) { pn = memcg->nodeinfo[nid]; info = rcu_dereference_protected(pn->shrinker_info, true); @@ -74,6 +76,13 @@ void free_shrinker_info(struct mem_cgroup *memcg) } } +void free_shrinker_info(struct mem_cgroup *memcg) +{ + mutex_lock(&shrinker_mutex); + __free_shrinker_info(memcg); + mutex_unlock(&shrinker_mutex); +} + int alloc_shrinker_info(struct mem_cgroup *memcg) { int nid, ret = 0; @@ -98,8 +107,8 @@ int alloc_shrinker_info(struct mem_cgroup *memcg) return ret; err: + __free_shrinker_info(memcg); mutex_unlock(&shrinker_mutex); - free_shrinker_info(memcg); return -ENOMEM; } diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c index affa64437302..cafb56630132 100644 --- a/mm/shrinker_debug.c +++ b/mm/shrinker_debug.c @@ -57,8 +57,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v) if (!count_per_node) return -ENOMEM; - rcu_read_lock(); - memcg_aware = shrinker->flags & SHRINKER_MEMCG_AWARE; memcg = mem_cgroup_iter(NULL, NULL, NULL); @@ -88,8 +86,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v) } } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); - rcu_read_unlock(); - kfree(count_per_node); return ret; } @@ -187,10 +183,12 @@ int shrinker_debugfs_add(struct shrinker *shrinker) } shrinker->debugfs_entry = entry; - debugfs_create_file("count", 0440, entry, shrinker, - &shrinker_debugfs_count_fops); - debugfs_create_file("scan", 0220, entry, shrinker, - &shrinker_debugfs_scan_fops); + if (shrinker->count_objects) + debugfs_create_file("count", 0440, entry, shrinker, + &shrinker_debugfs_count_fops); + if (shrinker->scan_objects) + debugfs_create_file("scan", 0220, entry, shrinker, + &shrinker_debugfs_scan_fops); return 0; } diff --git a/mm/slab.h b/mm/slab.h index 1bf9c3021ae3..9ded319495a0 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -11,11 +11,35 @@ #include #include #include +#include /* * Internal slab definitions */ +/* slab's alloc_flags definitions */ +#define SLAB_ALLOC_DEFAULT 0x00 /* no flags */ +#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */ +#define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */ +#define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */ + +static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags) +{ + return !(alloc_flags & SLAB_ALLOC_NOLOCK); +} + +void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags, + unsigned int alloc_flags, int node) + __assume_kmalloc_alignment __alloc_size(1); + +static __always_inline __alloc_size(1) void *_kmalloc_flags_noprof(size_t size, + gfp_t flags, unsigned int alloc_flags, int node, kmalloc_token_t token) +{ + return __kmalloc_flags_noprof(PASS_TOKEN_PARAMS(size, token), flags, alloc_flags, node); +} +#define kmalloc_flags_noprof(...) _kmalloc_flags_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) +#define kmalloc_flags(...) alloc_hooks(kmalloc_flags_noprof(__VA_ARGS__)) + #ifdef CONFIG_64BIT # ifdef system_has_cmpxchg128 # define system_has_freelist_aba() system_has_cmpxchg128() @@ -155,7 +179,7 @@ static inline void *slab_address(const struct slab *slab) static inline int slab_nid(const struct slab *slab) { - return memdesc_nid(slab->flags); + return memdesc_nid(&slab->flags); } static inline pg_data_t *slab_pgdat(const struct slab *slab) @@ -603,7 +627,7 @@ static inline struct slabobj_ext *slab_obj_ext(struct slab *slab, } int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab); + gfp_t gfp, unsigned int alloc_flags); #else /* CONFIG_SLAB_OBJ_EXT */ @@ -633,7 +657,8 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s) #ifdef CONFIG_MEMCG bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, - gfp_t flags, size_t size, void **p); + gfp_t flags, unsigned int slab_alloc_flags, + size_t size, void **p); void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, int objects, unsigned long obj_exts); #endif diff --git a/mm/slub.c b/mm/slub.c index f87e693aca5d..550efd79d146 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -213,10 +213,12 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); static DEFINE_STATIC_KEY_FALSE(strict_numa); #endif -/* Structure holding parameters for get_from_partial() call chain */ -struct partial_context { - gfp_t flags; - unsigned int orig_size; +/* Structure holding extra parameters for slab allocations */ +struct slab_alloc_context { + unsigned long caller_addr; + size_t orig_size; + unsigned int alloc_flags; + struct list_lru *lru; }; /* Structure holding parameters for get_partial_node_bulk() */ @@ -2045,12 +2047,16 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node, #endif /* CONFIG_SLUB_DEBUG */ /* - * The allocated objcg pointers array is not accounted directly. + * The allocated objcg pointers array or sheaf is not accounted directly. * Moreover, it should not come from DMA buffer and is not readily - * reclaimable. So those GFP bits should be masked off. + * reclaimable. Node restriction for the parent allocation also should + * not apply to the slab's internal objects, as well as __GFP_COMP used + * for new slab allocations. + * So those GFP bits should be masked off. */ #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ - __GFP_ACCOUNT | __GFP_NOFAIL) + __GFP_ACCOUNT | __GFP_NOFAIL | \ + __GFP_THISNODE | __GFP_COMP) #ifdef CONFIG_SLAB_OBJ_EXT @@ -2154,10 +2160,11 @@ static inline size_t obj_exts_alloc_size(struct kmem_cache *s, } int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab) + gfp_t gfp, unsigned int alloc_flags) { - bool allow_spin = gfpflags_allow_spinning(gfp); + const bool allow_spin = alloc_flags_allow_spinning(alloc_flags); unsigned int objects = objs_per_slab(s, slab); + bool new_slab = alloc_flags & SLAB_ALLOC_NEW_SLAB; unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; @@ -2165,21 +2172,13 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp &= ~OBJCGS_CLEAR_MASK; /* Prevent recursive extension vector allocation */ - gfp |= __GFP_NO_OBJ_EXT; + alloc_flags |= SLAB_ALLOC_NO_RECURSE; + alloc_flags &= ~SLAB_ALLOC_NEW_SLAB; sz = obj_exts_alloc_size(s, slab, gfp); - /* - * Note that allow_spin may be false during early boot and its - * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting - * architectures with cmpxchg16b, early obj_exts will be missing for - * very early allocations on those. - */ - if (unlikely(!allow_spin)) - vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, - slab_nid(slab)); - else - vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); + /* This will use kmalloc_nolock() if alloc_flags say so */ + vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab)); if (!vec) { /* @@ -2255,7 +2254,7 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin) } /* - * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its + * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its * corresponding extension will be NULL. alloc_tag_sub() will throw a * warning if slab has extensions but the extension of an object is * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that @@ -2330,7 +2329,7 @@ static inline void init_slab_obj_exts(struct slab *slab) } static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, - gfp_t gfp, bool new_slab) + gfp_t gfp, unsigned int alloc_flags) { return 0; } @@ -2350,10 +2349,10 @@ static inline void alloc_slab_obj_exts_early(struct kmem_cache *s, static inline unsigned long prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab, - gfp_t flags, void *p) + gfp_t flags, unsigned int alloc_flags, void *p) { if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, false)) { + alloc_slab_obj_exts(slab, s, flags, alloc_flags)) { pr_warn_once("%s, %s: Failed to create slab extension vector!\n", __func__, s->name); return 0; @@ -2365,7 +2364,8 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab, /* Should be called only if mem_alloc_profiling_enabled() */ static noinline void -__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) +__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, + unsigned int alloc_flags) { unsigned long obj_exts; struct slabobj_ext *obj_ext; @@ -2377,11 +2377,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) return; - if (flags & __GFP_NO_OBJ_EXT) + if (alloc_flags & SLAB_ALLOC_NO_RECURSE) return; slab = virt_to_slab(object); - obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, object); + obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object); /* * Currently obj_exts is used only for allocation profiling. * If other users appear then mem_alloc_profiling_enabled() @@ -2400,10 +2400,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) } static inline void -alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) +alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, + unsigned int alloc_flags) { if (mem_alloc_profiling_enabled()) - __alloc_tagging_slab_alloc_hook(s, object, flags); + __alloc_tagging_slab_alloc_hook(s, object, flags, alloc_flags); } /* Should be called only if mem_alloc_profiling_enabled() */ @@ -2442,7 +2443,8 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, #else /* CONFIG_MEM_ALLOC_PROFILING */ static inline void -alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) +alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, + unsigned int alloc_flags) { } @@ -2460,8 +2462,9 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); static __fastpath_inline -bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, - gfp_t flags, size_t size, void **p) +bool memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, + size_t size, void **p, + const struct slab_alloc_context *ac) { if (likely(!memcg_kmem_online())) return true; @@ -2469,7 +2472,8 @@ bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) return true; - if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p))) + if (likely(__memcg_slab_post_alloc_hook(s, ac->lru, flags, + ac->alloc_flags, size, p))) return true; if (likely(size == 1)) { @@ -2557,14 +2561,15 @@ bool memcg_slab_post_charge(void *p, gfp_t flags) put_slab_obj_exts(obj_exts); } - return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p); + return __memcg_slab_post_alloc_hook(s, NULL, flags, SLAB_ALLOC_DEFAULT, + 1, &p); } #else /* CONFIG_MEMCG */ static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, - struct list_lru *lru, - gfp_t flags, size_t size, - void **p) + gfp_t flags, + size_t size, void **p, + const struct slab_alloc_context *ac) { return true; } @@ -2757,7 +2762,7 @@ static inline void *setup_object(struct kmem_cache *s, void *object) } static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp, - unsigned int capacity) + unsigned int alloc_flags, unsigned int capacity) { struct slab_sheaf *sheaf; size_t sheaf_size; @@ -2768,10 +2773,10 @@ static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp, * bucket) */ if (s->flags & SLAB_KMALLOC) - gfp |= __GFP_NO_OBJ_EXT; + alloc_flags |= SLAB_ALLOC_NO_RECURSE; sheaf_size = struct_size(sheaf, objects, capacity); - sheaf = kzalloc(sheaf_size, gfp); + sheaf = kmalloc_flags(sheaf_size, gfp | __GFP_ZERO, alloc_flags, NUMA_NO_NODE); if (unlikely(!sheaf)) return NULL; @@ -2784,20 +2789,20 @@ static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp, } static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, - gfp_t gfp) + gfp_t gfp, unsigned int alloc_flags) { - if (gfp & __GFP_NO_OBJ_EXT) + if (alloc_flags & SLAB_ALLOC_NO_RECURSE) return NULL; gfp &= ~OBJCGS_CLEAR_MASK; - return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity); + return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity); } static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) { /* - * If the sheaf was created with __GFP_NO_OBJ_EXT flag then its + * If the sheaf was created with SLAB_ALLOC_NO_RECURSE flag then its * corresponding extension is NULL and alloc_tag_sub() will throw a * warning, therefore replace NULL with CODETAG_EMPTY to indicate * that the extension for this sheaf is expected to be NULL. @@ -3351,12 +3356,14 @@ static inline void init_freelist_randomization(void) { } #endif /* CONFIG_SLAB_FREELIST_RANDOM */ static __always_inline void account_slab(struct slab *slab, int order, - struct kmem_cache *s, gfp_t gfp) + struct kmem_cache *s, gfp_t gfp, + unsigned int alloc_flags) { if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT) && !slab_obj_exts(slab)) - alloc_slab_obj_exts(slab, s, gfp, true); + alloc_slab_obj_exts(slab, s, gfp, + alloc_flags | SLAB_ALLOC_NEW_SLAB); mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), PAGE_SIZE << order); @@ -3377,9 +3384,10 @@ static __always_inline void unaccount_slab(struct slab *slab, int order, } /* Allocate and initialize a slab without building its freelist. */ -static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, + unsigned int alloc_flags, int node) { - bool allow_spin = gfpflags_allow_spinning(flags); + bool allow_spin = alloc_flags_allow_spinning(alloc_flags); struct slab *slab; struct kmem_cache_order_objects oo = s->oo; gfp_t alloc_gfp; @@ -3397,10 +3405,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; - /* - * __GFP_RECLAIM could be cleared on the first allocation attempt, - * so pass allow_spin flag directly. - */ slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin); if (unlikely(!slab)) { oo = s->min; @@ -3432,20 +3436,22 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * to prevent the array from being overwritten. */ alloc_slab_obj_exts_early(s, slab); - account_slab(slab, oo_order(oo), s, flags); + account_slab(slab, oo_order(oo), s, flags, alloc_flags); return slab; } -static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, + unsigned int alloc_flags, int node) { if (unlikely(flags & GFP_SLAB_BUG_MASK)) flags = kmalloc_fix_flags(flags); WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); - return allocate_slab(s, - flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); + flags &= GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK; + + return allocate_slab(s, flags, alloc_flags, node); } static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin) @@ -3687,8 +3693,9 @@ static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab, * and put the slab to the partial (or full) list. */ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, - int orig_size, bool allow_spin) + const struct slab_alloc_context *ac) { + bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags); struct kmem_cache_node *n; struct slab_obj_iter iter; bool needs_add_partial; @@ -3705,7 +3712,7 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, /* alloc_debug_processing() always expects a valid freepointer */ set_freepointer(s, object, slab->freelist); - if (!alloc_debug_processing(s, slab, object, orig_size)) { + if (!alloc_debug_processing(s, slab, object, ac->orig_size)) { /* * It's not really expected that this would fail on a * freshly allocated slab, but a concurrent memory @@ -3818,7 +3825,8 @@ static bool get_partial_node_bulk(struct kmem_cache *s, */ static void *get_from_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, - struct partial_context *pc) + gfp_t gfp_flags, + const struct slab_alloc_context *ac) { struct slab *slab, *slab2; unsigned long flags; @@ -3833,7 +3841,7 @@ static void *get_from_partial_node(struct kmem_cache *s, if (!n || !n->nr_partial) return NULL; - if (gfpflags_allow_spinning(pc->flags)) + if (alloc_flags_allow_spinning(ac->alloc_flags)) spin_lock_irqsave(&n->list_lock, flags); else if (!spin_trylock_irqsave(&n->list_lock, flags)) return NULL; @@ -3841,12 +3849,12 @@ static void *get_from_partial_node(struct kmem_cache *s, struct freelist_counters old, new; - if (!pfmemalloc_match(slab, pc->flags)) + if (!pfmemalloc_match(slab, gfp_flags)) continue; if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { object = alloc_single_from_partial(s, n, slab, - pc->orig_size); + ac->orig_size); if (object) break; continue; @@ -3880,15 +3888,16 @@ static void *get_from_partial_node(struct kmem_cache *s, /* * Get an object from somewhere. Search in increasing NUMA distances. */ -static void *get_from_any_partial(struct kmem_cache *s, struct partial_context *pc) +static void *get_from_any_partial(struct kmem_cache *s, gfp_t gfp_flags, + const struct slab_alloc_context *ac) { #ifdef CONFIG_NUMA struct zonelist *zonelist; struct zoneref *z; struct zone *zone; - enum zone_type highest_zoneidx = gfp_zone(pc->flags); + enum zone_type highest_zoneidx = gfp_zone(gfp_flags); unsigned int cpuset_mems_cookie; - bool allow_spin = gfpflags_allow_spinning(pc->flags); + bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags); /* * The defrag ratio allows a configuration of the tradeoffs between @@ -3922,16 +3931,17 @@ static void *get_from_any_partial(struct kmem_cache *s, struct partial_context * if (allow_spin) cpuset_mems_cookie = read_mems_allowed_begin(); - zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); + zonelist = node_zonelist(mempolicy_slab_node(), gfp_flags); for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; n = get_node(s, zone_to_nid(zone)); - if (n && cpuset_zone_allowed(zone, pc->flags) && + if (n && cpuset_zone_allowed(zone, gfp_flags) && n->nr_partial > s->min_partial) { - void *object = get_from_partial_node(s, n, pc); + void *object = get_from_partial_node(s, n, + gfp_flags, ac); if (object) { /* @@ -3953,8 +3963,8 @@ static void *get_from_any_partial(struct kmem_cache *s, struct partial_context * /* * Get an object from a partial slab */ -static void *get_from_partial(struct kmem_cache *s, int node, - struct partial_context *pc) +static void *get_from_partial(struct kmem_cache *s, int node, gfp_t flags, + const struct slab_alloc_context *ac) { int searchnode = node; void *object; @@ -3962,11 +3972,11 @@ static void *get_from_partial(struct kmem_cache *s, int node, if (node == NUMA_NO_NODE) searchnode = numa_mem_id(); - object = get_from_partial_node(s, get_node(s, searchnode), pc); - if (object || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE))) + object = get_from_partial_node(s, get_node(s, searchnode), flags, ac); + if (object || (node != NUMA_NO_NODE && (flags & __GFP_THISNODE))) return object; - return get_from_any_partial(s, pc); + return get_from_any_partial(s, flags, ac); } static bool has_pcs_used(int cpu, struct kmem_cache *s) @@ -4444,45 +4454,41 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab, * slab. */ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, unsigned int orig_size) + const struct slab_alloc_context *ac) { - bool allow_spin = gfpflags_allow_spinning(gfpflags); + bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags); + gfp_t trynode_flags; void *object; struct slab *slab; - struct partial_context pc; bool try_thisnode = true; stat(s, ALLOC_SLOWPATH); new_objects: - pc.flags = gfpflags; + trynode_flags = gfpflags; /* * When a preferred node is indicated but no __GFP_THISNODE * * 1) try to get a partial slab from target node only by having - * __GFP_THISNODE in pc.flags for get_from_partial() + * __GFP_THISNODE in trynode_flags for get_from_partial() * 2) if 1) failed, try to allocate a new slab from target node with - * GPF_NOWAIT | __GFP_THISNODE opportunistically + * (at most) GFP_NOWAIT | __GFP_THISNODE opportunistically * 3) if 2) failed, retry with original gfpflags which will allow * get_from_partial() try partial lists of other nodes before * potentially allocating new page from other nodes */ if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) && try_thisnode)) { - if (unlikely(!allow_spin)) - /* Do not upgrade gfp to NOWAIT from more restrictive mode */ - pc.flags = gfpflags | __GFP_THISNODE; - else - pc.flags = GFP_NOWAIT | __GFP_THISNODE; + trynode_flags &= GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_ACCOUNT; + trynode_flags |= __GFP_NOWARN | __GFP_THISNODE; } - pc.orig_size = orig_size; - object = get_from_partial(s, node, &pc); + object = get_from_partial(s, node, trynode_flags, ac); if (object) goto success; - slab = new_slab(s, pc.flags, node); + slab = new_slab(s, trynode_flags, ac->alloc_flags, node); if (unlikely(!slab)) { if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) @@ -4497,7 +4503,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, stat(s, ALLOC_SLAB); if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { - object = alloc_single_from_new_slab(s, slab, orig_size, allow_spin); + object = alloc_single_from_new_slab(s, slab, ac); if (likely(object)) goto success; @@ -4515,16 +4521,13 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, success: if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) - set_track(s, object, TRACK_ALLOC, addr, gfpflags); + set_track(s, object, TRACK_ALLOC, ac->caller_addr, gfpflags); return object; } -static __always_inline void *__slab_alloc_node(struct kmem_cache *s, - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) +static __always_inline int apply_strict_numa_policy(int node) { - void *object; - #ifdef CONFIG_NUMA if (static_branch_unlikely(&strict_numa) && node == NUMA_NO_NODE) { @@ -4545,10 +4548,7 @@ static __always_inline void *__slab_alloc_node(struct kmem_cache *s, } } #endif - - object = ___slab_alloc(s, gfpflags, node, addr, orig_size); - - return object; + return node; } static __fastpath_inline @@ -4565,14 +4565,13 @@ struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) } static __fastpath_inline -bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, - gfp_t flags, size_t size, void **p, bool init, - unsigned int orig_size) +bool slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size, + void **p, const struct slab_alloc_context *ac) { + bool init = slab_want_init_on_alloc(flags, s); unsigned int zero_size = s->object_size; - bool kasan_init = init; - size_t i; gfp_t init_flags = flags & gfp_allowed_mask; + bool kasan_init = false; /* * For kmalloc object, the allocated size (object_size) can be larger @@ -4586,39 +4585,44 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, * orig_size if we track it. */ if (slub_debug_orig_size(s)) - zero_size = orig_size; + zero_size = ac->orig_size; /* - * When slab_debug is enabled, avoid memory initialization integrated - * into KASAN and instead zero out the memory via the memset below with - * the proper size. Otherwise, KASAN might overwrite SLUB redzones and - * cause false-positive reports. This does not lead to a performance + * ARM64 can set memory tags and zero the memory using a single + * instruction. Since HW_TAGS KASAN uses that while tagging the object, + * separate zeroing is unnecessary. + * + * However, KASAN never zeroes memory when slab_debug is enabled to + * avoid overwriting SLUB redzones. This does not lead to a performance * penalty on production builds, as slab_debug is not intended to be * enabled there. */ - if (__slub_debug_enabled()) - kasan_init = false; + if (kasan_has_integrated_init() && !__slub_debug_enabled()) { + kasan_init = init; + init = false; + } - /* - * As memory initialization might be integrated into KASAN, - * kasan_slab_alloc and initialization memset must be - * kept together to avoid discrepancies in behavior. - * - * As p[i] might get tagged, memset and kmemleak hook come after KASAN. - */ - for (i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init); - if (p[i] && init && (!kasan_init || - !kasan_has_integrated_init())) + + /* + * memset and hooks come after KASAN as p[i] might get tagged + * + * kfence zeroes the object instead of SLUB to avoid overwriting + * its own redzone starting at orig_size, which could happen + * with SLUB zeroing full s->object_size + */ + if (init && p[i] && !is_kfence_address(p[i])) memset(p[i], 0, zero_size); - if (gfpflags_allow_spinning(flags)) + + if (alloc_flags_allow_spinning(ac->alloc_flags)) kmemleak_alloc_recursive(p[i], s->object_size, 1, s->flags, init_flags); kmsan_slab_alloc(s, p[i], init_flags); - alloc_tagging_slab_alloc_hook(s, p[i], flags); + alloc_tagging_slab_alloc_hook(s, p[i], flags, ac->alloc_flags); } - return memcg_slab_post_alloc_hook(s, lru, flags, size, p); + return memcg_slab_post_alloc_hook(s, flags, size, p, ac); } /* @@ -4630,7 +4634,8 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, * unlocked. */ static struct slub_percpu_sheaves * -__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp) +__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, + gfp_t gfp, unsigned int alloc_flags) { struct slab_sheaf *empty = NULL; struct slab_sheaf *full; @@ -4656,7 +4661,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, return NULL; } - allow_spin = gfpflags_allow_spinning(gfp); + allow_spin = alloc_flags_allow_spinning(alloc_flags); full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin); @@ -4684,7 +4689,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, return NULL; if (!empty) { - empty = alloc_empty_sheaf(s, gfp); + empty = alloc_empty_sheaf(s, gfp, alloc_flags); if (!empty) return NULL; } @@ -4742,34 +4747,12 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, } static __fastpath_inline -void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node) +void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node) { struct slub_percpu_sheaves *pcs; bool node_requested; void *object; -#ifdef CONFIG_NUMA - if (static_branch_unlikely(&strict_numa) && - node == NUMA_NO_NODE) { - - struct mempolicy *mpol = current->mempolicy; - - if (mpol) { - /* - * Special BIND rule support. If the local node - * is in permitted set then do not redirect - * to a particular node. - * Otherwise we apply the memory policy to get - * the node we need to allocate on. - */ - if (mpol->mode != MPOL_BIND || - !node_isset(numa_mem_id(), mpol->nodes)) - - node = mempolicy_slab_node(); - } - } -#endif - node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE; /* @@ -4787,7 +4770,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node) pcs = this_cpu_ptr(s->cpu_sheaves); if (unlikely(pcs->main->size == 0)) { - pcs = __pcs_replace_empty_main(s, pcs, gfp); + pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags); if (unlikely(!pcs)) return NULL; } @@ -4817,8 +4800,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node) } static __fastpath_inline -unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size, - void **p) +unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *main; @@ -4853,7 +4835,7 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size, } full = barn_replace_empty_sheaf(barn, pcs->main, - gfpflags_allow_spinning(gfp)); + /* allow_spin = */ true); if (full) { stat(s, BARN_GET); @@ -4907,44 +4889,48 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size, * * Otherwise we can simply pick the next object from the lockless free list. */ -static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) +static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, + gfp_t gfpflags, int node, const struct slab_alloc_context *ac) { void *object; - bool init = false; s = slab_pre_alloc_hook(s, gfpflags); if (unlikely(!s)) return NULL; - object = kfence_alloc(s, orig_size, gfpflags); + object = kfence_alloc(s, ac->orig_size, gfpflags); if (unlikely(object)) goto out; - object = alloc_from_pcs(s, gfpflags, node); + node = apply_strict_numa_policy(node); - if (!object) - object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); + object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node); + + if (unlikely(!object)) + object = ___slab_alloc(s, gfpflags, node, ac); maybe_wipe_obj_freeptr(s, object); - init = slab_want_init_on_alloc(gfpflags, s); out: /* - * When init equals 'true', like for kzalloc() family, only - * @orig_size bytes might be zeroed instead of s->object_size * In case this fails due to memcg_slab_post_alloc_hook(), * object is set to NULL */ - slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size); + slab_post_alloc_hook(s, gfpflags, 1, &object, ac); return object; } void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) { - void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); @@ -4955,8 +4941,15 @@ EXPORT_SYMBOL(kmem_cache_alloc_noprof); void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, gfp_t gfpflags) { - void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + .lru = lru, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); @@ -4988,7 +4981,14 @@ EXPORT_SYMBOL(kmem_cache_charge); */ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) { - void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); + void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, node, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); @@ -5042,7 +5042,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) if (unlikely(size > s->sheaf_capacity)) { - sheaf = __alloc_empty_sheaf(s, gfp, size); + sheaf = __alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT, size); if (!sheaf) return NULL; @@ -5087,7 +5087,7 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) if (!sheaf) - sheaf = alloc_empty_sheaf(s, gfp); + sheaf = alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT); if (sheaf) { sheaf->capacity = s->sheaf_capacity; @@ -5231,7 +5231,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, struct slab_sheaf *sheaf) { void *ret = NULL; - bool init; + const struct slab_alloc_context ac = { + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; if (sheaf->size == 0) goto out; @@ -5241,10 +5244,8 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, if (likely(!ret)) ret = sheaf->objects[--sheaf->size]; - init = slab_want_init_on_alloc(gfp, s); - /* add __GFP_NOFAIL to force successful memcg charging */ - slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size); + slab_post_alloc_hook(s, gfp | __GFP_NOFAIL, 1, &ret, &ac); out: trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE); @@ -5312,15 +5313,16 @@ void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) EXPORT_SYMBOL(__kmalloc_large_node_noprof); static __always_inline -void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, - unsigned long caller, kmalloc_token_t token) +void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node, + kmalloc_token_t token, const struct slab_alloc_context *ac) { + const size_t size = ac->orig_size; struct kmem_cache *s; void *ret; if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ret = __kmalloc_large_node_noprof(size, flags, node); - trace_kmalloc(caller, ret, size, + trace_kmalloc(ac->caller_addr, ret, size, PAGE_SIZE << get_order(size), flags, node); return ret; } @@ -5330,35 +5332,49 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, s = kmalloc_slab(size, b, flags, token); - ret = slab_alloc_node(s, NULL, flags, node, caller, size); + ret = slab_alloc_node(s, flags, node, ac); ret = kasan_kmalloc(s, ret, size, flags); - trace_kmalloc(caller, ret, size, s->size, flags, node); + trace_kmalloc(ac->caller_addr, ret, size, s->size, flags, node); return ret; } void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node) { - return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, - _RET_IP_, PASS_TOKEN_PARAM(token)); + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node, + PASS_TOKEN_PARAM(token), &ac); } EXPORT_SYMBOL(__kmalloc_node_noprof); void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags) { - return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_, - PASS_TOKEN_PARAM(token)); + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + return __do_kmalloc_node(NULL, flags, NUMA_NO_NODE, + PASS_TOKEN_PARAM(token), &ac); } EXPORT_SYMBOL(__kmalloc_noprof); -void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node) +static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, + int node, const struct slab_alloc_context *ac) { - gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; - size_t orig_size = size; struct kmem_cache *s; bool can_retry = true; void *ret; + VM_WARN_ON_ONCE(alloc_flags_allow_spinning(ac->alloc_flags)); VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | - __GFP_NO_OBJ_EXT)); + __GFP_NOWARN | __GFP_NOMEMALLOC)); + + gfp_flags |= __GFP_NOWARN | __GFP_NOMEMALLOC; if (unlikely(!size)) return ZERO_SIZE_PTR; @@ -5374,10 +5390,12 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) return NULL; + node = apply_strict_numa_policy(node); + retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return NULL; - s = kmalloc_slab(size, NULL, alloc_gfp, PASS_TOKEN_PARAM(token)); + s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token)); if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) /* @@ -5391,17 +5409,17 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in */ return NULL; - ret = alloc_from_pcs(s, alloc_gfp, node); + ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node); if (ret) goto success; /* * Do not call slab_alloc_node(), since trylock mode isn't * compatible with slab_pre_alloc_hook/should_failslab and - * kfence_alloc. Hence call __slab_alloc_node() (at most twice) + * kfence_alloc. Hence call ___slab_alloc() (at most twice) * and slab_post_alloc_hook() directly. */ - ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, orig_size); + ret = ___slab_alloc(s, gfp_flags, node, ac); /* * It's possible we failed due to trylock as we preempted someone with @@ -5414,8 +5432,8 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in size = s->object_size + 1; /* * Another alternative is to - * if (memcg) alloc_gfp &= ~__GFP_ACCOUNT; - * else if (!memcg) alloc_gfp |= __GFP_ACCOUNT; + * if (memcg) gfp_flags &= ~__GFP_ACCOUNT; + * else if (!memcg) gfp_flags |= __GFP_ACCOUNT; * to retry from bucket of the same size. */ can_retry = false; @@ -5424,27 +5442,49 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in success: maybe_wipe_obj_freeptr(s, ret); - slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, - slab_want_init_on_alloc(alloc_gfp, s), orig_size); + slab_post_alloc_hook(s, gfp_flags, 1, &ret, ac); - ret = kasan_kmalloc(s, ret, orig_size, alloc_gfp); + ret = kasan_kmalloc(s, ret, ac->orig_size, gfp_flags); return ret; } + +void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node) +{ + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_NOLOCK, + }; + + return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token), + gfp_flags, node, &ac); +} EXPORT_SYMBOL_GPL(_kmalloc_nolock_noprof); void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node, unsigned long caller) { - return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, - caller, PASS_TOKEN_PARAM(token)); + const struct slab_alloc_context ac = { + .caller_addr = caller, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node, + PASS_TOKEN_PARAM(token), &ac); } EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof); void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) { - void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, - _RET_IP_, size); + void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); @@ -5456,7 +5496,14 @@ EXPORT_SYMBOL(__kmalloc_cache_noprof); void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) { - void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); + void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, node, &ac); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); @@ -5465,6 +5512,30 @@ void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, } EXPORT_SYMBOL(__kmalloc_cache_node_noprof); +/* + * The only version of kmalloc_node() that takes alloc_flags and thus can + * determine on its own whether to handle the allocation via kmalloc_nolock() or + * normally + */ +void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags, + unsigned int alloc_flags, int node) +{ + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = alloc_flags, + }; + + if (alloc_flags_allow_spinning(alloc_flags)) { + return __do_kmalloc_node(NULL, flags, node, + PASS_TOKEN_PARAM(token), &ac); + } else { + return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token), + flags, node, &ac); + } +} + + static noinline void free_to_partial_list( struct kmem_cache *s, struct slab *slab, void *head, void *tail, int bulk_cnt, @@ -5818,7 +5889,7 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, if (!allow_spin) return NULL; - empty = alloc_empty_sheaf(s, GFP_NOWAIT); + empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT); if (empty) goto got_empty; @@ -6002,7 +6073,7 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); - empty = alloc_empty_sheaf(s, GFP_NOWAIT); + empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT); if (!empty) goto fail; @@ -6825,14 +6896,19 @@ void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long { bool allow_block; void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; /* * It doesn't really make sense to fallback to vmalloc for sub page * requests */ - ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), + ret = __do_kmalloc_node(PASS_BUCKET_PARAM(b), kmalloc_gfp_adjust(flags, size), - node, _RET_IP_, PASS_TOKEN_PARAM(token)); + node, PASS_TOKEN_PARAM(token), &ac); if (ret || size <= PAGE_SIZE) return ret; @@ -7222,9 +7298,6 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min, unsigned int refilled; struct slab *slab; - if (WARN_ON_ONCE(!gfpflags_allow_spinning(gfp))) - return 0; - refilled = __refill_objects_node(s, p, gfp, min, max, get_node(s, local_node), /* allow_spin = */ true); @@ -7238,7 +7311,7 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min, new_slab: - slab = new_slab(s, gfp, local_node); + slab = new_slab(s, gfp, SLAB_ALLOC_DEFAULT, local_node); if (!slab) goto out; @@ -7260,10 +7333,14 @@ static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, int i; if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; for (i = 0; i < size; i++) { - p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, &ac); if (unlikely(!p[i])) goto error; @@ -7293,8 +7370,7 @@ static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, * Allocate @size objects from @s and places them into @p. @size must be larger * than 0. * - * Interrupts must be enabled when calling this function and @flags must allow - * spinning. + * Interrupts must be enabled when calling this function. * * Unlike alloc_pages_bulk(), this function does not check for already allocated * objects in @p, and thus the caller does not need to zero it. @@ -7306,6 +7382,10 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, { unsigned int i = 0; void *kfence_obj; + const struct slab_alloc_context ac = { + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; if (!size) return false; @@ -7328,7 +7408,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size--; } - i = alloc_from_pcs_bulk(s, flags, size, p); + i = alloc_from_pcs_bulk(s, size, p); if (i < size) { /* * If we ran out of memory, don't bother with freeing back to @@ -7356,8 +7436,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, out: /* memcg and kmem_cache debug support and memory initialization */ - return likely(slab_post_alloc_hook(s, NULL, flags, size, p, - slab_want_init_on_alloc(flags, s), s->object_size)); + return likely(slab_post_alloc_hook(s, flags, size, p, &ac)); } EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); @@ -7555,7 +7634,7 @@ static int init_percpu_sheaves(struct kmem_cache *s) if (!s->sheaf_capacity) pcs->main = &bootstrap_sheaf; else - pcs->main = alloc_empty_sheaf(s, GFP_KERNEL); + pcs->main = alloc_empty_sheaf(s, GFP_KERNEL, SLAB_ALLOC_DEFAULT); if (!pcs->main) return -ENOMEM; @@ -7583,7 +7662,7 @@ static void early_kmem_cache_node_alloc(int node) BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); - slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); + slab = new_slab(kmem_cache_node, GFP_NOWAIT, SLAB_ALLOC_DEFAULT, node); BUG_ON(!slab); if (slab_nid(slab) != node) { @@ -8421,7 +8500,8 @@ static void __init bootstrap_cache_sheaves(struct kmem_cache *s) pcs = per_cpu_ptr(s->cpu_sheaves, cpu); - pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity); + pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, + SLAB_ALLOC_DEFAULT, capacity); if (!pcs->main) { failed = true; @@ -8888,14 +8968,12 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s, enum slab_stat_type { SL_ALL, /* All slabs */ SL_PARTIAL, /* Only partially allocated slabs */ - SL_CPU, /* Only slabs used for cpu caches */ SL_OBJECTS, /* Determine allocated objects not slabs */ SL_TOTAL /* Determine object capacity not slabs */ }; #define SO_ALL (1 << SL_ALL) #define SO_PARTIAL (1 << SL_PARTIAL) -#define SO_CPU (1 << SL_CPU) #define SO_OBJECTS (1 << SL_OBJECTS) #define SO_TOTAL (1 << SL_TOTAL) @@ -9084,7 +9162,7 @@ SLAB_ATTR_RO(partial); static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) { - return show_slab_objects(s, buf, SO_CPU); + return sysfs_emit(buf, "0\n"); } SLAB_ATTR_RO(cpu_slabs); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 99e2be39671b..785d2b042fe3 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -342,8 +342,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone * * * Any initialization done here will be overwritten by memmap_init(). * - * hugetlb_vmemmap_init() will take care of initialization after - * memmap_init(). + * hugetlb_bootmem_struct_page_init() will take care of initialization + * after memmap_init(). */ p = vmemmap_alloc_block_zero(PAGE_SIZE, node); @@ -544,6 +544,12 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, #endif +#ifndef vmemmap_populate_finalize +static void __meminit vmemmap_populate_finalize(void) +{ +} +#endif + struct page * __meminit __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -564,6 +570,8 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, if (r < 0) return NULL; + vmemmap_populate_finalize(); + return pfn_to_page(pfn); } @@ -579,17 +587,6 @@ void __init sparse_vmemmap_init_nid_early(int nid) { hugetlb_vmemmap_init_early(nid); } - -/* - * This is called just before the initialization of page structures - * through memmap_init. Zones are now initialized, so any work that - * needs to be done that needs zone information can be done from - * here. - */ -void __init sparse_vmemmap_init_nid_late(int nid) -{ - hugetlb_vmemmap_init_late(nid); -} #endif static void subsection_mask_set(unsigned long *map, unsigned long pfn, @@ -601,7 +598,7 @@ static void subsection_mask_set(unsigned long *map, unsigned long pfn, bitmap_set(map, idx, end - idx + 1); } -void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages) +static void __init sparse_init_subsection_map_range(unsigned long pfn, unsigned long nr_pages) { int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); @@ -624,6 +621,15 @@ void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages } } +void __init sparse_init_subsection_map(void) +{ + int i, nid; + unsigned long start, end; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) + sparse_init_subsection_map_range(start, end - start); +} + #ifdef CONFIG_MEMORY_HOTPLUG /* Mark all memory sections within the pfn range as online */ diff --git a/mm/sparse.c b/mm/sparse.c index 16ac6df3c89f..058ef9300367 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -43,7 +43,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; #endif -int memdesc_nid(memdesc_flags_t mdf) +int memdesc_nid(const memdesc_flags_t *mdf) { return section_to_node_table[memdesc_section(mdf)]; } @@ -239,15 +239,8 @@ struct page __init *__populate_section_memmap(unsigned long pfn, struct dev_pagemap *pgmap) { unsigned long size = section_map_size(); - struct page *map; - phys_addr_t addr = __pa(MAX_DMA_ADDRESS); - map = memmap_alloc(size, size, addr, nid, false); - if (!map) - panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", - __func__, size, PAGE_SIZE, nid, &addr); - - return map; + return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ @@ -300,17 +293,14 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, unsigned long map_count) { unsigned long pnum; - struct page *map; - struct mem_section *ms; - if (sparse_usage_init(nid, map_count)) { - pr_err("%s: node[%d] usemap allocation failed", __func__, nid); - goto failed; - } + if (sparse_usage_init(nid, map_count)) + panic("Failed to allocate usemap for node %d\n", nid); sparse_vmemmap_init_nid_early(nid); for_each_present_section_nr(pnum_begin, pnum) { + struct mem_section *ms; unsigned long pfn = section_nr_to_pfn(pnum); if (pnum >= pnum_end) @@ -318,34 +308,18 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin, ms = __nr_to_section(pnum); if (!preinited_vmemmap_section(ms)) { + struct page *map; + map = __populate_section_memmap(pfn, PAGES_PER_SECTION, - nid, NULL, NULL); - if (!map) { - pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", - __func__, nid); - pnum_begin = pnum; - sparse_usage_fini(); - goto failed; - } + nid, NULL, NULL); + if (!map) + panic("Failed to allocate memmap for section %lu\n", pnum); memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), PAGE_SIZE)); sparse_init_early_section(nid, map, pnum, 0); } } sparse_usage_fini(); - return; -failed: - /* - * We failed to allocate, mark all the following pnums as not present, - * except the ones already initialized earlier. - */ - for_each_present_section_nr(pnum_begin, pnum) { - if (pnum >= pnum_end) - break; - ms = __nr_to_section(pnum); - if (!preinited_vmemmap_section(ms)) - ms->section_mem_map = 0; - } } /* @@ -369,9 +343,6 @@ void __init sparse_init(void) pnum_begin = first_present_section_nr(); nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); - /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ - set_pageblock_order(); - for_each_present_section_nr(pnum_begin + 1, pnum_end) { int nid = sparse_early_nid(__nr_to_section(pnum_end)); @@ -387,5 +358,6 @@ void __init sparse_init(void) } /* cover the last node */ sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); + sparse_init_subsection_map(); vmemmap_populate_print_last(); } diff --git a/mm/swap.c b/mm/swap.c index 588f50d8f1a8..58e4eff698cc 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -694,9 +694,12 @@ void lru_add_drain_cpu(int cpu) { struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); struct folio_batch *fbatch = &fbatches->lru_add; + unsigned int nr_folios = folio_batch_count(fbatch); - if (folio_batch_count(fbatch)) + if (nr_folios) { folio_batch_move_lru(fbatch, lru_add); + trace_mm_lru_add_drain_tp(cpu, nr_folios); + } fbatch = &fbatches->lru_move_tail; /* Disabling interrupts below acts as a compiler barrier. */ @@ -828,13 +831,13 @@ static bool cpu_needs_drain(unsigned int cpu) struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu); /* Check these in order of likelihood that they're not zero */ - return folio_batch_count(&fbatches->lru_add) || - folio_batch_count(&fbatches->lru_move_tail) || - folio_batch_count(&fbatches->lru_deactivate_file) || - folio_batch_count(&fbatches->lru_deactivate) || - folio_batch_count(&fbatches->lru_lazyfree) || - folio_batch_count(&fbatches->lru_activate) || - need_mlock_drain(cpu) || + return data_race(folio_batch_count(&fbatches->lru_add) || + folio_batch_count(&fbatches->lru_move_tail) || + folio_batch_count(&fbatches->lru_deactivate_file) || + folio_batch_count(&fbatches->lru_deactivate) || + folio_batch_count(&fbatches->lru_lazyfree) || + folio_batch_count(&fbatches->lru_activate) || + need_mlock_drain(cpu)) || has_bh_in_lru(cpu, NULL); } @@ -869,6 +872,8 @@ static inline void __lru_add_drain_all(bool force_all_cpus) if (WARN_ON(!mm_percpu_wq)) return; + trace_mm_lru_add_drain_all_tp(force_all_cpus); + /* * Guarantee folio_batch counter stores visible by this CPU * are visible to other CPUs before loading the current drain diff --git a/mm/swap.h b/mm/swap.h index 77d2d14eda42..44ab8e1e595b 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -230,8 +230,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); * folio_put_swap(): does the opposite thing of folio_dup_swap(). */ int folio_alloc_swap(struct folio *folio); -int folio_dup_swap(struct folio *folio, struct page *subpage); -void folio_put_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap(struct folio *folio, struct page *page); +void folio_put_swap(struct folio *folio, struct page *page); /* For internal use */ extern void __swap_cluster_free_entries(struct swap_info_struct *si, diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..6fd6e3415b71 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -836,7 +836,6 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, } blk_finish_plug(&plug); swap_read_unplug(splug); - lru_add_drain(); /* Push any new pages onto the LRU now */ skip: /* The page was likely read above, so no need for plugging here */ return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false); @@ -951,7 +950,6 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, pte_unmap(pte); blk_finish_plug(&plug); swap_read_unplug(splug); - lru_add_drain(); skip: /* The folio was likely read above, so no need for plugging here */ folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx, diff --git a/mm/swapfile.c b/mm/swapfile.c index 78b49b0658ad..a602e5820513 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1781,7 +1781,7 @@ int folio_alloc_swap(struct folio *folio) /** * folio_dup_swap() - Increase swap count of swap entries of a folio. * @folio: folio with swap entries bounded. - * @subpage: if not NULL, only increase the swap count of this subpage. + * @page: if not NULL, only increase the swap count of this page. * * Typically called when the folio is unmapped and have its swap entry to * take its place: Swap entries allocated to a folio has count == 0 and pinned @@ -1795,7 +1795,7 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ -int folio_dup_swap(struct folio *folio, struct page *subpage) +int folio_dup_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1803,8 +1803,8 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } @@ -1815,13 +1815,13 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) /** * folio_put_swap() - Decrease swap count of swap entries of a folio. * @folio: folio with swap entries bounded, must be in swap cache and locked. - * @subpage: if not NULL, only decrease the swap count of this subpage. + * @page: if not NULL, only decrease the swap count of this page. * * This won't free the swap slots even if swap count drops to zero, they are * still pinned by the swap cache. User may call folio_free_swap to free them. * Context: Caller must ensure the folio is locked and in the swap cache. */ -void folio_put_swap(struct folio *folio, struct page *subpage) +void folio_put_swap(struct folio *folio, struct page *page) { swp_entry_t entry = folio->swap; unsigned long nr_pages = folio_nr_pages(folio); @@ -1830,8 +1830,8 @@ void folio_put_swap(struct folio *folio, struct page *subpage) VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio); - if (subpage) { - entry.val += folio_page_idx(folio, subpage); + if (page) { + entry.val += folio_page_idx(folio, page); nr_pages = 1; } diff --git a/mm/truncate.c b/mm/truncate.c index 12cc89f89afc..b58ba940be47 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -177,7 +177,7 @@ int truncate_inode_folio(struct address_space *mapping, struct folio *folio) return 0; } -static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at, +static int folio_split_or_unmap(struct folio *folio, struct page *split_at, unsigned long min_order) { enum ttu_flags ttu_flags = @@ -186,7 +186,7 @@ static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at, TTU_IGNORE_MLOCK; int ret; - ret = try_folio_split_to_order(folio, split_at, min_order); + ret = folio_split(folio, min_order, split_at, NULL); /* * If the split fails, unmap the folio, so it will be refaulted @@ -252,7 +252,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) min_order = mapping_min_folio_order(folio->mapping); split_at = folio_page(folio, PAGE_ALIGN_DOWN(offset) / PAGE_SIZE); - if (!try_folio_split_or_unmap(folio, split_at, min_order)) { + if (!folio_split_or_unmap(folio, split_at, min_order)) { /* * try to split at offset + length to make sure folios within * the range can be dropped, especially to avoid memory waste @@ -279,7 +279,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) /* make sure folio2 is large and does not change its mapping */ if (folio_test_large(folio2) && folio2->mapping == folio->mapping) - try_folio_split_or_unmap(folio2, split_at2, min_order); + folio_split_or_unmap(folio2, split_at2, min_order); folio_unlock(folio2); out: diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 246af12bf801..33ef42d5a04e 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -2111,7 +2111,10 @@ static bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); - if (vma->vm_flags & VM_DROPPABLE) + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) + return false; + + if (!is_vm_hugetlb_page(vma) && (vma->vm_flags & VM_SPECIAL)) return false; vm_flags &= __VM_UFFD_FLAGS; @@ -2235,13 +2238,21 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, * userfaultfd and with the right tracking mode too. */ if (vma->vm_userfaultfd_ctx.ctx == ctx && - vma_test_all_mask(vma, vma_flags)) + (vma->vm_flags & __VM_UFFD_FLAGS) == vm_flags) goto skip; if (vma->vm_start > start) start = vma->vm_start; vma_end = min(end, vma->vm_end); + /* + * Re-registering into the same userfaultfd can remove WP mode. + * Clear any per-PTE uffd-wp state before dropping VM_UFFD_WP, + * matching the UFFDIO_UNREGISTER cleanup semantics. + */ + if (userfaultfd_wp(vma) && !(vm_flags & VM_UFFD_WP)) + uffd_wp_range(vma, start, vma_end - start, false); + new_vma_flags = vma->flags; vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS); vma_flags_set_mask(&new_vma_flags, vma_flags); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 1afca3568b9b..1191cda3b4e8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4045,8 +4045,8 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, if (!area) { bool nofail = gfp_mask & __GFP_NOFAIL; warn_alloc(gfp_mask, NULL, - "vmalloc error: size %lu, vm_struct allocation failed%s", - size, (nofail) ? ". Retrying." : ""); + "vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s", + size, align, (nofail) ? ". Retrying." : ""); if (nofail) { schedule_timeout_uninterruptible(1); goto again; @@ -4946,16 +4946,17 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this + * @gfp: allocation flags passed to the underlying memory allocator * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates - * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to - * be scattered pretty far, distance between two areas easily going up - * to gigabytes. To avoid interacting with regular vmallocs, these - * areas are allocated from top. + * congruent vmalloc areas for it. These areas tend to be scattered + * pretty far, distance between two areas easily going up to gigabytes. + * To avoid interacting with regular vmallocs, these areas are allocated + * from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans free blocks from the end looking @@ -4966,7 +4967,7 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) */ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); @@ -5004,14 +5005,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, return NULL; } - vms = kzalloc_objs(vms[0], nr_vms); - vas = kzalloc_objs(vas[0], nr_vms); + vms = kzalloc_objs(vms[0], nr_vms, gfp); + vas = kzalloc_objs(vas[0], nr_vms, gfp); if (!vas || !vms) goto err_free2; for (area = 0; area < nr_vms; area++) { - vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); - vms[area] = kzalloc_obj(struct vm_struct); + vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp); + vms[area] = kzalloc_obj(struct vm_struct, gfp); if (!vas[area] || !vms[area]) goto err_free; } @@ -5101,7 +5102,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, /* populate the kasan shadow space */ for (area = 0; area < nr_vms; area++) { - if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL)) + if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp)) goto err_free_shadow; } @@ -5158,7 +5159,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, continue; vas[area] = kmem_cache_zalloc( - vmap_area_cachep, GFP_KERNEL); + vmap_area_cachep, gfp); if (!vas[area]) goto err_free; } @@ -5198,9 +5199,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, kfree(vms[area]); } spin_unlock(&free_vmap_area_lock); - kfree(vas); - kfree(vms); - return NULL; + goto err_free2; } /** diff --git a/mm/vmpressure.c b/mm/vmpressure.c index f053554e5826..9629240d77ad 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -7,16 +7,15 @@ * * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. + * + * Tree-mode (cgroup v1 userspace eventfd) bookkeeping lives in + * mm/memcontrol-v1.c; this file holds the shared code and the in-kernel + * (tree=false) socket-pressure path that runs on cgroup v2. */ #include -#include #include -#include #include -#include -#include -#include #include #include #include @@ -35,7 +34,7 @@ * TODO: Make the window size depend on machine size, as we do for vmstat * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). */ -static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; +const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; /* * These thresholds are used when we account memory pressure through @@ -46,68 +45,6 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; static const unsigned int vmpressure_level_med = 60; static const unsigned int vmpressure_level_critical = 95; -/* - * When there are too little pages left to scan, vmpressure() may miss the - * critical pressure as number of pages will be less than "window size". - * However, in that case the vmscan priority will raise fast as the - * reclaimer will try to scan LRUs more deeply. - * - * The vmscan logic considers these special priorities: - * - * prio == DEF_PRIORITY (12): reclaimer starts with that value - * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed - * prio == 0 : close to OOM, kernel scans every page in an lru - * - * Any value in this range is acceptable for this tunable (i.e. from 12 to - * 0). Current value for the vmpressure_level_critical_prio is chosen - * empirically, but the number, in essence, means that we consider - * critical level when scanning depth is ~10% of the lru size (vmscan - * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one - * eights). - */ -static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); - -static struct vmpressure *work_to_vmpressure(struct work_struct *work) -{ - return container_of(work, struct vmpressure, work); -} - -static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) -{ - struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr); - - memcg = parent_mem_cgroup(memcg); - if (!memcg) - return NULL; - return memcg_to_vmpressure(memcg); -} - -enum vmpressure_levels { - VMPRESSURE_LOW = 0, - VMPRESSURE_MEDIUM, - VMPRESSURE_CRITICAL, - VMPRESSURE_NUM_LEVELS, -}; - -enum vmpressure_modes { - VMPRESSURE_NO_PASSTHROUGH = 0, - VMPRESSURE_HIERARCHY, - VMPRESSURE_LOCAL, - VMPRESSURE_NUM_MODES, -}; - -static const char * const vmpressure_str_levels[] = { - [VMPRESSURE_LOW] = "low", - [VMPRESSURE_MEDIUM] = "medium", - [VMPRESSURE_CRITICAL] = "critical", -}; - -static const char * const vmpressure_str_modes[] = { - [VMPRESSURE_NO_PASSTHROUGH] = "default", - [VMPRESSURE_HIERARCHY] = "hierarchy", - [VMPRESSURE_LOCAL] = "local", -}; - static enum vmpressure_levels vmpressure_level(unsigned long pressure) { if (pressure >= vmpressure_level_critical) @@ -117,8 +54,8 @@ static enum vmpressure_levels vmpressure_level(unsigned long pressure) return VMPRESSURE_LOW; } -static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, - unsigned long reclaimed) +enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed) { unsigned long scale = scanned + reclaimed; unsigned long pressure = 0; @@ -147,74 +84,6 @@ static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, return vmpressure_level(pressure); } -struct vmpressure_event { - struct eventfd_ctx *efd; - enum vmpressure_levels level; - enum vmpressure_modes mode; - struct list_head node; -}; - -static bool vmpressure_event(struct vmpressure *vmpr, - const enum vmpressure_levels level, - bool ancestor, bool signalled) -{ - struct vmpressure_event *ev; - bool ret = false; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ancestor && ev->mode == VMPRESSURE_LOCAL) - continue; - if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH) - continue; - if (level < ev->level) - continue; - eventfd_signal(ev->efd); - ret = true; - } - mutex_unlock(&vmpr->events_lock); - - return ret; -} - -static void vmpressure_work_fn(struct work_struct *work) -{ - struct vmpressure *vmpr = work_to_vmpressure(work); - unsigned long scanned; - unsigned long reclaimed; - enum vmpressure_levels level; - bool ancestor = false; - bool signalled = false; - - spin_lock(&vmpr->sr_lock); - /* - * Several contexts might be calling vmpressure(), so it is - * possible that the work was rescheduled again before the old - * work context cleared the counters. In that case we will run - * just after the old work returns, but then scanned might be zero - * here. No need for any locks here since we don't care if - * vmpr->reclaimed is in sync. - */ - scanned = vmpr->tree_scanned; - if (!scanned) { - spin_unlock(&vmpr->sr_lock); - return; - } - - reclaimed = vmpr->tree_reclaimed; - vmpr->tree_scanned = 0; - vmpr->tree_reclaimed = 0; - spin_unlock(&vmpr->sr_lock); - - level = vmpressure_calc_level(scanned, reclaimed); - - do { - if (vmpressure_event(vmpr, level, ancestor, signalled)) - signalled = true; - ancestor = true; - } while ((vmpr = vmpressure_parent(vmpr))); -} - /** * vmpressure() - Account memory pressure through scanned/reclaimed ratio * @gfp: reclaimer's gfp mask @@ -246,11 +115,12 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; /* - * The in-kernel users only care about the reclaim efficiency - * for this @memcg rather than the whole subtree, and there - * isn't and won't be any in-kernel user in a legacy cgroup. + * Only two combinations have a consumer: + * cgroup v2 + tree=false -> in-kernel socket pressure + * cgroup v1 + tree=true -> userspace eventfds (memory.pressure_level) + * Skip the other two: nothing consumes the result. */ - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !tree) + if (cgroup_subsys_on_dfl(memory_cgrp_subsys) == tree) return; vmpr = memcg_to_vmpressure(memcg); @@ -281,14 +151,7 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, return; if (tree) { - spin_lock(&vmpr->sr_lock); - scanned = vmpr->tree_scanned += scanned; - vmpr->tree_reclaimed += reclaimed; - spin_unlock(&vmpr->sr_lock); - - if (scanned < vmpressure_win) - return; - schedule_work(&vmpr->work); + vmpressure_v1_account_tree(vmpr, scanned, reclaimed); } else { enum vmpressure_levels level; @@ -330,134 +193,6 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, } } -/** - * vmpressure_prio() - Account memory pressure through reclaimer priority level - * @gfp: reclaimer's gfp mask - * @memcg: cgroup memory controller handle - * @prio: reclaimer's priority - * - * This function should be called from the reclaim path every time when - * the vmscan's reclaiming priority (scanning depth) changes. - * - * This function does not return any value. - */ -void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) -{ - /* - * We only use prio for accounting critical level. For more info - * see comment for vmpressure_level_critical_prio variable above. - */ - if (prio > vmpressure_level_critical_prio) - return; - - /* - * OK, the prio is below the threshold, updating vmpressure - * information before shrinker dives into long shrinking of long - * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 - * to the vmpressure() basically means that we signal 'critical' - * level. - */ - vmpressure(gfp, 0, memcg, true, vmpressure_win, 0); -} - -#define MAX_VMPRESSURE_ARGS_LEN (strlen("critical") + strlen("hierarchy") + 2) - -/** - * vmpressure_register_event() - Bind vmpressure notifications to an eventfd - * @memcg: memcg that is interested in vmpressure notifications - * @eventfd: eventfd context to link notifications with - * @args: event arguments (pressure level threshold, optional mode) - * - * This function associates eventfd context with the vmpressure - * infrastructure, so that the notifications will be delivered to the - * @eventfd. The @args parameter is a comma-delimited string that denotes a - * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium", - * or "critical") and an optional mode (one of vmpressure_str_modes, i.e. - * "hierarchy" or "local"). - * - * To be used as memcg event method. - * - * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could - * not be parsed. - */ -int vmpressure_register_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd, const char *args) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH; - enum vmpressure_levels level; - char *spec, *spec_orig; - char *token; - int ret = 0; - - spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL); - if (!spec) - return -ENOMEM; - - /* Find required level */ - token = strsep(&spec, ","); - ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token); - if (ret < 0) - goto out; - level = ret; - - /* Find optional mode */ - token = strsep(&spec, ","); - if (token) { - ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token); - if (ret < 0) - goto out; - mode = ret; - } - - ev = kzalloc_obj(*ev); - if (!ev) { - ret = -ENOMEM; - goto out; - } - - ev->efd = eventfd; - ev->level = level; - ev->mode = mode; - - mutex_lock(&vmpr->events_lock); - list_add(&ev->node, &vmpr->events); - mutex_unlock(&vmpr->events_lock); - ret = 0; -out: - kfree(spec_orig); - return ret; -} - -/** - * vmpressure_unregister_event() - Unbind eventfd from vmpressure - * @memcg: memcg handle - * @eventfd: eventfd context that was used to link vmpressure with the @cg - * - * This function does internal manipulations to detach the @eventfd from - * the vmpressure notifications, and then frees internal resources - * associated with the @eventfd (but the @eventfd itself is not freed). - * - * To be used as memcg event method. - */ -void vmpressure_unregister_event(struct mem_cgroup *memcg, - struct eventfd_ctx *eventfd) -{ - struct vmpressure *vmpr = memcg_to_vmpressure(memcg); - struct vmpressure_event *ev; - - mutex_lock(&vmpr->events_lock); - list_for_each_entry(ev, &vmpr->events, node) { - if (ev->efd != eventfd) - continue; - list_del(&ev->node); - kfree(ev); - break; - } - mutex_unlock(&vmpr->events_lock); -} - /** * vmpressure_init() - Initialize vmpressure control structure * @vmpr: Structure to be initialized @@ -468,9 +203,7 @@ void vmpressure_unregister_event(struct mem_cgroup *memcg, void vmpressure_init(struct vmpressure *vmpr) { spin_lock_init(&vmpr->sr_lock); - mutex_init(&vmpr->events_lock); - INIT_LIST_HEAD(&vmpr->events); - INIT_WORK(&vmpr->work, vmpressure_work_fn); + vmpressure_v1_init(vmpr); } /** @@ -482,9 +215,5 @@ void vmpressure_init(struct vmpressure *vmpr) */ void vmpressure_cleanup(struct vmpressure *vmpr) { - /* - * Make sure there is no pending work before eventfd infrastructure - * goes away. - */ - flush_work(&vmpr->work); + vmpressure_v1_cleanup(vmpr); } diff --git a/mm/vmscan.c b/mm/vmscan.c index b21a15f36cce..56fe5393f30f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -79,7 +79,7 @@ struct scan_control { * Nodemask of nodes allowed by the caller. If NULL, all nodes * are scanned. */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* * The memory cgroup that hit its limit and as a result is the @@ -612,45 +612,14 @@ typedef enum { PAGE_CLEAN, } pageout_t; -static pageout_t writeout(struct folio *folio, struct address_space *mapping, - struct swap_iocb **plug, struct list_head *folio_list) -{ - int res; - - folio_set_reclaim(folio); - - /* - * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled - * or we failed to allocate contiguous swap entries, in which case - * the split out folios get added back to folio_list. - */ - if (shmem_mapping(mapping)) - res = shmem_writeout(folio, plug, folio_list); - else - res = swap_writeout(folio, plug); - - if (res < 0) - handle_write_error(mapping, folio, res); - if (res == AOP_WRITEPAGE_ACTIVATE) { - folio_clear_reclaim(folio); - return PAGE_ACTIVATE; - } - - /* synchronous write? */ - if (!folio_test_writeback(folio)) - folio_clear_reclaim(folio); - - trace_mm_vmscan_write_folio(folio); - node_stat_add_folio(folio, NR_VMSCAN_WRITE); - return PAGE_SUCCESS; -} - /* * pageout is called by shrink_folio_list() for each dirty folio. */ static pageout_t pageout(struct folio *folio, struct address_space *mapping, struct swap_iocb **plug, struct list_head *folio_list) { + int res; + /* * We no longer attempt to writeback filesystem folios here, other * than tmpfs/shmem. That's taken care of in page-writeback. @@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, return PAGE_ACTIVATE; if (!folio_clear_dirty_for_io(folio)) return PAGE_CLEAN; - return writeout(folio, mapping, plug, folio_list); + + folio_set_reclaim(folio); + + /* + * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled + * or we failed to allocate contiguous swap entries, in which case + * the split out folios get added back to folio_list. + */ + if (shmem_mapping(mapping)) + res = shmem_writeout(folio, plug, folio_list); + else + res = swap_writeout(folio, plug); + + if (res < 0) + handle_write_error(mapping, folio, res); + if (res == AOP_WRITEPAGE_ACTIVATE) { + folio_clear_reclaim(folio); + return PAGE_ACTIVATE; + } + + /* synchronous write? */ + if (!folio_test_writeback(folio)) + folio_clear_reclaim(folio); + + trace_mm_vmscan_write_folio(folio); + node_stat_add_folio(folio, NR_VMSCAN_WRITE); + return PAGE_SUCCESS; } /* @@ -828,7 +823,6 @@ void folio_putback_lru(struct folio *folio) enum folio_references { FOLIOREF_RECLAIM, - FOLIOREF_RECLAIM_CLEAN, FOLIOREF_KEEP, FOLIOREF_ACTIVATE, }; @@ -925,10 +919,6 @@ static enum folio_references folio_check_references(struct folio *folio, return FOLIOREF_KEEP; } - /* Reclaim if clean, defer dirty folios to writeback */ - if (referenced_folio && folio_is_file_lru(folio)) - return FOLIOREF_RECLAIM_CLEAN; - return FOLIOREF_RECLAIM; } @@ -1240,7 +1230,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, stat->nr_ref_keep += nr_pages; goto keep_locked; case FOLIOREF_RECLAIM: - case FOLIOREF_RECLAIM_CLEAN: ; /* try to reclaim the folio below */ } @@ -1386,8 +1375,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, goto activate_locked; } - if (references == FOLIOREF_RECLAIM_CLEAN) - goto keep_locked; if (!may_enter_fs(folio, sc->gfp_mask)) goto keep_locked; if (!sc->may_writepage) @@ -6599,7 +6586,7 @@ static bool allow_direct_reclaim(pg_data_t *pgdat) * happens, the page allocator should not consider triggering the OOM killer. */ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, - nodemask_t *nodemask) + const nodemask_t *nodemask) { struct zoneref *z; struct zone *zone; @@ -6679,7 +6666,7 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, } unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *nodemask) + gfp_t gfp_mask, const nodemask_t *nodemask) { unsigned long nr_reclaimed; struct scan_control sc = { @@ -6711,11 +6698,11 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, return 1; set_task_reclaim_state(current, &sc.reclaim_state); - trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, 0); + trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, NULL); nr_reclaimed = do_try_to_free_pages(zonelist, &sc); - trace_mm_vmscan_direct_reclaim_end(nr_reclaimed, 0); + trace_mm_vmscan_direct_reclaim_end(nr_reclaimed, NULL); set_task_reclaim_state(current, NULL); return nr_reclaimed; @@ -7781,7 +7768,7 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, delayacct_freepages_end(); psi_memstall_leave(&pflags); - trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed, 0); + trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed, NULL); return sc->nr_reclaimed; } @@ -7917,6 +7904,10 @@ int user_proactive_reclaim(char *buf, if (signal_pending(current)) return -EINTR; + /* cgroup_rmdir() waits for us with cgroup_mutex held. */ + if (memcg && memcg_is_dying(memcg)) + return -EAGAIN; + /* * This is the final attempt, drain percpu lru caches in the * hope of introducing more evictable pages. diff --git a/mm/vmstat.c b/mm/vmstat.c index f534972f517d..7b93fbf9af09 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1568,7 +1568,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, static int frag_show(struct seq_file *m, void *arg) { pg_data_t *pgdat = (pg_data_t *)arg; - walk_zones_in_node(m, pgdat, true, false, frag_show_print); + walk_zones_in_node(m, pgdat, true, true, frag_show_print); return 0; } diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 63128ddb7959..3d566a3ee96f 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -21,6 +21,10 @@ * pool->lock * class->lock * zspage->lock + * + * When ZS_OBJ_CLASS_BITS > 0, zs_free() skips pool->lock; it picks + * the size_class from obj's encoded class_idx and serializes against + * page migration via class->lock. */ #include @@ -67,8 +71,8 @@ #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS #else /* - * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just - * be PAGE_SHIFT + * If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will + * just be PAGE_SHIFT */ #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG #endif @@ -88,8 +92,23 @@ #define OBJ_TAG_BITS 1 #define OBJ_TAG_MASK OBJ_ALLOCATED_TAG -#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS) -#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1) +/* + * obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long: + * + * |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->| + * +-----------------+-------------------------+-----------------------+ + * | PFN | class_idx | obj_idx | + * +-----------------+-------------------------+-----------------------+ + * MSB ^ LSB + * | + * +-- ZS_OBJ_PFN_SHIFT + * + * Encoding class_idx into obj lets zs_free() locate the size_class + * without holding pool->lock; class_idx is invariant across page + * migration (only PFN changes), so a lockless read of the obj value + * always yields a valid class_idx. + */ +#define ZS_OBJ_PFN_SHIFT (BITS_PER_LONG - _PFN_BITS) #define HUGE_BITS 1 #define FULLNESS_BITS 4 @@ -98,9 +117,61 @@ #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL)) +/* + * Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)). + * Computed at preprocessor time, for use in #if below. Kconfig + * restricts ZSMALLOC_CHAIN_SIZE to [4, 16]. + */ +#if ZS_MAX_PAGES_PER_ZSPAGE <= 4 +#define ZS_PAGES_PER_ZSPAGE_BITS 2 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8 +#define ZS_PAGES_PER_ZSPAGE_BITS 3 +#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16 +#define ZS_PAGES_PER_ZSPAGE_BITS 4 +#else +#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]" +#endif + +/* + * Bits to index an object within a single PAGE_SIZE at the smallest + * possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5. + * 32 is the hard floor of ZS_MIN_ALLOC_SIZE. + */ +#define ZS_OBJS_PER_PAGE_BITS (PAGE_SHIFT - 5) + +/* + * Bits to index any object in the densest possible zspage. Below this, + * ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still + * correct, but objects are coarser. + */ +#define ZS_OBJS_PER_ZSPAGE_BITS \ + (ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS) + +/* + * Encode class_idx only when obj has spare bits; otherwise + * ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback). + */ +#if BITS_PER_LONG >= 64 && \ + ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS +#define ZS_OBJ_CLASS_BITS (CLASS_BITS + 1) +#else +#define ZS_OBJ_CLASS_BITS 0 +#endif +#define ZS_OBJ_CLASS_MASK ((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1) + +#define ZS_OBJ_IDX_BITS (ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS) +#define ZS_OBJ_IDX_MASK ((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1) + +/* + * Belt-and-suspenders: the #if above already guarantees this when + * class_idx is enabled. Catches future tweaks that bypass it. + */ +static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS, + "zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE"); + /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */ #define ZS_MIN_ALLOC_SIZE \ - MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS)) + MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS)) /* each chunk includes extra space to keep handle */ #define ZS_MAX_ALLOC_SIZE PAGE_SIZE @@ -396,10 +467,13 @@ static void cache_free_zspage(struct zspage *zspage) kmem_cache_free(zspage_cachep, zspage); } -/* class->lock(which owns the handle) synchronizes races */ +/* + * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the + * handle locklessly, so prevent store tearing here. + */ static void record_obj(unsigned long handle, unsigned long obj) { - *(unsigned long *)handle = obj; + WRITE_ONCE(*(unsigned long *)handle, obj); } static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc) @@ -565,8 +639,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v) total_freeable += freeable; } - seq_puts(s, "\n"); - seq_printf(s, " %5s %5s ", "Total", ""); + seq_printf(s, "\n %5s %5s ", "Total", ""); for (fg = ZS_INUSE_RATIO_10; fg < NR_FULLNESS_GROUPS; fg++) seq_printf(s, "%9lu ", inuse_totals[fg]); @@ -721,33 +794,36 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc) static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc, unsigned int *obj_idx) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); - *obj_idx = (obj & OBJ_INDEX_MASK); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); + *obj_idx = (obj & ZS_OBJ_IDX_MASK); } static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc) { - *zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS); + *zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT); } /** - * location_to_obj - get obj value encoded from (, ) + * location_to_obj - encode (, , ) into obj value * @zpdesc: zpdesc object resides in zspage * @obj_idx: object index + * @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0 */ -static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx) +static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx, + unsigned int class_idx) { unsigned long obj; - obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS; - obj |= obj_idx & OBJ_INDEX_MASK; + obj = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT; + obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS; + obj |= obj_idx & ZS_OBJ_IDX_MASK; return obj; } static unsigned long handle_to_obj(unsigned long handle) { - return *(unsigned long *)handle; + return READ_ONCE(*(unsigned long *)handle); } static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj, @@ -801,13 +877,26 @@ static int trylock_zspage(struct zspage *zspage) return 0; } -static void __free_zspage(struct zs_pool *pool, struct size_class *class, - struct zspage *zspage) +/* + * Three free helpers, kept apart here: + * + * __free_zspage_lockless(): bare core; walks zpdescs and returns pages + * to the buddy allocator. Caller owns all zpdesc locks and has + * removed the zspage from its class list. Used by zs_free() outside + * class->lock so the buddy-side work does not stall the class. + * + * __free_zspage(): __free_zspage_lockless() + per-class accounting, + * under class->lock. Used by async_free_zspage(), the worker for + * zspages whose trylock_zspage() failed. + * + * free_zspage(): full wrapper - trylock zpdescs, remove from class + * list, call __free_zspage(); kicks deferred free on contention. + * Used by compaction. + */ +static inline void __free_zspage_lockless(struct zspage *zspage) { struct zpdesc *zpdesc, *next; - assert_spin_locked(&class->lock); - VM_BUG_ON(get_zspage_inuse(zspage)); VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0); @@ -823,7 +912,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, } while (zpdesc != NULL); cache_free_zspage(zspage); +} +static void __free_zspage(struct zs_pool *pool, struct size_class *class, + struct zspage *zspage) +{ + assert_spin_locked(&class->lock); + __free_zspage_lockless(zspage); class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1276,7 +1371,7 @@ static unsigned long obj_malloc(struct zs_pool *pool, kunmap_local(vaddr); mod_zspage_inuse(zspage, 1); - obj = location_to_obj(m_zpdesc, obj); + obj = location_to_obj(m_zpdesc, obj, zspage->class); record_obj(handle, obj); return obj; @@ -1381,37 +1476,97 @@ static void obj_free(int class_size, unsigned long obj) mod_zspage_inuse(zspage, -1); } +#if (ZS_OBJ_CLASS_BITS > 0) || defined(CONFIG_COMPACTION) +/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */ +static unsigned int obj_to_class_idx(unsigned long obj) +{ + return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK; +} +#endif + +/* + * Resolve @handle to its zspage / size_class and acquire class->lock. + * + * When class_idx is encoded in obj (ZS_OBJ_CLASS_BITS > 0), it is + * invariant under page migration, so the handle can be read locklessly + * to pick the size_class. Once class->lock is held migration is + * blocked and the handle is re-read to obtain a stable PFN. + * + * Otherwise (32-bit, or 64-bit fallback paths like UML where the + * encoding is disabled), fall back to pool->lock for the lookup. + */ +#if ZS_OBJ_CLASS_BITS > 0 +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + obj = handle_to_obj(handle); + *classp = pool->size_class[obj_to_class_idx(obj)]; + spin_lock(&(*classp)->lock); + /* Re-read under class->lock: PFN is now stable vs migration. */ + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *objp = obj; +} +#else +static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, + unsigned long *objp, struct zspage **zspagep, + struct size_class **classp) + __acquires(&(*classp)->lock) +{ + struct zpdesc *f_zpdesc; + unsigned long obj; + + read_lock(&pool->lock); + obj = handle_to_obj(handle); + obj_to_zpdesc(obj, &f_zpdesc); + *zspagep = get_zspage(f_zpdesc); + *classp = zspage_class(pool, *zspagep); + spin_lock(&(*classp)->lock); + read_unlock(&pool->lock); + *objp = obj; +} +#endif + void zs_free(struct zs_pool *pool, unsigned long handle) { struct zspage *zspage; - struct zpdesc *f_zpdesc; unsigned long obj; struct size_class *class; int fullness; + struct zspage *zspage_to_free = NULL; if (IS_ERR_OR_NULL((void *)handle)) return; - /* - * The pool->lock protects the race with zpage's migration - * so it's safe to get the page from handle. - */ - read_lock(&pool->lock); - obj = handle_to_obj(handle); - obj_to_zpdesc(obj, &f_zpdesc); - zspage = get_zspage(f_zpdesc); - class = zspage_class(pool, zspage); - spin_lock(&class->lock); - read_unlock(&pool->lock); + obj_class_get_and_lock(pool, handle, &obj, &zspage, &class); class_stat_sub(class, ZS_OBJS_INUSE, 1); obj_free(class->size, obj); fullness = fix_fullness_group(class, zspage); - if (fullness == ZS_INUSE_RATIO_0) - free_zspage(pool, class, zspage); + if (fullness == ZS_INUSE_RATIO_0) { + if (trylock_zspage(zspage)) { + remove_zspage(class, zspage); + class_stat_sub(class, ZS_OBJS_ALLOCATED, + class->objs_per_zspage); + zspage_to_free = zspage; + } else { + kick_deferred_free(pool); + } + } spin_unlock(&class->lock); + + if (zspage_to_free) { + __free_zspage_lockless(zspage_to_free); + atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); + } cache_free_handle(handle); } EXPORT_SYMBOL_GPL(zs_free); @@ -1644,9 +1799,6 @@ static void lock_zspage(struct zspage *zspage) } zspage_read_unlock(zspage); } -#endif /* CONFIG_COMPACTION */ - -#ifdef CONFIG_COMPACTION static void replace_sub_page(struct size_class *class, struct zspage *zspage, struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc) @@ -1762,7 +1914,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page, old_obj = handle_to_obj(handle); obj_to_location(old_obj, &dummy, &obj_idx); - new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx); + new_obj = location_to_obj(newzpdesc, obj_idx, + obj_to_class_idx(old_obj)); record_obj(handle, new_obj); } } diff --git a/mm/zswap.c b/mm/zswap.c index 761cd699e0a3..b5a17ea20237 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1217,7 +1217,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker, * Without memcg, use the zswap pool-wide metrics. */ if (!mem_cgroup_disabled()) { - mem_cgroup_flush_stats(memcg); + mem_cgroup_flush_stats_ratelimited(memcg); nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); } else { diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 2b74ed56eb16..2d2efb877975 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -77,9 +77,9 @@ static int vlan_group_prealloc_vid(struct vlan_group *vg, return 0; } -static void vlan_stacked_transfer_operstate(const struct net_device *rootdev, - struct net_device *dev, - struct vlan_dev_priv *vlan) +void vlan_stacked_transfer_operstate(const struct net_device *rootdev, + struct net_device *dev, + struct vlan_dev_priv *vlan) { if (!(vlan->flags & VLAN_FLAG_BRIDGE_BINDING)) netif_stacked_transfer_operstate(rootdev, dev); @@ -316,29 +316,6 @@ static void vlan_sync_address(struct net_device *dev, ether_addr_copy(vlan->real_dev_addr, dev->dev_addr); } -static void vlan_transfer_features(struct net_device *dev, - struct net_device *vlandev) -{ - struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); - - netif_inherit_tso_max(vlandev, dev); - - if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) - vlandev->hard_header_len = dev->hard_header_len; - else - vlandev->hard_header_len = dev->hard_header_len + VLAN_HLEN; - -#if IS_ENABLED(CONFIG_FCOE) - vlandev->fcoe_ddp_xid = dev->fcoe_ddp_xid; -#endif - - vlandev->priv_flags &= ~IFF_XMIT_DST_RELEASE; - vlandev->priv_flags |= (vlan->real_dev->priv_flags & IFF_XMIT_DST_RELEASE); - vlandev->hw_enc_features = vlan_tnl_features(vlan->real_dev); - - netdev_update_features(vlandev); -} - static int __vlan_device_event(struct net_device *dev, unsigned long event) { int err = 0; @@ -391,13 +368,11 @@ static void vlan_vid0_del(struct net_device *dev) static int vlan_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { - struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct vlan_group *grp; struct vlan_info *vlan_info; int i, flgs; struct net_device *vlandev; - struct vlan_dev_priv *vlan; bool last = false; LIST_HEAD(list); int err; @@ -447,54 +422,19 @@ static int vlan_device_event(struct notifier_block *unused, unsigned long event, if (vlandev->mtu <= dev->mtu) continue; - dev_set_mtu(vlandev, dev->mtu); + netdev_work_sched(vlandev, VLAN_WORK_MTU); } break; case NETDEV_FEAT_CHANGE: - /* Propagate device features to underlying device */ vlan_group_for_each_dev(grp, i, vlandev) - vlan_transfer_features(dev, vlandev); + netdev_work_sched(vlandev, VLAN_WORK_FEATURES); break; - case NETDEV_DOWN: { - struct net_device *tmp; - LIST_HEAD(close_list); - - /* Put all VLANs for this dev in the down state too. */ - vlan_group_for_each_dev(grp, i, vlandev) { - flgs = vlandev->flags; - if (!(flgs & IFF_UP)) - continue; - - vlan = vlan_dev_priv(vlandev); - if (!(vlan->flags & VLAN_FLAG_LOOSE_BINDING)) - list_add(&vlandev->close_list, &close_list); - } - - netif_close_many(&close_list, false); - - list_for_each_entry_safe(vlandev, tmp, &close_list, close_list) { - vlan_stacked_transfer_operstate(dev, vlandev, - vlan_dev_priv(vlandev)); - list_del_init(&vlandev->close_list); - } - list_del(&close_list); - break; - } + case NETDEV_DOWN: case NETDEV_UP: - /* Put all VLANs for this dev in the up state too. */ - vlan_group_for_each_dev(grp, i, vlandev) { - flgs = netif_get_flags(vlandev); - if (flgs & IFF_UP) - continue; - - vlan = vlan_dev_priv(vlandev); - if (!(vlan->flags & VLAN_FLAG_LOOSE_BINDING)) - dev_change_flags(vlandev, flgs | IFF_UP, - extack); - vlan_stacked_transfer_operstate(dev, vlandev, vlan); - } + vlan_group_for_each_dev(grp, i, vlandev) + netdev_work_sched(vlandev, VLAN_WORK_LINK_STATE); break; case NETDEV_UNREGISTER: diff --git a/net/8021q/vlan.h b/net/8021q/vlan.h index c7ffe591d593..c41caaf94095 100644 --- a/net/8021q/vlan.h +++ b/net/8021q/vlan.h @@ -125,6 +125,17 @@ static inline netdev_features_t vlan_tnl_features(struct net_device *real_dev) int vlan_filter_push_vids(struct vlan_info *vlan_info, __be16 proto); void vlan_filter_drop_vids(struct vlan_info *vlan_info, __be16 proto); +/* netdev_work events propagated from the real device, see vlan_dev_work(). */ +enum { + VLAN_WORK_LINK_STATE = BIT(0), /* sync up/down with real_dev */ + VLAN_WORK_MTU = BIT(1), /* clamp mtu to real_dev's */ + VLAN_WORK_FEATURES = BIT(2), /* re-inherit real_dev features */ +}; + +void vlan_stacked_transfer_operstate(const struct net_device *rootdev, + struct net_device *dev, + struct vlan_dev_priv *vlan); + /* found in vlan_dev.c */ void vlan_dev_set_ingress_priority(const struct net_device *dev, u32 skb_prio, u16 vlan_prio); diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 7aa3af8b10ea..ec2569b3f8da 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -270,6 +270,9 @@ static int vlan_dev_open(struct net_device *dev) !(vlan->flags & VLAN_FLAG_LOOSE_BINDING)) return -ENETDOWN; + /* The explicit open supersedes any deferred link-state sync */ + netdev_work_cancel(dev, VLAN_WORK_LINK_STATE); + if (!ether_addr_equal(dev->dev_addr, real_dev->dev_addr) && !vlan_dev_inherit_address(dev, real_dev)) { err = dev_uc_add(real_dev, dev->dev_addr); @@ -300,6 +303,9 @@ static int vlan_dev_stop(struct net_device *dev) struct vlan_dev_priv *vlan = vlan_dev_priv(dev); struct net_device *real_dev = vlan->real_dev; + /* The explicit close supersedes any deferred link-state sync */ + netdev_work_cancel(dev, VLAN_WORK_LINK_STATE); + dev_mc_unsync(real_dev, dev); dev_uc_unsync(real_dev, dev); @@ -1016,6 +1022,59 @@ static const struct ethtool_ops vlan_ethtool_ops = { .get_ts_info = vlan_ethtool_get_ts_info, }; +static void vlan_transfer_features(struct net_device *dev, + struct net_device *vlandev) +{ + struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); + + netif_inherit_tso_max(vlandev, dev); + + if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) + vlandev->hard_header_len = dev->hard_header_len; + else + vlandev->hard_header_len = dev->hard_header_len + VLAN_HLEN; + +#if IS_ENABLED(CONFIG_FCOE) + vlandev->fcoe_ddp_xid = dev->fcoe_ddp_xid; +#endif + + vlandev->priv_flags &= ~IFF_XMIT_DST_RELEASE; + vlandev->priv_flags |= (vlan->real_dev->priv_flags & IFF_XMIT_DST_RELEASE); + vlandev->hw_enc_features = vlan_tnl_features(vlan->real_dev); + + netdev_update_features(vlandev); +} + +static void vlan_dev_work(struct net_device *vlandev, unsigned long events) +{ + struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); + struct net_device *real_dev = vlan->real_dev; + bool loose = vlan->flags & VLAN_FLAG_LOOSE_BINDING; + unsigned int flgs; + + if (events & VLAN_WORK_LINK_STATE) { + flgs = netif_get_flags(vlandev); + if (real_dev->flags & IFF_UP) { + if (!(flgs & IFF_UP)) { + if (!loose) + netif_change_flags(vlandev, + flgs | IFF_UP, NULL); + vlan_stacked_transfer_operstate(real_dev, + vlandev, vlan); + } + } else if ((flgs & IFF_UP) && !loose) { + netif_change_flags(vlandev, flgs & ~IFF_UP, NULL); + vlan_stacked_transfer_operstate(real_dev, vlandev, vlan); + } + } + + if ((events & VLAN_WORK_MTU) && vlandev->mtu > real_dev->mtu) + netif_set_mtu(vlandev, real_dev->mtu); + + if (events & VLAN_WORK_FEATURES) + vlan_transfer_features(real_dev, vlandev); +} + static const struct net_device_ops vlan_netdev_ops = { .ndo_change_mtu = vlan_dev_change_mtu, .ndo_init = vlan_dev_init, @@ -1027,6 +1086,7 @@ static const struct net_device_ops vlan_netdev_ops = { .ndo_set_mac_address = vlan_dev_set_mac_address, .ndo_set_rx_mode = vlan_dev_set_rx_mode, .ndo_change_rx_flags = vlan_dev_change_rx_flags, + .ndo_work = vlan_dev_work, .ndo_eth_ioctl = vlan_dev_ioctl, .ndo_neigh_setup = vlan_dev_neigh_setup, .ndo_get_stats64 = vlan_dev_get_stats64, diff --git a/net/9p/client.c b/net/9p/client.c index f0dcf252af7e..ef64546c6d52 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -600,6 +600,8 @@ p9_client_rpc(struct p9_client *c, int8_t type, const char *fmt, ...) if (err == -ERESTARTSYS && c->status == Connected && type == P9_TFLUSH) { + if (fatal_signal_pending(current)) + goto recalc_sigpending; sigpending = 1; clear_thread_flag(TIF_SIGPENDING); goto again; @@ -765,7 +767,7 @@ static void p9_fid_destroy(struct p9_fid *fid) spin_lock_irqsave(&clnt->lock, flags); idr_remove(&clnt->fids, fid->fid); spin_unlock_irqrestore(&clnt->lock, flags); - kfree(fid->rdir); + kvfree(fid->rdir); kfree(fid); } @@ -1092,7 +1094,8 @@ struct p9_fid *p9_client_walk(struct p9_fid *oldfid, uint16_t nwname, clunk_fid: kfree(wqids); - p9_fid_put(fid); + if (fid != oldfid) + p9_fid_put(fid); fid = NULL; error: diff --git a/net/9p/trans_fd.c b/net/9p/trans_fd.c index dbad3213ba84..eb685b52aeb2 100644 --- a/net/9p/trans_fd.c +++ b/net/9p/trans_fd.c @@ -940,14 +940,12 @@ p9_fd_create_unix(struct p9_client *client, struct fs_context *fc) if (!addr || !strlen(addr)) return -EINVAL; - if (strlen(addr) >= UNIX_PATH_MAX) { + sun_server.sun_family = PF_UNIX; + if (strscpy(sun_server.sun_path, addr) < 0) { pr_err("%s (%d): address too long: %s\n", __func__, task_pid_nr(current), addr); return -ENAMETOOLONG; } - - sun_server.sun_family = PF_UNIX; - strcpy(sun_server.sun_path, addr); err = __sock_create(current->nsproxy->net_ns, PF_UNIX, SOCK_STREAM, 0, &csocket, 1); if (err < 0) { diff --git a/net/9p/trans_rdma.c b/net/9p/trans_rdma.c index aa5bd74d333f..b4274f10fa44 100644 --- a/net/9p/trans_rdma.c +++ b/net/9p/trans_rdma.c @@ -128,25 +128,36 @@ p9_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct p9_client *c = id->context; struct p9_trans_rdma *rdma = c->trans; + unsigned long flags; + switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_INIT); rdma->state = P9_RDMA_ADDR_RESOLVED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_ROUTE_RESOLVED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_ADDR_RESOLVED); rdma->state = P9_RDMA_ROUTE_RESOLVED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_ESTABLISHED: + spin_lock_irqsave(&rdma->req_lock, flags); BUG_ON(rdma->state != P9_RDMA_ROUTE_RESOLVED); rdma->state = P9_RDMA_CONNECTED; + spin_unlock_irqrestore(&rdma->req_lock, flags); break; case RDMA_CM_EVENT_DISCONNECTED: - if (rdma) + if (rdma) { + spin_lock_irqsave(&rdma->req_lock, flags); rdma->state = P9_RDMA_CLOSED; + spin_unlock_irqrestore(&rdma->req_lock, flags); + } c->status = Disconnected; break; @@ -184,6 +195,7 @@ recv_done(struct ib_cq *cq, struct ib_wc *wc) struct p9_req_t *req; int err = 0; int16_t tag; + unsigned long flags; req = NULL; ib_dma_unmap_single(rdma->cm_id->device, c->busa, client->msize, @@ -220,7 +232,10 @@ recv_done(struct ib_cq *cq, struct ib_wc *wc) err_out: p9_debug(P9_DEBUG_ERROR, "req %p err %d status %d\n", req, err, wc->status); - rdma->state = P9_RDMA_FLUSHING; + spin_lock_irqsave(&rdma->req_lock, flags); + if (rdma->state < P9_RDMA_FLUSHING) + rdma->state = P9_RDMA_FLUSHING; + spin_unlock_irqrestore(&rdma->req_lock, flags); client->status = Disconnected; goto out; } diff --git a/net/9p/trans_usbg.c b/net/9p/trans_usbg.c index 1ce70338999c..419cda13a7b5 100644 --- a/net/9p/trans_usbg.c +++ b/net/9p/trans_usbg.c @@ -804,7 +804,7 @@ static void usb9pfs_attr_release(struct config_item *item) usb_put_function_instance(&usb9pfs_opts->func_inst); } -static struct configfs_item_operations usb9pfs_item_ops = { +static const struct configfs_item_operations usb9pfs_item_ops = { .release = usb9pfs_attr_release, }; diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index 4cdab7094b27..b0d0094ec8e2 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -532,6 +532,11 @@ p9_virtio_zc_request(struct p9_client *client, struct p9_req_t *req, p9_debug(P9_DEBUG_TRANS, "virtio request kicked\n"); err = io_wait_event_killable(req->wq, READ_ONCE(req->status) >= REQ_STATUS_RCVD); + /* + * Make sure our req is coherent with regard to updates in other + * threads - echoes to wmb() in the callback + */ + smp_rmb(); // RERROR needs reply (== error string) in static data if (READ_ONCE(req->status) == REQ_STATUS_RCVD && unlikely(req->rc.sdata[4] == P9_RERROR)) diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c index 7588e64e7ba6..22622283f59b 100644 --- a/net/batman-adv/bat_iv_ogm.c +++ b/net/batman-adv/bat_iv_ogm.c @@ -316,14 +316,23 @@ batadv_iv_ogm_aggr_packet(int buff_pos, int packet_len, const struct batadv_ogm_packet *ogm_packet) { int next_buff_pos = 0; + u16 tvlv_len; /* check if there is enough space for the header */ next_buff_pos += buff_pos + sizeof(*ogm_packet); if (next_buff_pos > packet_len) return false; + tvlv_len = ntohs(ogm_packet->tvlv_len); + + /* the fields of an aggregated OGM are accessed assuming (at least) + * 2-byte alignment, so a following OGM must start at an even offset. + */ + if (tvlv_len & 1) + return false; + /* check if there is enough space for the optional TVLV */ - next_buff_pos += ntohs(ogm_packet->tvlv_len); + next_buff_pos += tvlv_len; return next_buff_pos <= packet_len; } @@ -395,23 +404,14 @@ static void batadv_iv_ogm_send_to_if(struct batadv_forw_packet *forw_packet, /* send a batman ogm packet */ static void batadv_iv_ogm_emit(struct batadv_forw_packet *forw_packet) { - struct net_device *mesh_iface; - if (!forw_packet->if_incoming) { pr_err("Error - can't forward packet: incoming iface not specified\n"); return; } - mesh_iface = forw_packet->if_incoming->mesh_iface; - if (WARN_ON(!forw_packet->if_outgoing)) return; - if (forw_packet->if_outgoing->mesh_iface != mesh_iface) { - pr_warn("%s: mesh interface switch for queued OGM\n", __func__); - return; - } - if (forw_packet->if_incoming->if_status != BATADV_IF_ACTIVE) return; @@ -901,8 +901,7 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface) static void batadv_iv_ogm_schedule(struct batadv_hard_iface *hard_iface) { - if (hard_iface->if_status == BATADV_IF_NOT_IN_USE || - hard_iface->if_status == BATADV_IF_TO_BE_REMOVED) + if (hard_iface->if_status == BATADV_IF_TO_BE_REMOVED) return; mutex_lock(&hard_iface->bat_iv.ogm_buff_mutex); diff --git a/net/batman-adv/bat_v.c b/net/batman-adv/bat_v.c index fe7c0113d0df..db6f5bdcaa98 100644 --- a/net/batman-adv/bat_v.c +++ b/net/batman-adv/bat_v.c @@ -817,6 +817,7 @@ void batadv_v_hardif_init(struct batadv_hard_iface *hard_iface) hard_iface->bat_v.aggr_len = 0; skb_queue_head_init(&hard_iface->bat_v.aggr_list); + hard_iface->bat_v.aggr_list_enabled = false; INIT_DELAYED_WORK(&hard_iface->bat_v.aggr_wq, batadv_v_ogm_aggr_work); /* make sure it doesn't run until interface gets enabled */ diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c index 4841f0f1a9b1..262e40040007 100644 --- a/net/batman-adv/bat_v_elp.c +++ b/net/batman-adv/bat_v_elp.c @@ -90,12 +90,6 @@ static bool batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh, u32 throughput; int ret; - /* don't query throughput when no longer associated with any - * batman-adv interface - */ - if (!mesh_iface) - return false; - /* if the user specified a customised value for this interface, then * return it directly */ @@ -311,8 +305,7 @@ static void batadv_v_elp_periodic_work(struct work_struct *work) goto out; /* we are in the process of shutting this interface down */ - if (hard_iface->if_status == BATADV_IF_NOT_IN_USE || - hard_iface->if_status == BATADV_IF_TO_BE_REMOVED) + if (hard_iface->if_status == BATADV_IF_TO_BE_REMOVED) goto out; /* the interface was enabled but may not be ready yet */ diff --git a/net/batman-adv/bat_v_ogm.c b/net/batman-adv/bat_v_ogm.c index 81926ef9c02c..e921d49f7ece 100644 --- a/net/batman-adv/bat_v_ogm.c +++ b/net/batman-adv/bat_v_ogm.c @@ -115,14 +115,14 @@ static void batadv_v_ogm_start_timer(struct batadv_priv *bat_priv) /** * batadv_v_ogm_send_to_if() - send a batman ogm using a given interface - * @bat_priv: the bat priv with all the mesh interface information * @skb: the OGM to send * @hard_iface: the interface to use to send the OGM */ -static void batadv_v_ogm_send_to_if(struct batadv_priv *bat_priv, - struct sk_buff *skb, +static void batadv_v_ogm_send_to_if(struct sk_buff *skb, struct batadv_hard_iface *hard_iface) { + struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); + if (hard_iface->if_status != BATADV_IF_ACTIVE) { kfree_skb(skb); return; @@ -189,7 +189,6 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface) /** * batadv_v_ogm_aggr_send() - flush & send aggregation queue - * @bat_priv: the bat priv with all the mesh interface information * @hard_iface: the interface with the aggregation queue to flush * * Aggregates all OGMv2 packets currently in the aggregation queue into a @@ -199,8 +198,7 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface) * * Caller needs to hold the hard_iface->bat_v.aggr_list.lock. */ -static void batadv_v_ogm_aggr_send(struct batadv_priv *bat_priv, - struct batadv_hard_iface *hard_iface) +static void batadv_v_ogm_aggr_send(struct batadv_hard_iface *hard_iface) { unsigned int aggr_len = hard_iface->bat_v.aggr_len; struct sk_buff *skb_aggr; @@ -230,35 +228,37 @@ static void batadv_v_ogm_aggr_send(struct batadv_priv *bat_priv, consume_skb(skb); } - batadv_v_ogm_send_to_if(bat_priv, skb_aggr, hard_iface); + batadv_v_ogm_send_to_if(skb_aggr, hard_iface); } /** * batadv_v_ogm_queue_on_if() - queue a batman ogm on a given interface - * @bat_priv: the bat priv with all the mesh interface information * @skb: the OGM to queue * @hard_iface: the interface to queue the OGM on */ -static void batadv_v_ogm_queue_on_if(struct batadv_priv *bat_priv, - struct sk_buff *skb, +static void batadv_v_ogm_queue_on_if(struct sk_buff *skb, struct batadv_hard_iface *hard_iface) { - if (hard_iface->mesh_iface != bat_priv->mesh_iface) { - kfree_skb(skb); - return; - } + struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); if (!READ_ONCE(bat_priv->aggregated_ogms)) { - batadv_v_ogm_send_to_if(bat_priv, skb, hard_iface); + batadv_v_ogm_send_to_if(skb, hard_iface); return; } spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + if (!hard_iface->bat_v.aggr_list_enabled) { + kfree_skb(skb); + goto unlock; + } + if (!batadv_v_ogm_queue_left(skb, hard_iface)) - batadv_v_ogm_aggr_send(bat_priv, hard_iface); + batadv_v_ogm_aggr_send(hard_iface); hard_iface->bat_v.aggr_len += batadv_v_ogm_len(skb); __skb_queue_tail(&hard_iface->bat_v.aggr_list, skb); + +unlock: spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); } @@ -350,7 +350,7 @@ static void batadv_v_ogm_send_meshif(struct batadv_priv *bat_priv) break; } - batadv_v_ogm_queue_on_if(bat_priv, skb_tmp, hard_iface); + batadv_v_ogm_queue_on_if(skb_tmp, hard_iface); batadv_hardif_put(hard_iface); } rcu_read_unlock(); @@ -390,14 +390,12 @@ void batadv_v_ogm_aggr_work(struct work_struct *work) { struct batadv_hard_iface_bat_v *batv; struct batadv_hard_iface *hard_iface; - struct batadv_priv *bat_priv; batv = container_of(work, struct batadv_hard_iface_bat_v, aggr_wq.work); hard_iface = container_of(batv, struct batadv_hard_iface, bat_v); - bat_priv = netdev_priv(hard_iface->mesh_iface); spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); - batadv_v_ogm_aggr_send(bat_priv, hard_iface); + batadv_v_ogm_aggr_send(hard_iface); spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); batadv_v_ogm_start_queue_timer(hard_iface); @@ -415,6 +413,10 @@ int batadv_v_ogm_iface_enable(struct batadv_hard_iface *hard_iface) { struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); + spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + hard_iface->bat_v.aggr_list_enabled = true; + spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); + enable_delayed_work(&hard_iface->bat_v.aggr_wq); batadv_v_ogm_start_queue_timer(hard_iface); @@ -432,6 +434,7 @@ void batadv_v_ogm_iface_disable(struct batadv_hard_iface *hard_iface) disable_delayed_work_sync(&hard_iface->bat_v.aggr_wq); spin_lock_bh(&hard_iface->bat_v.aggr_list.lock); + hard_iface->bat_v.aggr_list_enabled = false; batadv_v_ogm_aggr_list_free(hard_iface); spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock); } @@ -589,7 +592,7 @@ static void batadv_v_ogm_forward(struct batadv_priv *bat_priv, if_outgoing->net_dev->name, ntohl(ogm_forward->throughput), ogm_forward->ttl, if_incoming->net_dev->name); - batadv_v_ogm_queue_on_if(bat_priv, skb, if_outgoing); + batadv_v_ogm_queue_on_if(skb, if_outgoing); out: batadv_orig_ifinfo_put(orig_ifinfo); @@ -837,14 +840,23 @@ batadv_v_ogm_aggr_packet(int buff_pos, int packet_len, const struct batadv_ogm2_packet *ogm2_packet) { int next_buff_pos = 0; + u16 tvlv_len; /* check if there is enough space for the header */ next_buff_pos += buff_pos + sizeof(*ogm2_packet); if (next_buff_pos > packet_len) return false; + tvlv_len = ntohs(ogm2_packet->tvlv_len); + + /* the fields of an aggregated OGMv2 are accessed assuming (at least) + * 2-byte alignment, so a following OGMv2 must start at an even offset. + */ + if (tvlv_len & 1) + return false; + /* check if there is enough space for the optional TVLV */ - next_buff_pos += ntohs(ogm2_packet->tvlv_len); + next_buff_pos += tvlv_len; return next_buff_pos <= packet_len; } diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 5c73f6ba16cf..f9a1fadf8de9 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -344,7 +344,6 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac, struct sk_buff *skb; struct ethhdr *ethhdr; struct batadv_hard_iface *primary_if; - struct net_device *mesh_iface; u8 *hw_src; struct batadv_bla_claim_dst local_claim_dest; __be32 zeroip = 0; @@ -357,14 +356,10 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac, sizeof(local_claim_dest)); local_claim_dest.type = claimtype; - mesh_iface = READ_ONCE(primary_if->mesh_iface); - if (!mesh_iface) - goto out; - skb = arp_create(ARPOP_REPLY, ETH_P_ARP, /* IP DST: 0.0.0.0 */ zeroip, - mesh_iface, + primary_if->mesh_iface, /* IP SRC: 0.0.0.0 */ zeroip, /* Ethernet DST: Broadcast */ @@ -442,7 +437,7 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac, } skb_reset_mac_header(skb); - skb->protocol = eth_type_trans(skb, mesh_iface); + skb->protocol = eth_type_trans(skb, primary_if->mesh_iface); batadv_inc_counter(bat_priv, BATADV_CNT_RX); batadv_add_counter(bat_priv, BATADV_CNT_RX_BYTES, skb->len + ETH_HLEN); diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index aaea155b9403..c40c9e02391b 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -215,10 +215,13 @@ static void batadv_dat_purge(struct work_struct *work) */ static bool batadv_compare_dat(const struct hlist_node *node, const void *data2) { - const void *data1 = container_of(node, struct batadv_dat_entry, - hash_entry); + const struct batadv_dat_entry *entry1; + const struct batadv_dat_entry *entry2; - return memcmp(data1, data2, sizeof(__be32)) == 0; + entry1 = container_of(node, struct batadv_dat_entry, hash_entry); + entry2 = data2; + + return entry1->ip == entry2->ip && entry1->vid == entry2->vid; } /** @@ -345,6 +348,9 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, if (dat_entry->ip != ip) continue; + if (dat_entry->vid != vid) + continue; + if (!kref_get_unless_zero(&dat_entry->refcount)) continue; @@ -1060,6 +1066,9 @@ static u16 batadv_arp_get_type(struct batadv_priv *bat_priv, * @skb: the buffer containing the packet to extract the VID from * @hdr_size: the size of the batman-adv header encapsulating the packet * + * The caller must ensure that at least @hdr_size + ETH_HLEN bytes are + * accessible after skb->data. + * * Return: If the packet embedded in the skb is vlan tagged this function * returns the VID with the BATADV_VLAN_HAS_TAG flag. Otherwise BATADV_NO_FLAGS * is returned. @@ -1142,6 +1151,10 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1237,6 +1250,10 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1299,6 +1316,10 @@ void batadv_dat_snoop_outgoing_arp_reply(struct batadv_priv *bat_priv, if (!READ_ONCE(bat_priv->distributed_arp_table)) return; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + return; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1347,6 +1368,10 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, if (!READ_ONCE(bat_priv->distributed_arp_table)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(skb, &hdr_size); type = batadv_arp_get_type(bat_priv, skb, hdr_size); @@ -1741,6 +1766,7 @@ void batadv_dat_snoop_incoming_dhcp_ack(struct batadv_priv *bat_priv, struct ethhdr *ethhdr; __be32 ip_src, yiaddr; unsigned short vid; + int hdr_size_tmp; __be16 proto; u8 *hw_src; @@ -1757,8 +1783,10 @@ void batadv_dat_snoop_incoming_dhcp_ack(struct batadv_priv *bat_priv, if (!batadv_dat_check_dhcp_ack(skb, proto, &ip_src, chaddr, &yiaddr)) return; + hdr_size_tmp = hdr_size; + vid = batadv_dat_get_vid(skb, &hdr_size_tmp); + ethhdr = (struct ethhdr *)(skb->data + hdr_size); hw_src = ethhdr->h_source; - vid = batadv_dat_get_vid(skb, &hdr_size); batadv_dat_entry_add(bat_priv, yiaddr, chaddr, vid); batadv_dat_entry_add(bat_priv, ip_src, hw_src, vid); @@ -1798,6 +1826,10 @@ bool batadv_dat_drop_broadcast_packet(struct batadv_priv *bat_priv, if (batadv_forw_packet_is_rebroadcast(forw_packet)) goto out; + /* first, find out the vid. */ + if (!pskb_may_pull(forw_packet->skb, hdr_size + ETH_HLEN)) + goto out; + vid = batadv_dat_get_vid(forw_packet->skb, &hdr_size); type = batadv_arp_get_type(bat_priv, forw_packet->skb, hdr_size); diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c index 1e42cf99f8b3..8a006a0473a8 100644 --- a/net/batman-adv/fragmentation.c +++ b/net/batman-adv/fragmentation.c @@ -386,6 +386,8 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb, * @skb: skb to forward * @recv_if: interface that the skb is received on * @orig_node_src: originator that the skb is received from + * @rx_result: set to NET_RX_SUCCESS when the fragment was forwarded and + * NET_RX_DROP when it was dropped; only valid when true is returned * * Look up the next-hop of the fragments payload and check if the merged packet * will exceed the MTU towards the next-hop. If so, the fragment is forwarded @@ -395,7 +397,8 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb, */ bool batadv_frag_skb_fwd(struct sk_buff *skb, struct batadv_hard_iface *recv_if, - struct batadv_orig_node *orig_node_src) + struct batadv_orig_node *orig_node_src, + int *rx_result) { struct batadv_priv *bat_priv = netdev_priv(recv_if->mesh_iface); struct batadv_neigh_node *neigh_node = NULL; @@ -414,12 +417,29 @@ bool batadv_frag_skb_fwd(struct sk_buff *skb, */ total_size = ntohs(packet->total_size); if (total_size > neigh_node->if_incoming->net_dev->mtu) { + if (packet->ttl < 2) { + kfree_skb(skb); + *rx_result = NET_RX_DROP; + ret = true; + goto out; + } + + if (skb_cow(skb, ETH_HLEN) < 0) { + kfree_skb(skb); + *rx_result = NET_RX_DROP; + ret = true; + goto out; + } + + packet = (struct batadv_frag_packet *)skb->data; + batadv_inc_counter(bat_priv, BATADV_CNT_FRAG_FWD); batadv_add_counter(bat_priv, BATADV_CNT_FRAG_FWD_BYTES, skb->len + ETH_HLEN); packet->ttl--; batadv_send_unicast_skb(skb, neigh_node); + *rx_result = NET_RX_SUCCESS; ret = true; } diff --git a/net/batman-adv/fragmentation.h b/net/batman-adv/fragmentation.h index dbf0871f8703..51e281027ab6 100644 --- a/net/batman-adv/fragmentation.h +++ b/net/batman-adv/fragmentation.h @@ -19,7 +19,8 @@ void batadv_frag_purge_orig(struct batadv_orig_node *orig, bool (*check_cb)(struct batadv_frag_table_entry *)); bool batadv_frag_skb_fwd(struct sk_buff *skb, struct batadv_hard_iface *recv_if, - struct batadv_orig_node *orig_node_src); + struct batadv_orig_node *orig_node_src, + int *rx_result); bool batadv_frag_skb_buffer(struct sk_buff **skb, struct batadv_orig_node *orig_node); int batadv_frag_send_packet(struct sk_buff *skb, diff --git a/net/batman-adv/gateway_client.c b/net/batman-adv/gateway_client.c index 305488a74a25..a5ac82eabd25 100644 --- a/net/batman-adv/gateway_client.c +++ b/net/batman-adv/gateway_client.c @@ -684,12 +684,13 @@ bool batadv_gw_out_of_range(struct batadv_priv *bat_priv, struct batadv_gw_node *gw_node = NULL; struct batadv_gw_node *curr_gw = NULL; struct batadv_neigh_ifinfo *curr_ifinfo, *old_ifinfo; - struct ethhdr *ethhdr = (struct ethhdr *)skb->data; + struct ethhdr *ethhdr; bool out_of_range = false; u8 curr_tq_avg; unsigned short vid; vid = batadv_get_vid(skb, 0); + ethhdr = (struct ethhdr *)skb->data; if (is_multicast_ether_addr(ethhdr->h_dest)) goto out; diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index 60cee2c2f2f4..b6867576bbaf 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -63,6 +63,7 @@ void batadv_hardif_release(struct kref *ref) struct batadv_hard_iface *hard_iface; hard_iface = container_of(ref, struct batadv_hard_iface, refcount); + netdev_put(hard_iface->mesh_iface, &hard_iface->meshif_dev_tracker); netdev_put(hard_iface->net_dev, &hard_iface->dev_tracker); kfree_rcu(hard_iface, rcu); @@ -75,21 +76,21 @@ void batadv_hardif_release(struct kref *ref) * Return: batadv_hard_iface of net_dev (with increased refcnt), NULL on errors */ struct batadv_hard_iface * -batadv_hardif_get_by_netdev(const struct net_device *net_dev) +batadv_hardif_get_by_netdev(struct net_device *net_dev) { struct batadv_hard_iface *hard_iface; + struct net_device *mesh_iface; - rcu_read_lock(); - list_for_each_entry_rcu(hard_iface, &batadv_hardif_list, list) { - if (hard_iface->net_dev == net_dev && - kref_get_unless_zero(&hard_iface->refcount)) - goto out; - } + ASSERT_RTNL(); - hard_iface = NULL; + mesh_iface = netdev_master_upper_dev_get(net_dev); + if (!mesh_iface || !batadv_meshif_is_valid(mesh_iface)) + return NULL; + + hard_iface = netdev_lower_dev_get_private(mesh_iface, net_dev); + if (!kref_get_unless_zero(&hard_iface->refcount)) + return NULL; -out: - rcu_read_unlock(); return hard_iface; } @@ -245,7 +246,7 @@ struct net_device *__batadv_get_real_netdev(struct net_device *netdev) } hard_iface = batadv_hardif_get_by_netdev(netdev); - if (!hard_iface || !hard_iface->mesh_iface) + if (!hard_iface) goto out; net = dev_net(hard_iface->mesh_iface); @@ -539,16 +540,10 @@ static void batadv_check_known_mac_addr(const struct batadv_hard_iface *hard_ifa const struct batadv_hard_iface *tmp_hard_iface; struct list_head *iter; - if (!mesh_iface) - return; - netdev_for_each_lower_private(mesh_iface, tmp_hard_iface, iter) { if (tmp_hard_iface == hard_iface) continue; - if (tmp_hard_iface->if_status == BATADV_IF_NOT_IN_USE) - continue; - if (!batadv_compare_eth(tmp_hard_iface->net_dev->dev_addr, hard_iface->net_dev->dev_addr)) continue; @@ -574,9 +569,6 @@ static void batadv_hardif_recalc_extra_skbroom(struct net_device *mesh_iface) rcu_read_lock(); netdev_for_each_lower_private_rcu(mesh_iface, hard_iface, iter) { - if (hard_iface->if_status == BATADV_IF_NOT_IN_USE) - continue; - lower_header_len = max_t(unsigned short, lower_header_len, hard_iface->net_dev->hard_header_len); @@ -723,38 +715,63 @@ batadv_hardif_deactivate_interface(struct batadv_hard_iface *hard_iface) } /** - * batadv_hardif_enable_interface() - Enslave hard interface to mesh interface - * @hard_iface: hard interface to add to mesh interface + * batadv_hardif_enable_interface() - Enslave interface to mesh interface + * @net_dev: netdev struct of the interface to add to mesh interface * @mesh_iface: netdev struct of the mesh interface * * Return: 0 on success or negative error number in case of failure */ -int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, +int batadv_hardif_enable_interface(struct net_device *net_dev, struct net_device *mesh_iface) { struct batadv_priv *bat_priv; __be16 ethertype = htons(ETH_P_BATMAN); int max_header_len = batadv_max_header_len(); + struct batadv_hard_iface *hard_iface; unsigned int required_mtu; unsigned int hardif_mtu; bool fragmentation; int ret; - hardif_mtu = READ_ONCE(hard_iface->net_dev->mtu); + ASSERT_RTNL(); + + if (!batadv_is_valid_iface(net_dev)) + return -EINVAL; + + hardif_mtu = READ_ONCE(net_dev->mtu); required_mtu = READ_ONCE(mesh_iface->mtu) + max_header_len; if (hardif_mtu < ETH_MIN_MTU + max_header_len) return -EINVAL; - if (hard_iface->if_status != BATADV_IF_NOT_IN_USE) - goto out; + hard_iface = kzalloc_obj(*hard_iface, GFP_ATOMIC); + if (!hard_iface) + return -ENOMEM; + + netdev_hold(net_dev, &hard_iface->dev_tracker, GFP_ATOMIC); + hard_iface->net_dev = net_dev; - kref_get(&hard_iface->refcount); + hard_iface->if_status = BATADV_IF_INACTIVE; + + INIT_HLIST_HEAD(&hard_iface->neigh_list); + + mutex_init(&hard_iface->bat_iv.ogm_buff_mutex); + spin_lock_init(&hard_iface->neigh_list_lock); + kref_init(&hard_iface->refcount); + + hard_iface->num_bcasts = BATADV_NUM_BCASTS_DEFAULT; + if (batadv_is_wifi_hardif(hard_iface)) + hard_iface->num_bcasts = BATADV_NUM_BCASTS_WIRELESS; + + WRITE_ONCE(hard_iface->hop_penalty, 0); + + batadv_v_hardif_init(hard_iface); netdev_hold(mesh_iface, &hard_iface->meshif_dev_tracker, GFP_ATOMIC); hard_iface->mesh_iface = mesh_iface; bat_priv = netdev_priv(hard_iface->mesh_iface); + bat_priv->hardif_generation++; ret = netdev_master_upper_dev_link(hard_iface->net_dev, mesh_iface, hard_iface, NULL, NULL); if (ret) @@ -764,9 +781,6 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, if (ret < 0) goto err_upper; - hard_iface->if_status = BATADV_IF_INACTIVE; - - kref_get(&hard_iface->refcount); hard_iface->batman_adv_ptype.type = ethertype; hard_iface->batman_adv_ptype.func = batadv_batman_skb_recv; hard_iface->batman_adv_ptype.dev = hard_iface->net_dev; @@ -802,42 +816,15 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, if (bat_priv->algo_ops->iface.enabled) bat_priv->algo_ops->iface.enabled(hard_iface); -out: return 0; err_upper: netdev_upper_dev_unlink(hard_iface->net_dev, mesh_iface); err_dev: - hard_iface->mesh_iface = NULL; - netdev_put(mesh_iface, &hard_iface->meshif_dev_tracker); batadv_hardif_put(hard_iface); return ret; } -/** - * batadv_hardif_cnt() - get number of interfaces enslaved to mesh interface - * @mesh_iface: mesh interface to check - * - * This function is only using RCU for locking - the result can therefore be - * off when another function is modifying the list at the same time. The - * caller can use the rtnl_lock to make sure that the count is accurate. - * - * Return: number of connected/enslaved hard interfaces - */ -static size_t batadv_hardif_cnt(struct net_device *mesh_iface) -{ - struct batadv_hard_iface *hard_iface; - struct list_head *iter; - size_t count = 0; - - rcu_read_lock(); - netdev_for_each_lower_private_rcu(mesh_iface, hard_iface, iter) - count++; - rcu_read_unlock(); - - return count; -} - /** * batadv_hardif_disable_interface() - Remove hard interface from mesh interface * @hard_iface: hard interface to be removed @@ -845,17 +832,18 @@ static size_t batadv_hardif_cnt(struct net_device *mesh_iface) void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface) { struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); - struct batadv_hard_iface *primary_if = NULL; + struct batadv_hard_iface *primary_if; + + ASSERT_RTNL(); batadv_hardif_deactivate_interface(hard_iface); if (hard_iface->if_status != BATADV_IF_INACTIVE) - goto out; + return; batadv_info(hard_iface->mesh_iface, "Removing interface: %s\n", hard_iface->net_dev->name); dev_remove_pack(&hard_iface->batman_adv_ptype); - batadv_hardif_put(hard_iface); primary_if = batadv_primary_if_get_selected(bat_priv); if (hard_iface == primary_if) { @@ -866,83 +854,23 @@ void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface) batadv_hardif_put(new_if); } + batadv_hardif_put(primary_if); bat_priv->algo_ops->iface.disable(hard_iface); - hard_iface->if_status = BATADV_IF_NOT_IN_USE; + hard_iface->if_status = BATADV_IF_TO_BE_REMOVED; /* delete all references to this hard_iface */ batadv_purge_orig_ref(bat_priv); batadv_purge_outstanding_packets(bat_priv, hard_iface); - netdev_put(hard_iface->mesh_iface, &hard_iface->meshif_dev_tracker); + bat_priv->hardif_generation++; netdev_upper_dev_unlink(hard_iface->net_dev, hard_iface->mesh_iface); batadv_hardif_recalc_extra_skbroom(hard_iface->mesh_iface); - /* nobody uses this interface anymore */ - if (batadv_hardif_cnt(hard_iface->mesh_iface) <= 1) + /* nobody uses this mesh interface anymore */ + if (list_empty(&hard_iface->mesh_iface->adj_list.lower)) batadv_gw_check_client_stop(bat_priv); - hard_iface->mesh_iface = NULL; - batadv_hardif_put(hard_iface); - -out: - batadv_hardif_put(primary_if); -} - -static struct batadv_hard_iface * -batadv_hardif_add_interface(struct net_device *net_dev) -{ - struct batadv_hard_iface *hard_iface; - - ASSERT_RTNL(); - - if (!batadv_is_valid_iface(net_dev)) - return NULL; - - hard_iface = kzalloc_obj(*hard_iface, GFP_ATOMIC); - if (!hard_iface) - return NULL; - - netdev_hold(net_dev, &hard_iface->dev_tracker, GFP_ATOMIC); - hard_iface->net_dev = net_dev; - - hard_iface->mesh_iface = NULL; - hard_iface->if_status = BATADV_IF_NOT_IN_USE; - - INIT_LIST_HEAD(&hard_iface->list); - INIT_HLIST_HEAD(&hard_iface->neigh_list); - - mutex_init(&hard_iface->bat_iv.ogm_buff_mutex); - spin_lock_init(&hard_iface->neigh_list_lock); - kref_init(&hard_iface->refcount); - - hard_iface->num_bcasts = BATADV_NUM_BCASTS_DEFAULT; - if (batadv_is_wifi_hardif(hard_iface)) - hard_iface->num_bcasts = BATADV_NUM_BCASTS_WIRELESS; - - WRITE_ONCE(hard_iface->hop_penalty, 0); - - batadv_v_hardif_init(hard_iface); - - kref_get(&hard_iface->refcount); - list_add_tail_rcu(&hard_iface->list, &batadv_hardif_list); - batadv_hardif_generation++; - - return hard_iface; -} - -static void batadv_hardif_remove_interface(struct batadv_hard_iface *hard_iface) -{ - ASSERT_RTNL(); - - /* first deactivate interface */ - if (hard_iface->if_status != BATADV_IF_NOT_IN_USE) - batadv_hardif_disable_interface(hard_iface); - - if (hard_iface->if_status != BATADV_IF_NOT_IN_USE) - return; - - hard_iface->if_status = BATADV_IF_TO_BE_REMOVED; batadv_hardif_put(hard_iface); } @@ -1106,10 +1034,6 @@ static int batadv_hard_if_event(struct notifier_block *this, batadv_wifi_net_device_event(event, net_dev); hard_iface = batadv_hardif_get_by_netdev(net_dev); - if (!hard_iface && (event == NETDEV_REGISTER || - event == NETDEV_POST_TYPE_CHANGE)) - hard_iface = batadv_hardif_add_interface(net_dev); - if (!hard_iface) goto out; @@ -1123,19 +1047,12 @@ static int batadv_hard_if_event(struct notifier_block *this, break; case NETDEV_UNREGISTER: case NETDEV_PRE_TYPE_CHANGE: - list_del_rcu(&hard_iface->list); - batadv_hardif_generation++; - - batadv_hardif_remove_interface(hard_iface); + batadv_hardif_disable_interface(hard_iface); break; case NETDEV_CHANGEMTU: - if (hard_iface->mesh_iface) - batadv_update_min_mtu(hard_iface->mesh_iface); + batadv_update_min_mtu(hard_iface->mesh_iface); break; case NETDEV_CHANGEADDR: - if (hard_iface->if_status == BATADV_IF_NOT_IN_USE) - goto hardif_put; - batadv_check_known_mac_addr(hard_iface); bat_priv = netdev_priv(hard_iface->mesh_iface); diff --git a/net/batman-adv/hard-interface.h b/net/batman-adv/hard-interface.h index af31696c3978..935f47ca9a48 100644 --- a/net/batman-adv/hard-interface.h +++ b/net/batman-adv/hard-interface.h @@ -21,12 +21,6 @@ * enum batadv_hard_if_state - State of a hard interface */ enum batadv_hard_if_state { - /** - * @BATADV_IF_NOT_IN_USE: interface is not used as slave interface of a - * batman-adv mesh interface - */ - BATADV_IF_NOT_IN_USE, - /** * @BATADV_IF_TO_BE_REMOVED: interface will be removed from mesh * interface @@ -74,8 +68,8 @@ u32 batadv_netdev_get_wifi_flags(struct net_device *net_dev); u32 batadv_hardif_get_wifi_flags(struct batadv_hard_iface *hard_iface); bool batadv_is_wifi_hardif(struct batadv_hard_iface *hard_iface); struct batadv_hard_iface* -batadv_hardif_get_by_netdev(const struct net_device *net_dev); -int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, +batadv_hardif_get_by_netdev(struct net_device *net_dev); +int batadv_hardif_enable_interface(struct net_device *net_dev, struct net_device *mesh_iface); void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface); int batadv_hardif_min_mtu(struct net_device *mesh_iface); diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index 3c4572284b53..fb2b0e35e349 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -59,11 +59,6 @@ #include "tp_meter.h" #include "translation-table.h" -/* List manipulations on hardif_list have to be rtnl_lock()'ed, - * list traversals just rcu-locked - */ -struct list_head batadv_hardif_list; -unsigned int batadv_hardif_generation; static int (*batadv_rx_handler[256])(struct sk_buff *skb, struct batadv_hard_iface *recv_if); @@ -95,7 +90,6 @@ static int __init batadv_init(void) if (ret < 0) return ret; - INIT_LIST_HEAD(&batadv_hardif_list); batadv_algo_init(); batadv_recv_handler_init(); @@ -450,9 +444,6 @@ int batadv_batman_skb_recv(struct sk_buff *skb, struct net_device *dev, if (unlikely(skb->mac_len != ETH_HLEN || !skb_mac_header(skb))) goto err_free; - if (!hard_iface->mesh_iface) - goto err_free; - bat_priv = netdev_priv(hard_iface->mesh_iface); if (READ_ONCE(bat_priv->mesh_state) != BATADV_MESH_ACTIVE) @@ -580,6 +571,9 @@ void batadv_recv_handler_unregister(u8 packet_type) * @skb: the buffer containing the packet * @header_len: length of the batman header preceding the ethernet header * + * The caller must ensure that at least @header_len + ETH_HLEN bytes are + * accessible after skb->data. + * * Return: VID with the BATADV_VLAN_HAS_TAG flag when the packet embedded in the * skb is vlan tagged. Otherwise BATADV_NO_FLAGS. */ diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h index f68fc8b7239c..e738758ee4a7 100644 --- a/net/batman-adv/main.h +++ b/net/batman-adv/main.h @@ -226,9 +226,6 @@ static inline int batadv_print_vid(unsigned short vid) return -1; } -extern struct list_head batadv_hardif_list; -extern unsigned int batadv_hardif_generation; - extern struct workqueue_struct *batadv_event_workqueue; int batadv_mesh_init(struct net_device *mesh_iface); diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index 44026810b99c..5cc72bbc4570 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -434,6 +434,7 @@ void batadv_interface_rx(struct net_device *mesh_iface, if (!pskb_may_pull(skb, VLAN_ETH_HLEN)) goto dropped; + ethhdr = eth_hdr(skb); vhdr = skb_vlan_eth_hdr(skb); /* drop batman-in-batman packets to prevent loops */ @@ -836,18 +837,7 @@ static int batadv_meshif_slave_add(struct net_device *dev, struct net_device *slave_dev, struct netlink_ext_ack *extack) { - struct batadv_hard_iface *hard_iface; - int ret = -EINVAL; - - hard_iface = batadv_hardif_get_by_netdev(slave_dev); - if (!hard_iface || hard_iface->mesh_iface) - goto out; - - ret = batadv_hardif_enable_interface(hard_iface, dev); - -out: - batadv_hardif_put(hard_iface); - return ret; + return batadv_hardif_enable_interface(slave_dev, dev); } /** diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c index 4cf9e3c54ad3..d2bc48c70714 100644 --- a/net/batman-adv/netlink.c +++ b/net/batman-adv/netlink.c @@ -968,7 +968,7 @@ batadv_netlink_dump_hardif(struct sk_buff *msg, struct netlink_callback *cb) bat_priv = netdev_priv(mesh_iface); rtnl_lock(); - cb->seq = batadv_hardif_generation << 1 | 1; + cb->seq = bat_priv->hardif_generation << 1 | 1; netdev_for_each_lower_private(mesh_iface, hard_iface, iter) { if (i++ < skip) @@ -1211,7 +1211,9 @@ batadv_netlink_get_hardif_from_ifindex(struct batadv_priv *bat_priv, if (!hard_dev) return ERR_PTR(-ENODEV); + rtnl_lock(); hard_iface = batadv_hardif_get_by_netdev(hard_dev); + rtnl_unlock(); if (!hard_iface) goto err_put_harddev; diff --git a/net/batman-adv/originator.c b/net/batman-adv/originator.c index 9b38bd9e8da7..48f837cf665a 100644 --- a/net/batman-adv/originator.c +++ b/net/batman-adv/originator.c @@ -1033,7 +1033,6 @@ batadv_purge_neigh_ifinfo(struct batadv_priv *bat_priv, /* don't purge if the interface is not (going) down */ if (if_outgoing->if_status != BATADV_IF_INACTIVE && - if_outgoing->if_status != BATADV_IF_NOT_IN_USE && if_outgoing->if_status != BATADV_IF_TO_BE_REMOVED) continue; @@ -1077,7 +1076,6 @@ batadv_purge_orig_ifinfo(struct batadv_priv *bat_priv, /* don't purge if the interface is not (going) down */ if (if_outgoing->if_status != BATADV_IF_INACTIVE && - if_outgoing->if_status != BATADV_IF_NOT_IN_USE && if_outgoing->if_status != BATADV_IF_TO_BE_REMOVED) continue; @@ -1127,10 +1125,8 @@ batadv_purge_orig_neighbors(struct batadv_priv *bat_priv, if (batadv_has_timed_out(last_seen, BATADV_PURGE_TIMEOUT) || if_incoming->if_status == BATADV_IF_INACTIVE || - if_incoming->if_status == BATADV_IF_NOT_IN_USE || if_incoming->if_status == BATADV_IF_TO_BE_REMOVED) { if (if_incoming->if_status == BATADV_IF_INACTIVE || - if_incoming->if_status == BATADV_IF_NOT_IN_USE || if_incoming->if_status == BATADV_IF_TO_BE_REMOVED) batadv_dbg(BATADV_DBG_BATMAN, bat_priv, "neighbor purge: originator %pM, neighbor: %pM, iface: %s\n", diff --git a/net/batman-adv/routing.c b/net/batman-adv/routing.c index cd4368b846ad..bbd40fe3a8e5 100644 --- a/net/batman-adv/routing.c +++ b/net/batman-adv/routing.c @@ -8,6 +8,7 @@ #include "main.h" #include +#include #include #include #include @@ -204,6 +205,59 @@ bool batadv_check_management_packet(struct sk_buff *skb, return true; } +/** + * batadv_skb_decrement_ttl() - decrement ttl in a batman-adv header, csum-safe + * @skb: the received packet with @skb->data pointing to the batman-adv header + * + * Supports the following packet types, all of which carry the TTL at offset 2: + * + * - batadv_ogm_packet + * - batadv_ogm2_packet + * - batadv_icmp_header + * - batadv_icmp_packet + * - batadv_icmp_tp_packet + * - batadv_icmp_packet_rr + * - batadv_unicast_packet + * - batadv_frag_packet + * - batadv_bcast_packet + * - batadv_mcast_packet + * - batadv_coded_packet + * - batadv_unicast_tvlv_packet + * + * Return: true if the packet may be forwarded (ttl decremented), + * false if it must be dropped (ttl would expire) + */ +static bool batadv_skb_decrement_ttl(struct sk_buff *skb) +{ + static const size_t ttl_offset = 2; + u8 *ttl_pos; + + BUILD_BUG_ON(offsetof(struct batadv_ogm_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_ogm2_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_header, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_tp_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_icmp_packet_rr, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_unicast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_frag_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_bcast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_mcast_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_coded_packet, ttl) != ttl_offset); + BUILD_BUG_ON(offsetof(struct batadv_unicast_tvlv_packet, ttl) != ttl_offset); + + ttl_pos = skb->data + ttl_offset; + + /* would expire on this hop -> drop, leave header + csum untouched */ + if (*ttl_pos < 2) + return false; + + skb_postpull_rcsum(skb, ttl_pos, 1); + (*ttl_pos)--; + skb_postpush_rcsum(skb, ttl_pos, 1); + + return true; +} + /** * batadv_recv_my_icmp_packet() - receive an icmp packet locally * @bat_priv: the bat priv with all the mesh interface information @@ -801,8 +855,8 @@ static bool batadv_check_unicast_ttvn(struct batadv_priv *bat_priv, if (skb_cow(skb, sizeof(*unicast_packet)) < 0) return false; - unicast_packet = (struct batadv_unicast_packet *)skb->data; vid = batadv_get_vid(skb, hdr_len); + unicast_packet = (struct batadv_unicast_packet *)skb->data; ethhdr = (struct ethhdr *)(skb->data + hdr_len); /* do not reroute multicast frames in a unicast header */ @@ -975,6 +1029,7 @@ int batadv_recv_unicast_packet(struct sk_buff *skb, hdr_size); batadv_orig_node_put(orig_node_gw); if (is_gw) { + orig_addr_gw = eth_hdr(skb)->h_source; batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): Dropped unicast pkt received from another backbone gw %pM.\n", __func__, orig_addr_gw); @@ -1114,10 +1169,9 @@ int batadv_recv_frag_packet(struct sk_buff *skb, /* Route the fragment if it is not for us and too big to be merged. */ if (!batadv_is_my_mac(bat_priv, frag_packet->dest) && - batadv_frag_skb_fwd(skb, recv_if, orig_node_src)) { + batadv_frag_skb_fwd(skb, recv_if, orig_node_src, &ret)) { /* skb was consumed */ skb = NULL; - ret = NET_RX_SUCCESS; goto put_orig_node; } @@ -1191,7 +1245,13 @@ int batadv_recv_bcast_packet(struct sk_buff *skb, if (batadv_is_my_mac(bat_priv, bcast_packet->orig)) goto free_skb; - if (bcast_packet->ttl-- < 2) + /* create a copy of the skb, if needed, to modify it. */ + if (skb_cow(skb, ETH_HLEN) < 0) + goto free_skb; + + bcast_packet = (struct batadv_bcast_packet *)skb->data; + + if (!batadv_skb_decrement_ttl(skb)) goto free_skb; orig_node = batadv_orig_hash_find(bat_priv, bcast_packet->orig); @@ -1298,7 +1358,7 @@ int batadv_recv_mcast_packet(struct sk_buff *skb, goto free_skb; mcast_packet = (struct batadv_mcast_packet *)skb->data; - if (mcast_packet->ttl-- < 2) + if (!batadv_skb_decrement_ttl(skb)) goto free_skb; tvlv_buff = (unsigned char *)(skb->data + hdr_size); @@ -1307,6 +1367,12 @@ int batadv_recv_mcast_packet(struct sk_buff *skb, if (tvlv_buff_len > skb->len - hdr_size) goto free_skb; + /* the fields of an multicast payload are accessed assuming (at least) + * 2-byte alignment, so a following packet must start at an even offset. + */ + if (tvlv_buff_len & 1) + goto free_skb; + ret = batadv_tvlv_containers_process(bat_priv, BATADV_MCAST, NULL, skb, tvlv_buff, tvlv_buff_len); if (ret >= 0) { diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c index 7e98cbfbbb70..00467aa79de9 100644 --- a/net/batman-adv/tp_meter.c +++ b/net/batman-adv/tp_meter.c @@ -87,6 +87,11 @@ #define BATADV_TP_PLEN (BATADV_TP_PACKET_LEN - ETH_HLEN - \ sizeof(struct batadv_unicast_packet)) +/** + * BATADV_TP_MAX_UNACKED - maximum number of packets a receiver didn't yet ack + */ +#define BATADV_TP_MAX_UNACKED 100 + static u8 batadv_tp_prerandom[4096] __read_mostly; /** @@ -353,28 +358,16 @@ batadv_tp_list_find_sender_session(struct batadv_priv *bat_priv, const u8 *dst, } /** - * batadv_tp_vars_common_release() - release batadv_tp_vars_common from lists + * batadv_tp_sender_release() - release batadv_tp_sender * and queue for free after rcu grace period - * @ref: kref pointer of the batadv_tp_vars_common + * @ref: kref pointer of the batadv_tp_sender */ -static void batadv_tp_vars_common_release(struct kref *ref) +static void batadv_tp_sender_release(struct kref *ref) { - struct batadv_tp_vars_common *tp_vars; - struct batadv_tp_unacked *un, *safe; - - tp_vars = container_of(ref, struct batadv_tp_vars_common, refcount); - - /* lock should not be needed because this object is now out of any - * context! - */ - spin_lock_bh(&tp_vars->unacked_lock); - list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { - list_del(&un->list); - kfree(un); - } - spin_unlock_bh(&tp_vars->unacked_lock); + struct batadv_tp_sender *tp_vars; - kfree_rcu(tp_vars, rcu); + tp_vars = container_of(ref, struct batadv_tp_sender, common.refcount); + kfree_rcu(tp_vars, common.rcu); } /** @@ -387,7 +380,7 @@ static void batadv_tp_sender_put(struct batadv_tp_sender *tp_vars) if (!tp_vars) return; - kref_put(&tp_vars->common.refcount, batadv_tp_vars_common_release); + kref_put(&tp_vars->common.refcount, batadv_tp_sender_release); } /** @@ -1140,9 +1133,6 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, init_waitqueue_head(&tp_vars->more_bytes); init_completion(&tp_vars->finished); - spin_lock_init(&tp_vars->common.unacked_lock); - INIT_LIST_HEAD(&tp_vars->common.unacked_list); - spin_lock_init(&tp_vars->cc_lock); tp_vars->prerandom_offset = 0; @@ -1246,6 +1236,33 @@ batadv_tp_list_find_receiver_session(struct batadv_priv *bat_priv, const u8 *dst return tp_vars; } +/** + * batadv_tp_receiver_release() - release batadv_tp_receiver + * and queue for free after rcu grace period + * @ref: kref pointer of the batadv_tp_receiver + */ +static void batadv_tp_receiver_release(struct kref *ref) +{ + struct batadv_tp_receiver *tp_vars; + struct batadv_tp_unacked *safe; + struct batadv_tp_unacked *un; + + tp_vars = container_of(ref, struct batadv_tp_receiver, common.refcount); + + /* lock should not be needed because this object is now out of any + * context! + */ + spin_lock_bh(&tp_vars->ack_seqno_lock); + list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { + list_del(&un->list); + kfree(un); + tp_vars->unacked_count--; + } + spin_unlock_bh(&tp_vars->ack_seqno_lock); + + kfree_rcu(tp_vars, common.rcu); +} + /** * batadv_tp_receiver_put() - decrement the batadv_tp_receiver * refcounter and possibly release it @@ -1256,7 +1273,7 @@ static void batadv_tp_receiver_put(struct batadv_tp_receiver *tp_vars) if (!tp_vars) return; - kref_put(&tp_vars->common.refcount, batadv_tp_vars_common_release); + kref_put(&tp_vars->common.refcount, batadv_tp_receiver_release); } /** @@ -1285,7 +1302,7 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t) bat_priv = tp_vars->common.bat_priv; /* if there is recent activity rearm the timer */ - if (!batadv_has_timed_out(tp_vars->last_recv_time, + if (!batadv_has_timed_out(READ_ONCE(tp_vars->last_recv_time), BATADV_TP_RECV_TIMEOUT)) { /* reset the receiver shutdown timer */ batadv_tp_reset_receiver_timer(tp_vars); @@ -1299,12 +1316,13 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t) if (batadv_tp_list_detach(&tp_vars->common)) batadv_tp_receiver_put(tp_vars); - spin_lock_bh(&tp_vars->common.unacked_lock); - list_for_each_entry_safe(un, safe, &tp_vars->common.unacked_list, list) { + spin_lock_bh(&tp_vars->ack_seqno_lock); + list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { list_del(&un->list); kfree(un); + tp_vars->unacked_count--; } - spin_unlock_bh(&tp_vars->common.unacked_lock); + spin_unlock_bh(&tp_vars->ack_seqno_lock); /* drop reference of timer */ if (WARN_ON(atomic_xchg(&tp_vars->receiving, 0) != 1)) @@ -1386,7 +1404,8 @@ static int batadv_tp_send_ack(struct batadv_priv *bat_priv, const u8 *dst, /** * batadv_tp_handle_out_of_order() - store an out of order packet * @tp_vars: the private data of the current TP meter session - * @skb: the buffer containing the received packet + * @seqno: sequence number of new received packet + * @payload_len: length of the received packet * * Store the out of order packet in the unacked list for late processing. This * packets are kept in this list so that they can be ACKed at once as soon as @@ -1395,66 +1414,108 @@ static int batadv_tp_send_ack(struct batadv_priv *bat_priv, const u8 *dst, * Return: true if the packed has been successfully processed, false otherwise */ static bool batadv_tp_handle_out_of_order(struct batadv_tp_receiver *tp_vars, - const struct sk_buff *skb) + u32 seqno, u32 payload_len) + __must_hold(&tp_vars->ack_seqno_lock) { - const struct batadv_icmp_tp_packet *icmp; - struct batadv_tp_unacked *un, *new; - u32 payload_len; - bool added = false; - - new = kmalloc_obj(*new, GFP_ATOMIC); - if (unlikely(!new)) - return false; - - icmp = (struct batadv_icmp_tp_packet *)skb->data; - - new->seqno = ntohl(icmp->seqno); - payload_len = skb->len - sizeof(struct batadv_unicast_packet); - new->len = payload_len; - - spin_lock_bh(&tp_vars->common.unacked_lock); - /* if the list is empty immediately attach this new object */ - if (list_empty(&tp_vars->common.unacked_list)) { - list_add(&new->list, &tp_vars->common.unacked_list); - goto out; - } - - /* otherwise loop over the list and either drop the packet because this - * is a duplicate or store it at the right position. + struct list_head *pos = &tp_vars->unacked_list; + struct batadv_tp_unacked *new = NULL; + u32 end_seqno = seqno + payload_len; + struct batadv_tp_unacked *safe; + struct batadv_tp_unacked *un; + + /* loop over the list to find either an existing entry which the new + * seqno range can be merged with or the position at which a new entry + * has to be inserted. * * The iteration is done in the reverse way because it is likely that * the last received packet (the one being processed now) has a bigger * seqno than all the others already stored. */ - list_for_each_entry_reverse(un, &tp_vars->common.unacked_list, list) { - /* check for duplicates */ - if (new->seqno == un->seqno) { - if (new->len > un->len) - un->len = new->len; - kfree(new); - added = true; - break; - } - - /* look for the right position */ - if (batadv_seq_before(new->seqno, un->seqno)) + list_for_each_entry_reverse(un, &tp_vars->unacked_list, list) { + /* look for the right position - an un which is smaller */ + if (batadv_seq_before(seqno, un->seqno)) continue; - /* as soon as an entry having a bigger seqno is found, the new - * one is attached _after_ it. In this way the list is kept in - * ascending order + /* smaller/equal seqno was found but they might be directly + * after another or overlapping. keep only a single entry + * + * It is already known that: + * + * un->seqno <= seqno + * + * When establishing that: + * + * seqno <= un->seqno + un->len + * + * Then it is not necessary to add a new entry because the + * smaller/equal seqno of un might already contain the new + * received packet or we only add new data directly after + * the end of un. The latter can be identified using: + * + * un->seqno + un->len <= end_seqno */ - list_add(&new->list, &un->list); - added = true; + if (!batadv_seq_before(un->seqno + un->len, seqno)) { + /* new data directly after un? */ + if (!batadv_seq_before(end_seqno, un->seqno + un->len)) + un->len = end_seqno - un->seqno; + + /* un now represents both old un + new range and has to + * be used to check if the gap to the next seqno range + * was closed + */ + new = un; + } else { + /* as soon as an entry having a smaller seqno is found, + * the new one is attached _after_ it. In this way the + * list is kept in ascending order + */ + pos = &un->list; + } + break; } - /* received packet with smallest seqno out of order; add it to front */ - if (!added) - list_add(&new->list, &tp_vars->common.unacked_list); + /* no entry to merge with was found; insert a new one after the entry + * with the next smaller seqno (or at the front of the list when the + * new seqno is the smallest or the list is empty) + */ + if (!new) { + new = kmalloc_obj(*new, GFP_ATOMIC); + if (unlikely(!new)) + return false; -out: - spin_unlock_bh(&tp_vars->common.unacked_lock); + new->seqno = seqno; + new->len = payload_len; + + list_add(&new->list, pos); + tp_vars->unacked_count++; + } + + /* check if new filled the gap to the next list entries */ + un = new; + list_for_each_entry_safe_continue(un, safe, &tp_vars->unacked_list, list) { + if (batadv_seq_before(end_seqno, un->seqno)) + break; + + /* next entry is overlapping or adjacent - combine both */ + if (batadv_seq_before(end_seqno, un->seqno + un->len)) { + end_seqno = un->seqno + un->len; + new->len = end_seqno - new->seqno; + } + + list_del(&un->list); + kfree(un); + tp_vars->unacked_count--; + } + + /* remove the last (biggest) unacked seqno when list is too large */ + if (tp_vars->unacked_count > BATADV_TP_MAX_UNACKED) { + un = list_last_entry(&tp_vars->unacked_list, + struct batadv_tp_unacked, list); + list_del(&un->list); + kfree(un); + tp_vars->unacked_count--; + } return true; } @@ -1465,6 +1526,7 @@ static bool batadv_tp_handle_out_of_order(struct batadv_tp_receiver *tp_vars, * @tp_vars: the private data of the current TP meter session */ static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars) + __must_hold(&tp_vars->ack_seqno_lock) { struct batadv_tp_unacked *un, *safe; u32 to_ack; @@ -1472,8 +1534,7 @@ static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars) /* go through the unacked packet list and possibly ACK them as * well */ - spin_lock_bh(&tp_vars->common.unacked_lock); - list_for_each_entry_safe(un, safe, &tp_vars->common.unacked_list, list) { + list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { /* the list is ordered, therefore it is possible to stop as soon * there is a gap between the last acked seqno and the seqno of * the packet under inspection @@ -1481,15 +1542,15 @@ static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars) if (batadv_seq_before(tp_vars->last_recv, un->seqno)) break; - to_ack = un->seqno + un->len - tp_vars->last_recv; + to_ack = un->seqno + un->len; - if (batadv_seq_before(tp_vars->last_recv, un->seqno + un->len)) - tp_vars->last_recv += to_ack; + if (batadv_seq_before(tp_vars->last_recv, to_ack)) + tp_vars->last_recv = to_ack; list_del(&un->list); kfree(un); + tp_vars->unacked_count--; } - spin_unlock_bh(&tp_vars->common.unacked_lock); } /** @@ -1512,7 +1573,7 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv, tp_vars = batadv_tp_list_find_receiver_session(bat_priv, icmp->orig, icmp->session); if (tp_vars) { - tp_vars->last_recv_time = jiffies; + WRITE_ONCE(tp_vars->last_recv_time, jiffies); goto out_unlock; } @@ -1535,13 +1596,14 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv, tp_vars->common.bat_priv = bat_priv; kref_init(&tp_vars->common.refcount); - spin_lock_init(&tp_vars->common.unacked_lock); - INIT_LIST_HEAD(&tp_vars->common.unacked_list); + spin_lock_init(&tp_vars->ack_seqno_lock); + INIT_LIST_HEAD(&tp_vars->unacked_list); + tp_vars->unacked_count = 0; kref_get(&tp_vars->common.refcount); timer_setup(&tp_vars->common.timer, batadv_tp_receiver_shutdown, 0); - tp_vars->last_recv_time = jiffies; + WRITE_ONCE(tp_vars->last_recv_time, jiffies); kref_get(&tp_vars->common.refcount); hlist_add_head_rcu(&tp_vars->common.list, &bat_priv->tp_receiver_list); @@ -1566,7 +1628,8 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv, { const struct batadv_icmp_tp_packet *icmp; struct batadv_tp_receiver *tp_vars; - size_t packet_size; + u32 payload_len; + u32 to_ack; u32 seqno; icmp = (struct batadv_icmp_tp_packet *)skb->data; @@ -1592,40 +1655,48 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv, goto out; } - tp_vars->last_recv_time = jiffies; + WRITE_ONCE(tp_vars->last_recv_time, jiffies); } + spin_lock_bh(&tp_vars->ack_seqno_lock); + /* if the packet is a duplicate, it may be the case that an ACK has been * lost. Resend the ACK */ - if (batadv_seq_before(seqno, tp_vars->last_recv)) + payload_len = skb->len - sizeof(struct batadv_unicast_packet); + to_ack = seqno + payload_len; + if (batadv_seq_before(to_ack, tp_vars->last_recv)) goto send_ack; /* if the packet is out of order enqueue it */ - if (ntohl(icmp->seqno) != tp_vars->last_recv) { + if (batadv_seq_before(tp_vars->last_recv, seqno)) { /* exit immediately (and do not send any ACK) if the packet has * not been enqueued correctly */ - if (!batadv_tp_handle_out_of_order(tp_vars, skb)) + if (!batadv_tp_handle_out_of_order(tp_vars, seqno, payload_len)) { + spin_unlock_bh(&tp_vars->ack_seqno_lock); goto out; + } /* send a duplicate ACK */ goto send_ack; } /* if everything was fine count the ACKed bytes */ - packet_size = skb->len - sizeof(struct batadv_unicast_packet); - tp_vars->last_recv += packet_size; + tp_vars->last_recv = to_ack; /* check if this ordered message filled a gap.... */ batadv_tp_ack_unordered(tp_vars); send_ack: + to_ack = tp_vars->last_recv; + spin_unlock_bh(&tp_vars->ack_seqno_lock); + /* send the ACK. If the received packet was out of order, the ACK that * is going to be sent is a duplicate (the sender will count them and * possibly enter Fast Retransmit as soon as it has reached 3) */ - batadv_tp_send_ack(bat_priv, icmp->orig, tp_vars->last_recv, + batadv_tp_send_ack(bat_priv, icmp->orig, to_ack, icmp->timestamp, icmp->session, icmp->uid); out: batadv_tp_receiver_put(tp_vars); diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c index 8b6c49c32c89..4bfad36a4b70 100644 --- a/net/batman-adv/translation-table.c +++ b/net/batman-adv/translation-table.c @@ -447,6 +447,9 @@ static void batadv_tt_local_event(struct batadv_priv *bat_priv, if (!batadv_compare_eth(entry->change.addr, common->addr)) continue; + if (entry->change.vid != tt_change_node->change.vid) + continue; + del_op_entry = entry->change.flags & BATADV_TT_CLIENT_DEL; if (del_op_requested != del_op_entry) { /* DEL+ADD in the same orig interval have no effect and @@ -3447,6 +3450,7 @@ static void batadv_tt_roam_purge(struct batadv_priv *bat_priv) * batadv_tt_check_roam_count() - check if a client has roamed too frequently * @bat_priv: the bat priv with all the mesh interface information * @client: mac address of the roaming client + * @vid: VLAN identifier * * This function checks whether the client already reached the * maximum number of possible roaming phases. In this case the ROAMING_ADV @@ -3454,7 +3458,7 @@ static void batadv_tt_roam_purge(struct batadv_priv *bat_priv) * * Return: true if the ROAMING_ADV can be sent, false otherwise */ -static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) +static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client, u16 vid) { struct batadv_tt_roam_node *tt_roam_node; bool ret = false; @@ -3467,6 +3471,9 @@ static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) if (!batadv_compare_eth(tt_roam_node->addr, client)) continue; + if (tt_roam_node->vid != vid) + continue; + if (batadv_has_timed_out(tt_roam_node->first_time, BATADV_ROAMING_MAX_TIME)) continue; @@ -3488,6 +3495,7 @@ static bool batadv_tt_check_roam_count(struct batadv_priv *bat_priv, u8 *client) atomic_set(&tt_roam_node->counter, BATADV_ROAMING_MAX_COUNT - 1); ether_addr_copy(tt_roam_node->addr, client); + tt_roam_node->vid = vid; list_add(&tt_roam_node->list, &bat_priv->tt.roam_list); ret = true; @@ -3524,7 +3532,7 @@ static void batadv_send_roam_adv(struct batadv_priv *bat_priv, u8 *client, /* before going on we have to check whether the client has * already roamed to us too many times */ - if (!batadv_tt_check_roam_count(bat_priv, client)) + if (!batadv_tt_check_roam_count(bat_priv, client, vid)) goto out; batadv_dbg(BATADV_DBG_TT, bat_priv, diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c index 403c85456870..49bf2ed9ecdc 100644 --- a/net/batman-adv/tvlv.c +++ b/net/batman-adv/tvlv.c @@ -411,7 +411,6 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, tvlv_handler->ogm_handler(bat_priv, orig_node, BATADV_NO_FLAGS, tvlv_value, tvlv_value_len); - tvlv_handler->flags |= BATADV_TVLV_HANDLER_OGM_CALLED; break; case BATADV_UNICAST_TVLV: if (!skb) @@ -443,6 +442,78 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, return NET_RX_SUCCESS; } +/** + * batadv_tvlv_hdr_next() - move a tvlv buffer cursor to the next container + * @tvlv_value: cursor into the tvlv buffer, advanced past the returned + * container's content on success + * @tvlv_value_len: remaining length of the tvlv buffer, reduced by the returned + * container's size on success + * + * Parses a single container header at the current cursor position and, if a + * complete container is available, advances the cursor and remaining length + * past it. The returned header stays valid; its content is located at + * (returned header + 1) and is ntohs(hdr->len) bytes long. + * + * Return: pointer to the next tvlv container header, or NULL if no further + * complete container is present in the buffer. + */ +static struct batadv_tvlv_hdr *batadv_tvlv_hdr_next(void **tvlv_value, u16 *tvlv_value_len) +{ + struct batadv_tvlv_hdr *tvlv_hdr; + u16 tvlv_value_cont_len; + void *tvlv_value_cont; + u16 tvlv_len; + + tvlv_value_cont = *tvlv_value; + tvlv_len = *tvlv_value_len; + + if (tvlv_len < sizeof(*tvlv_hdr)) + return NULL; + + tvlv_hdr = tvlv_value_cont; + tvlv_value_cont_len = ntohs(tvlv_hdr->len); + tvlv_value_cont = tvlv_hdr + 1; + tvlv_len -= sizeof(*tvlv_hdr); + + if (tvlv_value_cont_len > tvlv_len) + return NULL; + + /* the next tvlv header is accessed assuming (at least) 2-byte + * alignment, so it must start at an even offset. + */ + if (tvlv_value_cont_len & 1) + return NULL; + + *tvlv_value = (u8 *)tvlv_value_cont + tvlv_value_cont_len; + *tvlv_value_len = tvlv_len - tvlv_value_cont_len; + + return tvlv_hdr; +} + +/** + * batadv_tvlv_containers_contain() - check if a tvlv buffer holds a container + * @tvlv_value: tvlv content + * @tvlv_value_len: tvlv content length + * @type: tvlv container type to look for + * @version: tvlv container version to look for + * + * Return: true if a container of the given type and version is present in the + * tvlv buffer, false otherwise. + */ +static bool batadv_tvlv_containers_contain(void *tvlv_value, + u16 tvlv_value_len, u8 type, + u8 version) +{ + struct batadv_tvlv_hdr *tvlv_hdr; + + while ((tvlv_hdr = batadv_tvlv_hdr_next(&tvlv_value, &tvlv_value_len))) { + if (tvlv_hdr->type == type && tvlv_hdr->version == version) + return true; + } + + return false; +} + /** * batadv_tvlv_containers_process() - parse the given tvlv buffer to call the * appropriate handlers @@ -462,20 +533,16 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, struct sk_buff *skb, void *tvlv_value, u16 tvlv_value_len) { + u16 tvlv_value_start_len = tvlv_value_len; struct batadv_tvlv_handler *tvlv_handler; + void *tvlv_value_start = tvlv_value; struct batadv_tvlv_hdr *tvlv_hdr; u16 tvlv_value_cont_len; u8 cifnotfound = BATADV_TVLV_HANDLER_OGM_CIFNOTFND; int ret = NET_RX_SUCCESS; - while (tvlv_value_len >= sizeof(*tvlv_hdr)) { - tvlv_hdr = tvlv_value; + while ((tvlv_hdr = batadv_tvlv_hdr_next(&tvlv_value, &tvlv_value_len))) { tvlv_value_cont_len = ntohs(tvlv_hdr->len); - tvlv_value = tvlv_hdr + 1; - tvlv_value_len -= sizeof(*tvlv_hdr); - - if (tvlv_value_cont_len > tvlv_value_len) - break; tvlv_handler = batadv_tvlv_handler_get(bat_priv, tvlv_hdr->type, @@ -483,11 +550,9 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, ret |= batadv_tvlv_call_handler(bat_priv, tvlv_handler, packet_type, orig_node, skb, - tvlv_value, + tvlv_hdr + 1, tvlv_value_cont_len); batadv_tvlv_handler_put(tvlv_handler); - tvlv_value = (u8 *)tvlv_value + tvlv_value_cont_len; - tvlv_value_len -= tvlv_value_cont_len; } if (packet_type != BATADV_IV_OGM && @@ -500,12 +565,20 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv, if (!tvlv_handler->ogm_handler) continue; - if ((tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CIFNOTFND) && - !(tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CALLED)) - tvlv_handler->ogm_handler(bat_priv, orig_node, - cifnotfound, NULL, 0); + if (!(tvlv_handler->flags & BATADV_TVLV_HANDLER_OGM_CIFNOTFND)) + continue; + + /* if the corresponding container was present then the handler + * was already called from the loop above + */ + if (batadv_tvlv_containers_contain(tvlv_value_start, + tvlv_value_start_len, + tvlv_handler->type, + tvlv_handler->version)) + continue; - tvlv_handler->flags &= ~BATADV_TVLV_HANDLER_OGM_CALLED; + tvlv_handler->ogm_handler(bat_priv, orig_node, + cifnotfound, NULL, 0); } rcu_read_unlock(); diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index 5fd5bd358a24..cd12755d21f3 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -145,6 +145,12 @@ struct batadv_hard_iface_bat_v { /** @aggr_list: queue for to be aggregated OGM packets */ struct sk_buff_head aggr_list; + /** + * @aggr_list_enabled: aggr_list is active and new skbs can be + * enqueued. Protected by aggr_list.lock after initialization + */ + bool aggr_list_enabled:1; + /** @aggr_len: size of the OGM aggregate (excluding ethernet header) */ unsigned int aggr_len; @@ -208,9 +214,6 @@ struct batadv_wifi_net_device_state { * struct batadv_hard_iface - network device known to batman-adv */ struct batadv_hard_iface { - /** @list: list node for batadv_hardif_list */ - struct list_head list; - /** @if_status: status of the interface for batman-adv */ char if_status; @@ -1329,9 +1332,9 @@ struct batadv_tp_unacked { u32 seqno; /** @len: length of the packet */ - u16 len; + u32 len; - /** @list: list node for &batadv_tp_vars_common.unacked_list */ + /** @list: list node for &batadv_tp_receiver.unacked_list */ struct list_head list; }; @@ -1354,12 +1357,6 @@ struct batadv_tp_vars_common { /** @session: TP session identifier */ u8 session[2]; - /** @unacked_list: list of unacked packets (meta-info only) */ - struct list_head unacked_list; - - /** @unacked_lock: protect unacked_list */ - spinlock_t unacked_lock; - /** @refcount: number of context where the object is used */ struct kref refcount; @@ -1473,6 +1470,15 @@ struct batadv_tp_receiver { /** @last_recv_time: time (jiffies) a msg was received */ unsigned long last_recv_time; + + /** @unacked_list: list of unacked packets (meta-info only) */ + struct list_head unacked_list; + + /** @ack_seqno_lock: protect unacked_list + &batadv_tp_receiver.last_recv */ + spinlock_t ack_seqno_lock; + + /** @unacked_count: number of unacked entries */ + size_t unacked_count; }; /** @@ -1670,6 +1676,9 @@ struct batadv_priv { /** @tp_num: number of currently active tp sessions */ atomic_t tp_num; + /** @hardif_generation: generation counter added to netlink hardif dumps */ + unsigned int hardif_generation; + /** @orig_work: work queue callback item for orig node purging */ struct delayed_work orig_work; @@ -1952,6 +1961,9 @@ struct batadv_tt_roam_node { /** @addr: mac address of the client in the roaming phase */ u8 addr[ETH_ALEN]; + /** @vid: VLAN identifier */ + u16 vid; + /** * @counter: number of allowed roaming events per client within a single * OGM interval (changes are committed with each OGM) @@ -2282,13 +2294,6 @@ enum batadv_tvlv_handler_flags { * will call this handler even if its type was not found (with no data) */ BATADV_TVLV_HANDLER_OGM_CIFNOTFND = BIT(1), - - /** - * @BATADV_TVLV_HANDLER_OGM_CALLED: interval tvlv handling flag - the - * API marks a handler as being called, so it won't be called if the - * BATADV_TVLV_HANDLER_OGM_CIFNOTFND flag was set - */ - BATADV_TVLV_HANDLER_OGM_CALLED = BIT(2), }; #endif /* _NET_BATMAN_ADV_TYPES_H_ */ diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index cb1e329d66fd..d504a363a30f 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -632,7 +632,7 @@ static struct l2cap_chan *chan_create(void) if (!chan) return NULL; - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); chan->chan_type = L2CAP_CHAN_CONN_ORIENTED; chan->mode = L2CAP_MODE_LE_FLOWCTL; @@ -745,21 +745,6 @@ static inline void chan_ready_cb(struct l2cap_chan *chan) ifup(dev->netdev); } -static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan) -{ - struct l2cap_chan *chan; - - chan = chan_create(); - if (!chan) - return NULL; - - chan->ops = pchan->ops; - - BT_DBG("chan %p pchan %p", chan, pchan); - - return chan; -} - static void unregister_dev(struct lowpan_btle_dev *dev) { struct hci_dev *hdev = READ_ONCE(dev->hdev); @@ -797,20 +782,10 @@ static void chan_close_cb(struct l2cap_chan *chan) struct lowpan_btle_dev *dev = NULL; struct lowpan_peer *peer; int err = -ENOENT; - bool last = false, remove = true; + bool last = false; BT_DBG("chan %p conn %p", chan, chan->conn); - if (chan->conn && chan->conn->hcon) { - if (!is_bt_6lowpan(chan->conn->hcon)) - return; - - /* If conn is set, then the netdev is also there and we should - * not remove it. - */ - remove = false; - } - spin_lock(&devices_lock); list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) { @@ -837,10 +812,8 @@ static void chan_close_cb(struct l2cap_chan *chan) ifdown(dev->netdev); - if (remove) { - INIT_WORK(&entry->delete_netdev, delete_netdev); - schedule_work(&entry->delete_netdev); - } + INIT_WORK(&entry->delete_netdev, delete_netdev); + schedule_work(&entry->delete_netdev); } else { spin_unlock(&devices_lock); } @@ -901,7 +874,6 @@ static long chan_get_sndtimeo_cb(struct l2cap_chan *chan) static const struct l2cap_ops bt_6lowpan_chan_ops = { .name = "L2CAP 6LoWPAN channel", - .new_connection = chan_new_conn_cb, .recv = chan_recv_cb, .close = chan_close_cb, .state_change = chan_state_change_cb, @@ -1029,16 +1001,19 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, hci_dev_lock(hdev); hcon = hci_conn_hash_lookup_le(hdev, addr, le_addr_type); - hci_dev_unlock(hdev); - hci_dev_put(hdev); - - if (!hcon) + if (!hcon) { + hci_dev_unlock(hdev); + hci_dev_put(hdev); return -ENOENT; + } - *conn = (struct l2cap_conn *)hcon->l2cap_data; + *conn = l2cap_conn_hold_unless_zero(hcon->l2cap_data); BT_DBG("conn %p dst %pMR type %u", *conn, &hcon->dst, hcon->dst_type); + hci_dev_unlock(hdev); + hci_dev_put(hdev); + return 0; } @@ -1093,23 +1068,15 @@ static void disconnect_all_peers(void) } while (nchans); } -struct set_enable { - struct work_struct work; - bool flag; -}; - -static void do_enable_set(struct work_struct *work) +static void do_enable_set(bool flag) { - struct set_enable *set_enable = container_of(work, - struct set_enable, work); - - if (!set_enable->flag || enable_6lowpan != set_enable->flag) + if (!flag || enable_6lowpan != flag) /* Disconnect existing connections if 6lowpan is * disabled */ disconnect_all_peers(); - enable_6lowpan = set_enable->flag; + enable_6lowpan = flag; mutex_lock(&set_lock); if (listen_chan) { @@ -1121,22 +1088,11 @@ static void do_enable_set(struct work_struct *work) listen_chan = bt_6lowpan_listen(); mutex_unlock(&set_lock); - - kfree(set_enable); } static int lowpan_enable_set(void *data, u64 val) { - struct set_enable *set_enable; - - set_enable = kzalloc_obj(*set_enable); - if (!set_enable) - return -ENOMEM; - - set_enable->flag = !!val; - INIT_WORK(&set_enable->work, do_enable_set); - - schedule_work(&set_enable->work); + do_enable_set(!!val); return 0; } @@ -1185,18 +1141,22 @@ static ssize_t lowpan_control_write(struct file *fp, if (conn) { struct lowpan_peer *peer; - if (!is_bt_6lowpan(conn->hcon)) + if (!is_bt_6lowpan(conn->hcon)) { + l2cap_conn_put(conn); return -EINVAL; + } peer = lookup_peer(conn); if (peer) { BT_DBG("6LoWPAN connection already exists"); + l2cap_conn_put(conn); return -EALREADY; } BT_DBG("conn %p dst %pMR type %d user %u", conn, &conn->hcon->dst, conn->hcon->dst_type, addr_type); + l2cap_conn_put(conn); } ret = bt_6lowpan_connect(&addr, addr_type); @@ -1212,6 +1172,8 @@ static ssize_t lowpan_control_write(struct file *fp, return ret; ret = bt_6lowpan_disconnect(conn, addr_type); + if (conn) + l2cap_conn_put(conn); if (ret < 0) return ret; diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile index 41049b280887..ff466ea97436 100644 --- a/net/bluetooth/Makefile +++ b/net/bluetooth/Makefile @@ -25,3 +25,5 @@ bluetooth-$(CONFIG_BT_MSFTEXT) += msft.o bluetooth-$(CONFIG_BT_AOSPEXT) += aosp.o bluetooth-$(CONFIG_BT_DEBUGFS) += hci_debugfs.o bluetooth-$(CONFIG_BT_SELFTEST) += selftest.o + +CONTEXT_ANALYSIS := y diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index bcbc11c9cb15..411d66f24393 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -209,6 +209,7 @@ bool bt_sock_linked(struct bt_sock_list *l, struct sock *s) EXPORT_SYMBOL(bt_sock_linked); void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh) + __context_unsafe(/* conditional locking */) { const struct cred *old_cred; struct pid *old_pid; @@ -305,7 +306,7 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) restart: for (sk = bt_accept_get(parent, NULL); sk; sk = next) { - /* Prevent early freeing of sk due to unlink and sock_kill */ + /* The reference from bt_accept_get() keeps sk alive. */ lock_sock(sk); /* Check sk has not already been unlinked via @@ -321,13 +322,11 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) next = bt_accept_get(parent, sk); - /* sk is safely in the parent list so reduce reference count */ - sock_put(sk); - /* FIXME: Is this check still needed */ if (sk->sk_state == BT_CLOSED) { bt_accept_unlink(sk); release_sock(sk); + sock_put(sk); continue; } @@ -337,16 +336,6 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) if (newsock) sock_graft(sk, newsock); - /* Hand the caller a reference taken while sk is - * still locked. bt_accept_unlink() just dropped - * the accept-queue reference; without this hold a - * concurrent teardown (e.g. l2cap_conn_del() -> - * l2cap_sock_kill()) could free sk between - * release_sock() and the caller using it. Every - * caller drops this with sock_put() when done. - */ - sock_hold(sk); - release_sock(sk); if (next) sock_put(next); @@ -354,6 +343,7 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) } release_sock(sk); + sock_put(sk); } return NULL; @@ -826,7 +816,8 @@ EXPORT_SYMBOL(bt_sock_wait_ready); #ifdef CONFIG_PROC_FS static void *bt_seq_start(struct seq_file *seq, loff_t *pos) - __acquires(seq->private->l->lock) + __acquires_shared(&((struct bt_sock_list *) + pde_data(file_inode(seq->file)))->lock) { struct bt_sock_list *l = pde_data(file_inode(seq->file)); @@ -842,7 +833,8 @@ static void *bt_seq_next(struct seq_file *seq, void *v, loff_t *pos) } static void bt_seq_stop(struct seq_file *seq, void *v) - __releases(seq->private->l->lock) + __releases_shared(&((struct bt_sock_list *) + pde_data(file_inode(seq->file)))->lock) { struct bt_sock_list *l = pde_data(file_inode(seq->file)); diff --git a/net/bluetooth/bnep/Makefile b/net/bluetooth/bnep/Makefile index 8af9d56bb012..f42015cc3245 100644 --- a/net/bluetooth/bnep/Makefile +++ b/net/bluetooth/bnep/Makefile @@ -6,3 +6,5 @@ obj-$(CONFIG_BT_BNEP) += bnep.o bnep-objs := core.o sock.o netdev.o + +CONTEXT_ANALYSIS := y diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index add9a8f7535d..f7d88c33e23e 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -559,14 +559,18 @@ static int bnep_session(void *arg) return 0; } -static struct device *bnep_get_device(struct bnep_session *session) +static struct l2cap_conn *bnep_get_conn(struct bnep_session *session) { - struct l2cap_conn *conn = l2cap_pi(session->sock->sk)->chan->conn; + struct l2cap_chan *chan = l2cap_pi(session->sock->sk)->chan; + struct l2cap_conn *conn; - if (!conn || !conn->hcon) - return NULL; + l2cap_chan_lock(chan); + conn = chan->conn; + if (conn) + l2cap_conn_get(conn); + l2cap_chan_unlock(chan); - return &conn->hcon->dev; + return conn; } static const struct device_type bnep_type = { @@ -578,6 +582,7 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) u32 valid_flags = BIT(BNEP_SETUP_RESPONSE); struct net_device *dev; struct bnep_session *s, *ss; + struct l2cap_conn *conn = NULL; u8 dst[ETH_ALEN], src[ETH_ALEN]; int err; @@ -637,10 +642,18 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) bnep_set_default_proto_filter(s); #endif - SET_NETDEV_DEV(dev, bnep_get_device(s)); + conn = bnep_get_conn(s); + if (!conn) { + err = -ENOTCONN; + goto failed; + } + + SET_NETDEV_DEV(dev, &conn->hcon->dev); SET_NETDEV_DEVTYPE(dev, &bnep_type); err = register_netdev(dev); + l2cap_conn_put(conn); + conn = NULL; if (err) goto failed; @@ -662,6 +675,8 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) return 0; failed: + if (conn) + l2cap_conn_put(conn); up_write(&bnep_session_sem); free_netdev(dev); return err; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index c335372e4062..1966cd153d97 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -3178,26 +3178,11 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) conn->abort_reason = reason; - /* If the connection is pending check the command opcode since that - * might be blocking on hci_cmd_sync_work while waiting its respective - * event so we need to hci_cmd_sync_cancel to cancel it. - * - * hci_connect_le serializes the connection attempts so only one - * connection can be in BT_CONNECT at time. + /* Cancel the connect attempt. A return of 0 means the create command + * was still queued and got dequeued, so there is nothing to disconnect. */ - if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { - switch (hci_skb_event(hdev->sent_cmd)) { - case HCI_EV_CONN_COMPLETE: - case HCI_EV_LE_CONN_COMPLETE: - case HCI_EV_LE_ENHANCED_CONN_COMPLETE: - case HCI_EVT_LE_CIS_ESTABLISHED: - hci_cmd_sync_cancel(hdev, ECANCELED); - break; - } - /* Cancel connect attempt if still queued/pending */ - } else if (!hci_cancel_connect_sync(hdev, conn)) { + if (!hci_cancel_connect_sync(hdev, conn)) return 0; - } /* Run immediately if on cmd_sync_work since this may be called * as a result to MGMT_OP_DISCONNECT/MGMT_OP_UNPAIR which does diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 5ba9fe8261ec..d1e78ae7728e 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -62,6 +62,7 @@ static DEFINE_IDA(hci_index_ida); /* Get HCI device by index. * Device is held on return. */ static struct hci_dev *__hci_dev_get(int index, int *srcu_index) + __context_unsafe(/* conditional locking */) { struct hci_dev *hdev = NULL, *d; @@ -89,11 +90,13 @@ struct hci_dev *hci_dev_get(int index) } static struct hci_dev *hci_dev_get_srcu(int index, int *srcu_index) + __context_unsafe(/* conditional locking vs return */) { return __hci_dev_get(index, srcu_index); } static void hci_dev_put_srcu(struct hci_dev *hdev, int srcu_index) + __context_unsafe(/* conditional locking vs return */) { srcu_read_unlock(&hdev->srcu, srcu_index); hci_dev_put(hdev); diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c index 0635e4641db4..aadffaaff20e 100644 --- a/net/bluetooth/hci_debugfs.c +++ b/net/bluetooth/hci_debugfs.c @@ -1161,16 +1161,12 @@ static ssize_t force_no_mitm_write(struct file *file, size_t count, loff_t *ppos) { struct hci_dev *hdev = file->private_data; - char buf[32]; - size_t buf_size = min(count, (sizeof(buf) - 1)); bool enable; + int err; - if (copy_from_user(buf, user_buf, buf_size)) - return -EFAULT; - - buf[buf_size] = '\0'; - if (kstrtobool(buf, &enable)) - return -EINVAL; + err = kstrtobool_from_user(user_buf, count, &enable); + if (err) + return err; if (enable == hci_dev_test_flag(hdev, HCI_FORCE_NO_MITM)) return -EALREADY; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index b6d963ce26d0..5b867aa99332 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -269,7 +269,10 @@ static u8 hci_cc_reset(struct hci_dev *hdev, void *data, struct sk_buff *skb) { struct hci_ev_status *rp = data; - bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); + if (rp->status) + bt_dev_err(hdev, "status 0x%2.2x", rp->status); + else + bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); clear_bit(HCI_RESET, &hdev->flags); diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 3be8c3581c6c..b096fca86bb5 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -860,32 +860,6 @@ void hci_cmd_sync_cancel_entry(struct hci_dev *hdev, } EXPORT_SYMBOL(hci_cmd_sync_cancel_entry); -/* Dequeue one HCI command entry: - * - * - Lookup and cancel first entry that matches. - */ -bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev, - hci_cmd_sync_work_func_t func, - void *data, hci_cmd_sync_work_destroy_t destroy) -{ - struct hci_cmd_sync_work_entry *entry; - - mutex_lock(&hdev->cmd_sync_work_lock); - - entry = _hci_cmd_sync_lookup_entry(hdev, func, data, destroy); - if (!entry) { - mutex_unlock(&hdev->cmd_sync_work_lock); - return false; - } - - _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); - - mutex_unlock(&hdev->cmd_sync_work_lock); - - return true; -} -EXPORT_SYMBOL(hci_cmd_sync_dequeue_once); - /* Dequeue HCI command entry: * * - Lookup and cancel any entry that matches by function callback or data or @@ -3672,17 +3646,19 @@ static const struct hci_init_stage hci_init0[] = { int hci_reset_sync(struct hci_dev *hdev) { - int err; - set_bit(HCI_RESET, &hdev->flags); - err = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL, - HCI_CMD_TIMEOUT); - if (err) - return err; + return __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL, + HCI_CMD_TIMEOUT); +} - return 0; +/* Send a raw HCI reset for use by vendor drivers */ +int __hci_reset_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL, + HCI_INIT_TIMEOUT); } +EXPORT_SYMBOL(__hci_reset_sync); static int hci_init0_sync(struct hci_dev *hdev) { @@ -6633,6 +6609,11 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) bt_dev_dbg(hdev, "conn %p", conn); + /* Hold a reference so conn stays valid for the HCI_CONN_CREATE + * clear_bit() at done. + */ + hci_conn_get(conn); + clear_bit(HCI_CONN_SCANNING, &conn->flags); conn->state = BT_CONNECT; @@ -6645,6 +6626,7 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) hdev->le_scan_type == LE_SCAN_ACTIVE && !hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) { hci_conn_del(conn); + hci_conn_put(conn); return -EBUSY; } @@ -6690,6 +6672,12 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) &own_addr_type); if (err) goto done; + + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + /* Send command LE Extended Create Connection if supported */ if (use_ext_conn(hdev)) { err = hci_le_ext_create_conn_sync(hdev, conn, own_addr_type); @@ -6725,11 +6713,14 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) conn->conn_timeout, NULL); done: + clear_bit(HCI_CONN_CREATE, &conn->flags); + if (err == -ETIMEDOUT) hci_le_connect_cancel_sync(hdev, conn, 0x00); /* Re-enable advertising after the connection attempt is finished. */ hci_resume_advertising_sync(hdev); + hci_conn_put(conn); return err; } @@ -7004,10 +6995,25 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) else cp.role_switch = 0x00; - return __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, - sizeof(cp), &cp, - HCI_EV_CONN_COMPLETE, - conn->conn_timeout, NULL); + /* Hold a reference so conn stays valid for the HCI_CONN_CREATE + * clear_bit() below. + */ + hci_conn_get(conn); + + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + + err = __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, + sizeof(cp), &cp, + HCI_EV_CONN_COMPLETE, + conn->conn_timeout, NULL); + + clear_bit(HCI_CONN_CREATE, &conn->flags); + hci_conn_put(conn); + + return err; } int hci_connect_acl_sync(struct hci_dev *hdev, struct hci_conn *conn) @@ -7059,22 +7065,97 @@ int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn) return (err == -EEXIST) ? 0 : err; } -int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +static int hci_acl_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) { - if (conn->state != BT_OPEN) - return -EINVAL; + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock makes the HCI_CONN_CREATE test and the cancel + * atomic against the worker, which takes this lock to dequeue every + * entry: while it is held no other command can become pending, so + * hci_cmd_sync_cancel() cannot cancel an unrelated command. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + /* In flight: this connection owns the pending request, cancel it. */ + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + /* Still queued: a successful dequeue means it never started, so there + * is nothing to disconnect. + */ + entry = _hci_cmd_sync_lookup_entry(hdev, hci_acl_create_conn_sync, conn, + NULL); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_le_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock keeps the HCI_CONN_CREATE test and the cancel + * atomic against the cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + entry = _hci_cmd_sync_lookup_entry(hdev, hci_le_create_conn_sync, conn, + create_le_conn_complete); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } + +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_cis_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + /* LE Create CIS is shared by the whole CIG and cannot be dequeued + * per-connection, so only an in-flight command can be cancelled. + * cmd_sync_work_lock keeps the test and the cancel atomic against the + * cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + if (test_bit(HCI_CONN_CREATE_CIS, &conn->flags)) + hci_cmd_sync_cancel(hdev, ECANCELED); + + mutex_unlock(&hdev->cmd_sync_work_lock); + return -EBUSY; +} + +int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +{ switch (conn->type) { case ACL_LINK: - return !hci_cmd_sync_dequeue_once(hdev, - hci_acl_create_conn_sync, - conn, NULL); + return hci_acl_cancel_create_conn_sync(hdev, conn); case LE_LINK: - return !hci_cmd_sync_dequeue_once(hdev, hci_le_create_conn_sync, - conn, create_le_conn_complete); + return hci_le_cancel_create_conn_sync(hdev, conn); + case CIS_LINK: + return hci_cis_cancel_create_conn_sync(hdev, conn); + default: + return -ENOENT; } - - return -ENOENT; } int hci_le_conn_update_sync(struct hci_dev *hdev, struct hci_conn *conn, diff --git a/net/bluetooth/hidp/Makefile b/net/bluetooth/hidp/Makefile index f41b0aa02b23..53e139e41bdc 100644 --- a/net/bluetooth/hidp/Makefile +++ b/net/bluetooth/hidp/Makefile @@ -6,3 +6,5 @@ obj-$(CONFIG_BT_HIDP) += hidp.o hidp-objs := core.o sock.o + +CONTEXT_ANALYSIS := y diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 793a481d7066..2e95a153912c 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -1590,6 +1590,7 @@ static void iso_conn_big_sync(struct sock *sk) { int err; struct hci_dev *hdev; + struct iso_conn *conn; bdaddr_t src, dst; u8 src_type; @@ -1612,8 +1613,17 @@ static void iso_conn_big_sync(struct sock *sk) hci_dev_lock(hdev); lock_sock(sk); + /* The socket lock was dropped for hci_get_route(), so the connection + * may have been torn down meanwhile: iso_chan_del() clears conn and + * the broadcast teardown path can clear conn->hcon on its own. Check + * both before dereferencing conn->hcon. + */ + conn = iso_pi(sk)->conn; + if (!conn || !conn->hcon) + goto unlock; + if (!test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { - err = hci_conn_big_create_sync(hdev, iso_pi(sk)->conn->hcon, + err = hci_conn_big_create_sync(hdev, conn->hcon, &iso_pi(sk)->qos, iso_pi(sk)->sync_handle, iso_pi(sk)->bc_num_bis, @@ -1622,6 +1632,7 @@ static void iso_conn_big_sync(struct sock *sk) bt_dev_err(hdev, "hci_big_create_sync: %d", err); } +unlock: release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); @@ -2529,7 +2540,7 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) switch (pb) { case ISO_START: case ISO_SINGLE: - if (conn->rx_len) { + if (conn->rx_skb || conn->rx_len) { BT_ERR("Unexpected start frame (len %d)", skb->len); kfree_skb(conn->rx_skb); conn->rx_skb = NULL; @@ -2610,12 +2621,14 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) break; case ISO_CONT: - BT_DBG("Cont: frag len %d (expecting %d)", skb->len, + case ISO_END: + BT_DBG("%s: frag len %d (expecting %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len, conn->rx_len); - if (!conn->rx_len) { - BT_ERR("Unexpected continuation frame (len %d)", - skb->len); + if (!conn->rx_skb) { + BT_ERR("Unexpected ISO %s frame (len %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len); goto drop; } @@ -2631,17 +2644,9 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), skb->len); conn->rx_len -= skb->len; - break; - - case ISO_END: - if (!conn->rx_len) { - BT_ERR("Unexpected end frame (len %d)", skb->len); - goto drop; - } - skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), - skb->len); - conn->rx_len -= skb->len; + if (pb == ISO_CONT) + break; if (!conn->rx_len) { struct sk_buff *rx_skb = conn->rx_skb; @@ -2652,6 +2657,13 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) */ conn->rx_skb = NULL; iso_recv_frame(conn, rx_skb); + } else { + BT_ERR("ISO fragment incomplete (len %d, expected %d)", + skb->len, conn->rx_len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + goto drop; } break; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 62133eef9d2f..538ae9aa3479 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -522,7 +522,10 @@ void l2cap_chan_put(struct l2cap_chan *c) } EXPORT_SYMBOL_GPL(l2cap_chan_put); -void l2cap_chan_set_defaults(struct l2cap_chan *chan) +/* Initialise @chan with default values, inheriting from the parent channel + * @pchan when it is given. + */ +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan) { chan->fcs = L2CAP_FCS_CRC16; chan->max_tx = L2CAP_DEFAULT_MAX_TX; @@ -536,6 +539,31 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + if (pchan) { + BT_DBG("chan %p pchan %p", chan, pchan); + + chan->chan_type = pchan->chan_type; + chan->imtu = pchan->imtu; + chan->omtu = pchan->omtu; + chan->mode = pchan->mode; + chan->fcs = pchan->fcs; + chan->max_tx = pchan->max_tx; + chan->tx_win = pchan->tx_win; + chan->tx_win_max = pchan->tx_win_max; + chan->sec_level = pchan->sec_level; + chan->conf_state = pchan->conf_state; + chan->flags = pchan->flags; + chan->tx_credits = pchan->tx_credits; + chan->rx_credits = pchan->rx_credits; + + if (chan->chan_type == L2CAP_CHAN_FIXED) { + chan->scid = pchan->scid; + chan->dcid = pchan->scid; + } + + return; + } + chan->conf_state = 0; set_bit(CONF_NOT_COMPLETE, &chan->conf_state); @@ -1775,19 +1803,13 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) disable_delayed_work_sync(&conn->info_timer); disable_delayed_work_sync(&conn->id_addr_timer); + cancel_work_sync(&conn->pending_rx_work); + mutex_lock(&conn->lock); kfree_skb(conn->rx_skb); skb_queue_purge(&conn->pending_rx); - - /* We can not call flush_work(&conn->pending_rx_work) here since we - * might block if we are running on a worker from the same workqueue - * pending_rx_work is waiting on. - */ - if (work_pending(&conn->pending_rx_work)) - cancel_work_sync(&conn->pending_rx_work); - ida_destroy(&conn->tx_ida); l2cap_unregister_all_users(conn); @@ -3051,13 +3073,24 @@ static struct sk_buff *l2cap_build_cmd(struct l2cap_conn *conn, u8 code, return NULL; } -static inline int l2cap_get_conf_opt(void **ptr, int *type, int *olen, - unsigned long *val) +static inline int l2cap_get_conf_opt(void **ptr, void *end, int *type, + int *olen, unsigned long *val) { struct l2cap_conf_opt *opt = *ptr; int len; + /* opt->len is attacker-controlled. Validate that the full option + * (header + value) actually fits in the buffer before touching + * opt->val, otherwise the switch below reads past the end of the + * caller's buffer. + */ + if (end - *ptr < L2CAP_CONF_OPT_SIZE) + return -EINVAL; + len = L2CAP_CONF_OPT_SIZE + opt->len; + if (end - *ptr < len) + return -EINVAL; + *ptr += len; *type = opt->type; @@ -3429,6 +3462,7 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data void *ptr = rsp->data; void *endptr = data + data_size; void *req = chan->conf_req; + void *req_end = req + chan->conf_len; int len = chan->conf_len; int type, hint, olen; unsigned long val; @@ -3442,9 +3476,11 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data BT_DBG("chan %p", chan); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&req, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&req, req_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; hint = type & L2CAP_CONF_HINT; type &= L2CAP_CONF_MASK; @@ -3672,6 +3708,7 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, struct l2cap_conf_req *req = data; void *ptr = req->data; void *endptr = data + size; + void *rsp_end = rsp + len; int type, olen; unsigned long val; struct l2cap_conf_rfc rfc = { .mode = L2CAP_MODE_BASIC }; @@ -3680,9 +3717,11 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, BT_DBG("chan %p, rsp %p, len %d, req %p", chan, rsp, len, data); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_MTU: @@ -3933,6 +3972,7 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) { int type, olen; unsigned long val; + void *rsp_end = rsp + len; /* Use sane default values in case a misbehaving remote device * did not send an RFC or extended window size option. */ @@ -3951,9 +3991,11 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) return; while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_RFC: @@ -4010,6 +4052,38 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn, return 0; } +/* Allocate and initialise a channel for an incoming connection. + * + * The channel inherits its configuration from @pchan and is linked into @conn + * before ->new_connection() runs, so the conn list reference keeps it alive if + * the callback exposes it (e.g. via the socket accept queue) before this + * returns. The l2cap_chan_create() reference is taken over by the subsystem on + * success and dropped here on failure. + */ +static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn, + struct l2cap_chan *pchan) +{ + struct l2cap_chan *chan; + + chan = l2cap_chan_create(); + if (!chan) + return NULL; + + l2cap_chan_set_defaults(chan, pchan); + chan->ops = pchan->ops; + + __l2cap_chan_add(conn, chan); + + if (pchan->ops->new_connection && + pchan->ops->new_connection(pchan, chan) < 0) { + l2cap_chan_del(chan, 0); + l2cap_chan_put(chan); + return NULL; + } + + return chan; +} + static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u8 *data, u8 rsp_code) { @@ -4056,7 +4130,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, goto response; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) goto response; @@ -4074,8 +4148,6 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, chan->psm = psm; chan->dcid = scid; - __l2cap_chan_add(conn, chan); - dcid = chan->scid; __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); @@ -4807,6 +4879,7 @@ static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id) case L2CAP_ECHO_RSP: case L2CAP_INFO_RSP: case L2CAP_CONN_PARAM_UPDATE_RSP: + case L2CAP_LE_CONN_RSP: case L2CAP_ECRED_CONN_RSP: case L2CAP_ECRED_RECONF_RSP: /* First do a lookup since the remote may send bogus ids that @@ -4958,7 +5031,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; goto response_unlock; @@ -4973,8 +5046,6 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_le_flowctl_init(chan, __le16_to_cpu(req->credits)); dcid = chan->scid; @@ -5182,7 +5253,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, continue; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; continue; @@ -5197,8 +5268,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_ecred_init(chan, __le16_to_cpu(req->credits)); /* Init response */ @@ -6704,6 +6773,7 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) struct l2cap_conn *conn = chan->conn; struct l2cap_le_credits pkt; u16 return_credits = l2cap_le_rx_credits(chan); + int ident; if (chan->mode != L2CAP_MODE_LE_FLOWCTL && chan->mode != L2CAP_MODE_EXT_FLOWCTL) @@ -6721,9 +6791,18 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) pkt.cid = cpu_to_le16(chan->scid); pkt.credits = cpu_to_le16(return_credits); - chan->ident = l2cap_get_ident(conn); + ident = l2cap_get_ident(conn); + + l2cap_send_cmd(conn, ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); - l2cap_send_cmd(conn, chan->ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); + /* L2CAP_LE_CREDITS has no response so the ident is never released by + * l2cap_put_ident() - release it right away, otherwise the tx_ida + * range is exhausted after 254 packets and from then on credits are + * sent with the invalid ident 0, which some remote stacks ignore, + * stalling the channel. + */ + if (ident > 0) + ida_free(&conn->tx_ida, ident); } void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail) @@ -7478,14 +7557,12 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) goto next; l2cap_chan_lock(pchan); - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (chan) { bacpy(&chan->src, &hcon->src); bacpy(&chan->dst, &hcon->dst); chan->src_type = bdaddr_src_type(hcon); chan->dst_type = dst_type; - - __l2cap_chan_add(conn, chan); } l2cap_chan_unlock(pchan); @@ -7702,6 +7779,7 @@ struct l2cap_conn *l2cap_conn_hold_unless_zero(struct l2cap_conn *c) return c; } +EXPORT_SYMBOL(l2cap_conn_hold_unless_zero); int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 4853f1b33449..735167f73f31 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -43,7 +43,8 @@ static struct bt_sock_list l2cap_sk_list = { static const struct proto_ops l2cap_sock_ops; static void l2cap_sock_init(struct sock *sk, struct sock *parent); static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern); + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan); static void l2cap_sock_cleanup_listen(struct sock *parent); bool l2cap_is_socket(struct socket *sock) @@ -1284,6 +1285,23 @@ static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg, return err; } +/* Release the sock's ref on chan and clear the pointer so that the ref is + * dropped exactly once even if both l2cap_sock_kill() and + * l2cap_sock_destruct() run. Setting chan->data to NULL first stops any other + * task from dereferencing the now-dead sock pointer. + */ +static void l2cap_sock_put_chan(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + if (!chan) + return; + + chan->data = NULL; + l2cap_pi(sk)->chan = NULL; + l2cap_chan_put(chan); +} + /* Kill socket (only if zapped and orphan) * Must be called on unlocked socket, with l2cap channel lock. */ @@ -1294,13 +1312,9 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); - /* Sock is dead, so set chan data to NULL, avoid other task use invalid - * sock pointer. - */ - l2cap_pi(sk)->chan->data = NULL; - /* Kill poor orphan */ + l2cap_sock_put_chan(sk); - l2cap_chan_put(l2cap_pi(sk)->chan); + /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); sock_put(sk); } @@ -1351,6 +1365,7 @@ static int __l2cap_wait_ack(struct sock *sk, struct l2cap_chan *chan) } static int l2cap_sock_shutdown(struct socket *sock, int how) + __context_unsafe(/* complex chan->conn locking */) { struct sock *sk = sock->sk; struct l2cap_chan *chan; @@ -1492,8 +1507,8 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) /* Close not yet accepted channels. * - * bt_accept_dequeue() now returns sk with an extra reference held - * (taken while sk was still locked) so a concurrent l2cap_conn_del() + * bt_accept_dequeue() returns sk with its temporary queue-walk + * reference held, so a concurrent l2cap_conn_del() * -> l2cap_sock_kill() cannot free sk under us. * * cleanup_listen() runs under the parent sk lock, so unlike @@ -1543,12 +1558,13 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) } } -static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) +static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { struct sock *sk, *parent = chan->data; if (!parent) - return NULL; + return -EINVAL; lock_sock(parent); @@ -1556,25 +1572,28 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); release_sock(parent); - return NULL; + return -ENOBUFS; } sk = l2cap_sock_alloc(sock_net(parent), NULL, BTPROTO_L2CAP, - GFP_ATOMIC, 0); + GFP_ATOMIC, 0, new_chan); if (!sk) { release_sock(parent); - return NULL; - } + return -ENOMEM; + } bt_sock_reclassify_lock(sk, BTPROTO_L2CAP); l2cap_sock_init(sk, parent); + /* The conn list reference taken by l2cap_new_connection() keeps new_chan + * alive once release_sock() lets another task free this socket. + */ bt_accept_enqueue(parent, sk, false); release_sock(parent); - return l2cap_pi(sk)->chan; + return 0; } static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) @@ -1871,10 +1890,7 @@ static void l2cap_sock_destruct(struct sock *sk) BT_DBG("sk %p", sk); - if (l2cap_pi(sk)->chan) { - l2cap_pi(sk)->chan->data = NULL; - l2cap_chan_put(l2cap_pi(sk)->chan); - } + l2cap_sock_put_chan(sk); list_for_each_entry_safe(rx_busy, next, &l2cap_pi(sk)->rx_busy, list) { kfree_skb(rx_busy->skb); @@ -1907,30 +1923,12 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) BT_DBG("sk %p", sk); if (parent) { - struct l2cap_chan *pchan = l2cap_pi(parent)->chan; - sk->sk_type = parent->sk_type; bt_sk(sk)->flags = bt_sk(parent)->flags; - chan->chan_type = pchan->chan_type; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->conf_state = pchan->conf_state; - chan->mode = pchan->mode; - chan->fcs = pchan->fcs; - chan->max_tx = pchan->max_tx; - chan->tx_win = pchan->tx_win; - chan->tx_win_max = pchan->tx_win_max; - chan->sec_level = pchan->sec_level; - chan->flags = pchan->flags; - chan->tx_credits = pchan->tx_credits; - chan->rx_credits = pchan->rx_credits; - - if (chan->chan_type == L2CAP_CHAN_FIXED) { - chan->scid = pchan->scid; - chan->dcid = pchan->scid; - } - + /* Channel configuration is inherited from the parent by + * l2cap_new_connection(). + */ security_sk_clone(parent, sk); } else { switch (sk->sk_type) { @@ -1956,7 +1954,7 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) chan->mode = L2CAP_MODE_BASIC; } - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); } /* Default config options */ @@ -1975,10 +1973,10 @@ static struct proto l2cap_proto = { }; static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern) + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan) { struct sock *sk; - struct l2cap_chan *chan; sk = bt_sock_alloc(net, sock, &l2cap_proto, proto, prio, kern); if (!sk) @@ -1989,16 +1987,7 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); - chan = l2cap_chan_create(); - if (!chan) { - sk_free(sk); - if (sock) - sock->sk = NULL; - return NULL; - } - - l2cap_chan_hold(chan); - + /* The sock takes ownership of the caller's reference on chan. */ l2cap_pi(sk)->chan = chan; return sk; @@ -2008,6 +1997,7 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; + struct l2cap_chan *chan; BT_DBG("sock %p", sock); @@ -2022,10 +2012,16 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, sock->ops = &l2cap_sock_ops; - sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); - if (!sk) + chan = l2cap_chan_create(); + if (!chan) return -ENOMEM; + sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern, chan); + if (!sk) { + l2cap_chan_put(chan); + return -ENOMEM; + } + l2cap_sock_init(sk, NULL); bt_sock_link(&l2cap_sk_list, sk); return 0; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index d23ca1dd0893..733a4b70e10c 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -5375,6 +5375,8 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, if (monitor->state == ADV_MONITOR_STATE_NOT_REGISTERED) monitor->state = ADV_MONITOR_STATE_REGISTERED; hci_update_passive_scan(hdev); + } else { + hci_free_adv_monitor(hdev, monitor); } mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, @@ -7658,6 +7660,8 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) if (!err) { struct hci_conn_params *params; + hci_dev_lock(hdev); + params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); @@ -7666,6 +7670,7 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) device_flags_changed(NULL, hdev, &cp->addr.bdaddr, cp->addr.type, hdev->conn_flags, params ? params->flags : 0); + hci_dev_unlock(hdev); } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_ADD_DEVICE, diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index 2f008167cbaa..d7badce8746c 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -291,7 +291,7 @@ static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, monitor->state = ADV_MONITOR_STATE_OFFLOADED; unlock: - if (status) + if (status && msft->resuming) hci_free_adv_monitor(hdev, monitor); hci_dev_unlock(hdev); diff --git a/net/bluetooth/rfcomm/Makefile b/net/bluetooth/rfcomm/Makefile index 593e5c48c131..15f909f40f25 100644 --- a/net/bluetooth/rfcomm/Makefile +++ b/net/bluetooth/rfcomm/Makefile @@ -7,3 +7,5 @@ obj-$(CONFIG_BT_RFCOMM) += rfcomm.o rfcomm-y := core.o sock.o rfcomm-$(CONFIG_BT_RFCOMM_TTY) += tty.o + +CONTEXT_ANALYSIS := y diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index feb302a491fa..958081adb9b5 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -60,6 +60,7 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) } static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err) + __must_hold(&d->lock) { struct sock *sk = d->owner, *parent; diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index fcc597be5bbd..c05f79b7aa31 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -570,10 +570,23 @@ static void __sco_sock_close(struct sock *sk) /* Must be called on unlocked socket. */ static void sco_sock_close(struct sock *sk) { + struct sco_conn *conn; + + lock_sock(sk); + conn = sco_pi(sk)->conn; + if (conn) + sco_conn_hold(conn); + release_sock(sk); + + if (conn) + disable_delayed_work_sync(&conn->timeout_work); + lock_sock(sk); - sco_sock_clear_timer(sk); __sco_sock_close(sk); release_sock(sk); + + if (conn) + sco_conn_put(conn); } static void sco_sock_init(struct sock *sk, struct sock *parent) diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 031d3022cb1e..c4470958b0d5 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -3201,34 +3201,19 @@ static const struct l2cap_ops smp_chan_ops = { .get_sndtimeo = l2cap_chan_no_get_sndtimeo, }; -static inline struct l2cap_chan *smp_new_conn_cb(struct l2cap_chan *pchan) +static inline int smp_new_conn_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - struct l2cap_chan *chan; - - BT_DBG("pchan %p", pchan); - - chan = l2cap_chan_create(); - if (!chan) - return NULL; - - chan->chan_type = pchan->chan_type; - chan->ops = &smp_chan_ops; - chan->scid = pchan->scid; - chan->dcid = chan->scid; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->mode = pchan->mode; + new_chan->ops = &smp_chan_ops; /* Other L2CAP channels may request SMP routines in order to * change the security level. This means that the SMP channel * lock must be considered in its own category to avoid lockdep * warnings. */ - atomic_set(&chan->nesting, L2CAP_NESTING_SMP); - - BT_DBG("created chan %p", chan); + atomic_set(&new_chan->nesting, L2CAP_NESTING_SMP); - return chan; + return 0; } static const struct l2cap_ops smp_root_chan_ops = { @@ -3288,7 +3273,7 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid) l2cap_add_scid(chan, cid); - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); if (cid == L2CAP_CID_SMP) { u8 bdaddr_type; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 7fdee8f52ee2..5d51f6cb7d15 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -452,12 +452,8 @@ static int bpf_test_finish(const union bpf_attr *kattr, } if (data_out) { - int len = sinfo ? copy_size - frag_size : copy_size; - - if (len < 0) { - err = -ENOSPC; - goto out; - } + u32 head_len = size - frag_size; + u32 len = min(copy_size, head_len); if (copy_to_user(data_out, data, len)) goto out; diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index 7ed19aa8ae59..c52613431f88 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -392,6 +392,9 @@ void br_dev_delete(struct net_device *dev, struct list_head *head) br_fdb_delete_by_port(br, NULL, 0, 1); + timer_shutdown_sync(&br->hello_timer); + timer_shutdown_sync(&br->topology_change_timer); + timer_shutdown_sync(&br->tcn_timer); cancel_delayed_work_sync(&br->gc_work); br_sysfs_delbr(br->dev); diff --git a/net/bridge/br_nf_core.c b/net/bridge/br_nf_core.c index a8c67035e23c..5b3a85b4be8f 100644 --- a/net/bridge/br_nf_core.c +++ b/net/bridge/br_nf_core.c @@ -70,6 +70,7 @@ void br_netfilter_rtable_init(struct net_bridge *br) struct rtable *rt = &br->fake_rtable; rcuref_init(&rt->dst.__rcuref, 1); + save_dst_trace_buffer(&rt->dst, 1); rt->dst.dev = br->dev; dst_init_metrics(&rt->dst, br->metrics, false); dst_metric_set(&rt->dst, RTAX_MTU, br->dev->mtu); diff --git a/net/bridge/br_stp.c b/net/bridge/br_stp.c index 46919d73d42f..c7e7e924f155 100644 --- a/net/bridge/br_stp.c +++ b/net/bridge/br_stp.c @@ -382,7 +382,8 @@ void br_topology_change_detection(struct net_bridge *br) { int isroot = br_is_root_bridge(br); - if (br->stp_enabled != BR_KERNEL_STP) + if (br->stp_enabled != BR_KERNEL_STP || + !(br->dev->flags & IFF_UP)) return; br_info(br, "topology change detected, %s\n", diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index f20c039e44c8..042d31278713 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -39,6 +39,8 @@ #define COUNTER_OFFSET(n) (SMP_ALIGN(n * sizeof(struct ebt_counter))) #define COUNTER_BASE(c, n, cpu) ((struct ebt_counter *)(((char *)c) + \ COUNTER_OFFSET(n) * cpu)) +#define MAX_EBT_ENTRIES (((INT_MAX - sizeof(struct ebt_table_info)) / \ + NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter)) struct ebt_pernet { struct list_head tables; @@ -1124,10 +1126,9 @@ static int do_replace(struct net *net, sockptr_t arg, unsigned int len) return -EINVAL; /* overflow check */ - if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) / - NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter)) + if (tmp.nentries >= MAX_EBT_ENTRIES) return -ENOMEM; - if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter)) + if (tmp.num_counters >= MAX_EBT_ENTRIES) return -ENOMEM; tmp.name[sizeof(tmp.name) - 1] = 0; @@ -2265,10 +2266,9 @@ static int compat_copy_ebt_replace_from_user(struct ebt_replace *repl, if (tmp.entries_size == 0) return -EINVAL; - if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) / - NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter)) + if (tmp.nentries >= MAX_EBT_ENTRIES) return -ENOMEM; - if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter)) + if (tmp.num_counters >= MAX_EBT_ENTRIES) return -ENOMEM; memcpy(repl, &tmp, offsetof(struct ebt_replace, hook_entry)); diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index 219c40680260..e4c9aa1f64e2 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -44,7 +44,9 @@ static void nft_meta_bridge_get_eval(const struct nft_expr *expr, if (!br_dev || !br_vlan_enabled(br_dev)) goto err; - br_vlan_get_pvid_rcu(in, &p_pvid); + if (br_vlan_get_pvid_rcu(in, &p_pvid)) + goto err; + nft_reg_store16(dest, p_pvid); return; } @@ -107,12 +109,30 @@ static int nft_meta_bridge_get_init(const struct nft_ctx *ctx, NULL, NFT_DATA_VALUE, len); } +static int nft_meta_bridge_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct nft_meta *priv = nft_expr_priv(expr); + unsigned int hooks; + + switch (priv->key) { + case NFT_META_BRI_IIFHWADDR: + hooks = 1 << NF_BR_PRE_ROUTING; + break; + default: + return nft_meta_get_validate(ctx, expr); + } + + return nft_chain_validate_hooks(ctx->chain, hooks); +} + static struct nft_expr_type nft_meta_bridge_type; static const struct nft_expr_ops nft_meta_bridge_get_ops = { .type = &nft_meta_bridge_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), .eval = nft_meta_bridge_get_eval, .init = nft_meta_bridge_get_init, + .validate = nft_meta_bridge_get_validate, .dump = nft_meta_get_dump, }; @@ -168,7 +188,6 @@ static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, switch (priv->key) { case NFT_META_BRI_BROUTE: - case NFT_META_BRI_IIFHWADDR: hooks = 1 << NF_BR_PRE_ROUTING; break; default: diff --git a/net/core/Makefile b/net/core/Makefile index dc17c5a61e9a..b3fdcb4e355f 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -13,7 +13,7 @@ obj-y += dev.o dev_api.o dev_addr_lists.o dst.o netevent.o \ neighbour.o rtnetlink.o utils.o link_watch.o filter.o \ sock_diag.o dev_ioctl.o tso.o sock_reuseport.o \ fib_notifier.o xdp.o flow_offload.o gro.o \ - netdev-genl.o netdev-genl-gen.o gso.o + netdev-genl.o netdev-genl-gen.o netdev_work.o gso.o obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o diff --git a/net/core/dev.c b/net/core/dev.c index 5c01dfaa6c44..547e10516258 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1875,6 +1875,7 @@ const char *netdev_cmd_to_name(enum netdev_cmd cmd) N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE) N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA) N(XDP_FEAT_CHANGE) + N(DEBUG_UNREGISTER) } #undef N return "UNKNOWN_NETDEV_EVENT"; @@ -4018,6 +4019,9 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, return NULL; } +/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) + * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). + */ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) { netdev_features_t features; @@ -4089,7 +4093,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d skb->prev = skb; skb = validate_xmit_skb(skb, dev, again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) continue; if (!head) @@ -4860,8 +4864,11 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) goto recursion_alert; skb = validate_xmit_skb(skb, dev, &again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) { + if (PTR_ERR(skb) == -EINPROGRESS) + rc = NET_XMIT_SUCCESS; goto out; + } HARD_TX_LOCK(dev, txq, cpu); @@ -9822,6 +9829,7 @@ int netif_change_flags(struct net_device *dev, unsigned int flags, __dev_notify_flags(dev, old_flags, changes, 0, NULL); return ret; } +EXPORT_SYMBOL(netif_change_flags); int __netif_set_mtu(struct net_device *dev, int new_mtu) { @@ -11574,6 +11582,14 @@ int netdev_refcnt_read(const struct net_device *dev) } EXPORT_SYMBOL(netdev_refcnt_read); +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +static void dump_netdev_trace_buffer(const struct net_device *dev); +static void erase_netdev_trace_buffer(const struct net_device *dev); +#else +static inline void dump_netdev_trace_buffer(const struct net_device *dev) { } +static inline void erase_netdev_trace_buffer(const struct net_device *dev) { } +#endif + int netdev_unregister_timeout_secs __read_mostly = 10; #define WAIT_REFS_MIN_MSECS 1 @@ -11647,11 +11663,16 @@ static struct net_device *netdev_wait_allrefs_any(struct list_head *list) if (time_after(jiffies, warning_time + READ_ONCE(netdev_unregister_timeout_secs) * HZ)) { + rtnl_lock(); list_for_each_entry(dev, list, todo_list) { pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", dev->name, netdev_refcnt_read(dev)); ref_tracker_dir_print(&dev->refcnt_tracker, 10); + call_netdevice_notifiers(NETDEV_DEBUG_UNREGISTER, dev); + dump_netdev_trace_buffer(dev); } + __rtnl_unlock(); + rcu_barrier(); warning_time = jiffies; } @@ -12051,6 +12072,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, dev->priv_len = sizeof_priv; +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + INIT_LIST_HEAD(&dev->netdev_trace_buffer_list); +#endif ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev"); #ifdef CONFIG_PCPU_DEV_REFCNT dev->pcpu_refcnt = alloc_percpu(int); @@ -12093,6 +12117,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, INIT_LIST_HEAD(&dev->ptype_all); INIT_LIST_HEAD(&dev->ptype_specific); INIT_LIST_HEAD(&dev->net_notifier_list); + INIT_LIST_HEAD(&dev->work_node); #ifdef CONFIG_NET_SCHED hash_init(dev->qdisc_hash); #endif @@ -12226,6 +12251,8 @@ void free_netdev(struct net_device *dev) mutex_destroy(&dev->lock); + erase_netdev_trace_buffer(dev); + /* Compatibility with error handling in drivers */ if (dev->reg_state == NETREG_UNINITIALIZED || dev->reg_state == NETREG_DUMMY) { @@ -13330,3 +13357,172 @@ static int __init net_dev_init(void) } subsys_initcall(net_dev_init); + +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + +#define NETDEV_TRACE_BUFFER_SIZE 32768 +static struct netdev_trace_buffer { + struct list_head list; + int prev_count; + atomic_t count; + int nr_entries; + unsigned long entries[20]; +} netdev_trace_buffer[NETDEV_TRACE_BUFFER_SIZE]; +static LIST_HEAD(netdev_trace_buffer_list); +static DEFINE_SPINLOCK(netdev_trace_buffer_lock); +static bool netdev_trace_buffer_exhausted; + +static int netdev_trace_buffer_init(void) +{ + int i; + + for (i = 0; i < NETDEV_TRACE_BUFFER_SIZE; i++) + list_add_tail(&netdev_trace_buffer[i].list, &netdev_trace_buffer_list); + return 0; +} +pure_initcall(netdev_trace_buffer_init); + +static void dump_netdev_trace_buffer(const struct net_device *dev) +{ + struct netdev_trace_buffer *ptr; + int count, balance = 0, pos = 0; + + dump_dst_trace_buffer(dev); + list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list, + /* list elements can't go away. */ 1) { + pos++; + count = atomic_read(&ptr->count); + balance += count; + if (ptr->prev_count == count) + continue; + ptr->prev_count = count; + pr_info("Call trace for %s[%d] %+d at\n", dev->name, pos, count); + stack_trace_print(ptr->entries, ptr->nr_entries, 4); + cond_resched(); + } + if (!netdev_trace_buffer_exhausted) + pr_info("balance as of %s[%d] is %d\n", dev->name, pos, balance); +} + +static void erase_netdev_trace_buffer(const struct net_device *dev) +{ + struct netdev_trace_buffer *ptr; + unsigned long flags; + + spin_lock_irqsave(&netdev_trace_buffer_lock, flags); + while (!list_empty(&dev->netdev_trace_buffer_list)) { + ptr = list_first_entry(&dev->netdev_trace_buffer_list, typeof(*ptr), list); + list_del(&ptr->list); + list_add_tail(&ptr->list, &netdev_trace_buffer_list); + } + spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags); +} + +int trim_netdev_trace(unsigned long *entries, int nr_entries) +{ +#ifdef CONFIG_KALLSYMS + char buffer[32] = { }; + char *cp; + int i; + + if (in_softirq()) { + static unsigned long __data_racy caller; + + if (!caller) { + for (i = 0; i < nr_entries; i++) { + snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]); + cp = strchr(buffer, ' '); + if (cp) + *cp = '\0'; + if (!strcmp(buffer, "handle_softirqs")) { + caller = entries[i]; + break; + } + } + } + for (i = 0; i < nr_entries; i++) + if (entries[i] == caller) + return i + 1; + } else if (current->flags & PF_WQ_WORKER) { + static unsigned long __data_racy caller; + + if (!caller) { + for (i = 0; i < nr_entries; i++) { + snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]); + cp = strchr(buffer, ' '); + if (cp) + *cp = '\0'; + if (!strcmp(buffer, "process_one_work")) { + caller = entries[i]; + break; + } + } + } + for (i = 0; i < nr_entries; i++) + if (entries[i] == caller) + return i + 1; + } else { + for (i = 0; i < nr_entries; i++) { + snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]); + cp = strchr(buffer, ' '); + if (cp) + *cp = '\0'; + if (buffer[0] == 'k') { + if (!strcmp(buffer, "ksys_unshare")) + return i + 1; + } else if (buffer[0] == 's') { + if (!strcmp(buffer, "sock_sendmsg_nosec") || + !strcmp(buffer, "sock_recvmsg_nosec")) + return i + 1; + } else if (buffer[0] == '_') { + if (!strcmp(buffer, "__sys_bind") || + !strcmp(buffer, "__sock_release") || + !strcmp(buffer, "__sys_bpf")) + return i + 1; + } else { + if (!strcmp(buffer, "do_sock_setsockopt")) + return i + 1; + } + } + } +#endif + return nr_entries; +} +EXPORT_SYMBOL(trim_netdev_trace); + +void save_netdev_trace_buffer(struct net_device *dev, int delta) +{ + struct netdev_trace_buffer *ptr; + unsigned long entries[ARRAY_SIZE(ptr->entries)]; + unsigned long nr_entries; + unsigned long flags; + + if (in_nmi()) + return; + nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1); + nr_entries = trim_netdev_trace(entries, nr_entries); + list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list, + /* list elements can't go away. */ 1) { + if (ptr->nr_entries == nr_entries && + !memcmp(ptr->entries, entries, nr_entries * sizeof(unsigned long))) { + atomic_add(delta, &ptr->count); + return; + } + } + spin_lock_irqsave(&netdev_trace_buffer_lock, flags); + if (!list_empty(&netdev_trace_buffer_list)) { + ptr = list_first_entry(&netdev_trace_buffer_list, typeof(*ptr), list); + list_del(&ptr->list); + ptr->prev_count = 0; + atomic_set(&ptr->count, delta); + ptr->nr_entries = nr_entries; + memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long)); + list_add_tail_rcu(&ptr->list, &dev->netdev_trace_buffer_list); + } else { + netdev_trace_buffer_exhausted = true; + } + spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags); +} +EXPORT_SYMBOL(save_netdev_trace_buffer); + +#endif diff --git a/net/core/dev.h b/net/core/dev.h index 4121c50e7c88..5d0b0305d3ba 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -167,10 +167,19 @@ int dev_change_carrier(struct net_device *dev, bool new_carrier); void __dev_set_rx_mode(struct net_device *dev); int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify); void netif_rx_mode_init(struct net_device *dev); -bool netif_rx_mode_clean(struct net_device *dev); +void netif_rx_mode_run(struct net_device *dev); void netif_rx_mode_sync(struct net_device *dev); void netif_rx_mode_cancel_retry(struct net_device *dev); +/* Events for the async netdev work, tracked in netdev->work_core_pending. */ +enum netdev_work_core { + NETDEV_WORK_RX_MODE = BIT(0), /* run the rx_mode update */ +}; + +void __netdev_work_core_sched(struct net_device *dev, unsigned long event); +unsigned long +__netdev_work_core_cancel(struct net_device *dev, unsigned long mask); + void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, unsigned int gchanges, u32 portid, const struct nlmsghdr *nlh); diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index e17f64a65e17..08528ca0a8b3 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -12,17 +12,10 @@ #include #include #include -#include #include #include "dev.h" -static void netdev_rx_mode_work(struct work_struct *work); - -static LIST_HEAD(rx_mode_list); -static DEFINE_SPINLOCK(rx_mode_lock); -static DECLARE_WORK(rx_mode_work, netdev_rx_mode_work); - /* * General list handling functions */ @@ -1281,7 +1274,7 @@ void netif_rx_mode_cancel_retry(struct net_device *dev) dev->rx_mode_retry_count = 0; } -static void netif_rx_mode_run(struct net_device *dev) +void netif_rx_mode_run(struct net_device *dev) { struct netdev_hw_addr_list uc_snap, mc_snap, uc_ref, mc_ref; const struct net_device_ops *ops = dev->netdev_ops; @@ -1339,49 +1332,9 @@ static void netif_rx_mode_run(struct net_device *dev) } } -static void netdev_rx_mode_work(struct work_struct *work) -{ - struct net_device *dev; - - rtnl_lock(); - - while (true) { - spin_lock_bh(&rx_mode_lock); - if (list_empty(&rx_mode_list)) { - spin_unlock_bh(&rx_mode_lock); - break; - } - dev = list_first_entry(&rx_mode_list, struct net_device, - rx_mode_node); - list_del_init(&dev->rx_mode_node); - /* We must free netdev tracker under - * the spinlock protection. - */ - netdev_tracker_free(dev, &dev->rx_mode_tracker); - spin_unlock_bh(&rx_mode_lock); - - netdev_lock_ops(dev); - netif_rx_mode_run(dev); - netdev_unlock_ops(dev); - /* Use __dev_put() because netdev_tracker_free() was already - * called above. Must be after netdev_unlock_ops() to prevent - * netdev_run_todo() from freeing the device while still in use. - */ - __dev_put(dev); - } - - rtnl_unlock(); -} - static void netif_rx_mode_queue(struct net_device *dev) { - spin_lock_bh(&rx_mode_lock); - if (list_empty(&dev->rx_mode_node)) { - list_add_tail(&dev->rx_mode_node, &rx_mode_list); - netdev_hold(dev, &dev->rx_mode_tracker, GFP_ATOMIC); - } - spin_unlock_bh(&rx_mode_lock); - schedule_work(&rx_mode_work); + __netdev_work_core_sched(dev, NETDEV_WORK_RX_MODE); } static void netif_rx_mode_retry(struct timer_list *t) @@ -1394,7 +1347,6 @@ static void netif_rx_mode_retry(struct timer_list *t) void netif_rx_mode_init(struct net_device *dev) { - INIT_LIST_HEAD(&dev->rx_mode_node); __hw_addr_init(&dev->rx_mode_addr_cache); timer_setup(&dev->rx_mode_retry_timer, netif_rx_mode_retry, 0); } @@ -1442,24 +1394,6 @@ void dev_set_rx_mode(struct net_device *dev) netif_addr_unlock_bh(dev); } -bool netif_rx_mode_clean(struct net_device *dev) -{ - bool clean = false; - - spin_lock_bh(&rx_mode_lock); - if (!list_empty(&dev->rx_mode_node)) { - list_del_init(&dev->rx_mode_node); - clean = true; - /* We must release netdev tracker under - * the spinlock protection. - */ - netdev_tracker_free(dev, &dev->rx_mode_tracker); - } - spin_unlock_bh(&rx_mode_lock); - - return clean; -} - /** * netif_rx_mode_sync() - sync rx mode inline * @dev: network device @@ -1473,11 +1407,6 @@ bool netif_rx_mode_clean(struct net_device *dev) */ void netif_rx_mode_sync(struct net_device *dev) { - if (netif_rx_mode_clean(dev)) { + if (__netdev_work_core_cancel(dev, NETDEV_WORK_RX_MODE)) netif_rx_mode_run(dev); - /* Use __dev_put() because netdev_tracker_free() was already - * called inside netif_rx_mode_clean(). - */ - __dev_put(dev); - } } diff --git a/net/core/dst.c b/net/core/dst.c index 092861133023..45452fa62e24 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -44,6 +44,82 @@ const struct dst_metrics dst_default_metrics = { }; EXPORT_SYMBOL(dst_default_metrics); +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + +#define DST_TRACE_BUFFER_SIZE 8192 +static struct dst_trace_buffer { + struct dst_entry *dst; // no-ref + struct net_device *ndev; // no-ref + int seq; + int delta; + u32 caller_id; + int nr_entries; + unsigned long entries[20]; +} dst_trace_buffer[DST_TRACE_BUFFER_SIZE]; +static bool dst_trace_buffer_exhausted; + +void dump_dst_trace_buffer(const struct net_device *ndev) +{ + struct dst_trace_buffer *ptr; + int count, balance = 0; + int i; + + for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++) { + ptr = &dst_trace_buffer[i]; + if (!ptr->dst || ptr->ndev != ndev) + continue; + count = ptr->delta; + balance += count; + pr_info("Call trace for %s@%p[%d] %+d %c%u at\n", ndev->name, ptr->dst, ptr->seq, + count, ptr->caller_id & 0x80000000 ? 'C' : 'T', + ptr->caller_id & ~0x80000000); + stack_trace_print(ptr->entries, ptr->nr_entries, 4); + } + if (!dst_trace_buffer_exhausted) + pr_info("balance for %s@dst_entry is %d\n", ndev->name, balance); + else + pr_info("balance for %s@dst_entry is unknown\n", ndev->name); +} + +static void erase_dst_trace_buffer(struct dst_entry *dst) +{ + int i; + + for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++) + if (dst_trace_buffer[i].dst == dst) + dst_trace_buffer[i].dst = NULL; +} + +void save_dst_trace_buffer(struct dst_entry *dst, int delta) +{ + struct dst_trace_buffer *ptr; + unsigned long entries[ARRAY_SIZE(ptr->entries)]; + unsigned long nr_entries; + int i; + + if (!dst->dev) + return; + nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1); + nr_entries = trim_netdev_trace(entries, nr_entries); + for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++) { + ptr = &dst_trace_buffer[i]; + if (!ptr->dst && !cmpxchg(&ptr->dst, NULL, dst)) { + ptr->ndev = dst->dev; + ptr->seq = atomic_inc_return(&dst->dst_trace_seq); + ptr->delta = delta; + ptr->caller_id = in_task() ? task_pid_nr(current) : + 0x80000000 + raw_smp_processor_id(); + ptr->nr_entries = nr_entries; + memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long)); + return; + } + } + dst_trace_buffer_exhausted = true; +} +EXPORT_SYMBOL(save_dst_trace_buffer); + +#endif + void dst_init(struct dst_entry *dst, struct dst_ops *ops, struct net_device *dev, int initial_obsolete, unsigned short flags) @@ -67,6 +143,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops, #endif dst->lwtstate = NULL; rcuref_init(&dst->__rcuref, 1); + save_dst_trace_buffer(dst, 1); INIT_LIST_HEAD(&dst->rt_uncached); dst->rt_uncached_list = NULL; dst->__use = 0; @@ -116,6 +193,10 @@ static void dst_destroy(struct dst_entry *dst) lwtstate_put(dst->lwtstate); +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + erase_dst_trace_buffer(dst); +#endif + if (dst->flags & DST_METADATA) metadata_dst_free((struct metadata_dst *)dst); else @@ -165,6 +246,8 @@ static void dst_count_dec(struct dst_entry *dst) void dst_release(struct dst_entry *dst) { + if (dst) + save_dst_trace_buffer(dst, -1); if (dst && rcuref_put(&dst->__rcuref)) { #ifdef CONFIG_DST_CACHE if (dst->flags & DST_METADATA) { @@ -182,6 +265,8 @@ EXPORT_SYMBOL(dst_release); void dst_release_immediate(struct dst_entry *dst) { + if (dst) + save_dst_trace_buffer(dst, -1); if (dst && rcuref_put(&dst->__rcuref)) { dst_count_dec(dst); dst_destroy(dst); diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c index cf374c208732..22e5e5e1a9c4 100644 --- a/net/core/fib_rules.c +++ b/net/core/fib_rules.c @@ -51,7 +51,6 @@ bool fib_rule_matchall(const struct fib_rule *rule) return false; return true; } -EXPORT_SYMBOL_GPL(fib_rule_matchall); int fib_default_rule_add(struct fib_rules_ops *ops, u32 pref, u32 table) @@ -78,7 +77,6 @@ int fib_default_rule_add(struct fib_rules_ops *ops, list_add_tail(&r->list, &ops->rules_list); return 0; } -EXPORT_SYMBOL(fib_default_rule_add); static u32 fib_default_rule_pref(struct fib_rules_ops *ops) { @@ -172,6 +170,7 @@ fib_rules_register(const struct fib_rules_ops *tmpl, struct net *net) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&ops->rules_list); + mutex_init(&ops->lock); ops->fro_net = net; err = __fib_rules_register(ops); @@ -182,7 +181,6 @@ fib_rules_register(const struct fib_rules_ops *tmpl, struct net *net) return ops; } -EXPORT_SYMBOL_GPL(fib_rules_register); static void fib_rules_cleanup_ops(struct fib_rules_ops *ops) { @@ -207,7 +205,6 @@ void fib_rules_unregister(struct fib_rules_ops *ops) fib_rules_cleanup_ops(ops); kfree_rcu(ops, rcu); } -EXPORT_SYMBOL_GPL(fib_rules_unregister); static int uid_range_set(struct fib_kuid_range *range) { @@ -363,7 +360,6 @@ int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl, return err; } -EXPORT_SYMBOL_GPL(fib_rules_lookup); static int call_fib_rule_notifier(struct notifier_block *nb, enum fib_event_type event_type, @@ -391,7 +387,7 @@ static int call_fib_rule_notifiers(struct net *net, .rule = rule, }; - ASSERT_RTNL_NET(net); + lockdep_assert_held(&ops->lock); /* Paired with READ_ONCE() in fib_rules_seq() */ WRITE_ONCE(ops->fib_rules_seq, ops->fib_rules_seq + 1); @@ -423,7 +419,6 @@ int fib_rules_dump(struct net *net, struct notifier_block *nb, int family, return err; } -EXPORT_SYMBOL_GPL(fib_rules_dump); unsigned int fib_rules_seq_read(const struct net *net, int family) { @@ -439,7 +434,6 @@ unsigned int fib_rules_seq_read(const struct net *net, int family) return fib_rules_seq; } -EXPORT_SYMBOL_GPL(fib_rules_seq_read); static struct fib_rule *rule_find(struct fib_rules_ops *ops, struct fib_rule_hdr *frh, @@ -740,10 +734,10 @@ static int fib_nl2rule(struct net *net, struct nlmsghdr *nlh, return err; } -static int fib_nl2rule_rtnl(struct fib_rule *nlrule, - struct fib_rules_ops *ops, - struct nlattr *tb[], - struct netlink_ext_ack *extack) +static int fib_nl2rule_locked(struct fib_rule *nlrule, + struct fib_rules_ops *ops, + struct nlattr *tb[], + struct netlink_ext_ack *extack) { if (!tb[FRA_PRIORITY]) nlrule->pref = fib_default_rule_pref(ops); @@ -754,12 +748,14 @@ static int fib_nl2rule_rtnl(struct fib_rule *nlrule, return -EINVAL; } + rcu_read_lock(); + if (tb[FRA_IIFNAME]) { struct net_device *dev; - dev = __dev_get_by_name(nlrule->fr_net, nlrule->iifname); + dev = dev_get_by_name_rcu(nlrule->fr_net, nlrule->iifname); if (dev) { - nlrule->iifindex = dev->ifindex; + nlrule->iifindex = READ_ONCE(dev->ifindex); nlrule->iif_is_l3_master = netif_is_l3_master(dev); } } @@ -767,13 +763,15 @@ static int fib_nl2rule_rtnl(struct fib_rule *nlrule, if (tb[FRA_OIFNAME]) { struct net_device *dev; - dev = __dev_get_by_name(nlrule->fr_net, nlrule->oifname); + dev = dev_get_by_name_rcu(nlrule->fr_net, nlrule->oifname); if (dev) { - nlrule->oifindex = dev->ifindex; + nlrule->oifindex = READ_ONCE(dev->ifindex); nlrule->oif_is_l3_master = netif_is_l3_master(dev); } } + rcu_read_unlock(); + return 0; } @@ -883,6 +881,7 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, struct nlattr *tb[FRA_MAX + 1]; bool user_priority = false; struct fib_rule_hdr *frh; + bool unlock_rtnl = false; frh = nlmsg_payload(nlh, sizeof(*frh)); if (!frh) { @@ -908,10 +907,13 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, if (err) goto errout; - if (!rtnl_held) + if (!rtnl_held && ops->need_rtnl && ops->need_rtnl(net)) { + unlock_rtnl = true; rtnl_net_lock(net); + } + mutex_lock(&ops->lock); - err = fib_nl2rule_rtnl(rule, ops, tb, extack); + err = fib_nl2rule_locked(rule, ops, tb, extack); if (err) goto errout_free; @@ -959,7 +961,7 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, list_for_each_entry(r, &ops->rules_list, list) { if (r->action == FR_ACT_GOTO && r->target == rule->pref && - rtnl_dereference(r->ctarget) == NULL) { + !rcu_access_pointer(r->ctarget)) { rcu_assign_pointer(r->ctarget, rule); if (--ops->unresolved_rules == 0) break; @@ -978,7 +980,8 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, fib_rule_get(rule); - if (!rtnl_held) + mutex_unlock(&ops->lock); + if (unlock_rtnl) rtnl_net_unlock(net); notify_rule_change(RTM_NEWRULE, rule, ops, nlh, NETLINK_CB(skb).portid); @@ -988,7 +991,8 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, return 0; errout_free: - if (!rtnl_held) + mutex_unlock(&ops->lock); + if (unlock_rtnl) rtnl_net_unlock(net); kfree(rule); errout: @@ -1037,10 +1041,9 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, if (err) goto errout; - if (!rtnl_held) - rtnl_net_lock(net); + mutex_lock(&ops->lock); - err = fib_nl2rule_rtnl(nlrule, ops, tb, extack); + err = fib_nl2rule_locked(nlrule, ops, tb, extack); if (err) goto errout_free; @@ -1055,11 +1058,8 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, goto errout_free; } - if (ops->delete) { - err = ops->delete(rule); - if (err) - goto errout_free; - } + if (ops->delete) + ops->delete(rule); if (rule->tun_id) ip_tunnel_unneed_metadata(); @@ -1068,7 +1068,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, if (rule->action == FR_ACT_GOTO) { ops->nr_goto_rules--; - if (rtnl_dereference(rule->ctarget) == NULL) + if (!rcu_access_pointer(rule->ctarget)) ops->unresolved_rules--; } @@ -1086,7 +1086,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, if (&n->list == &ops->rules_list || n->pref != rule->pref) n = NULL; list_for_each_entry(r, &ops->rules_list, list) { - if (rtnl_dereference(r->ctarget) != rule) + if (rcu_access_pointer(r->ctarget) != rule) continue; rcu_assign_pointer(r->ctarget, n); if (!n) @@ -1096,8 +1096,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops, NULL); - if (!rtnl_held) - rtnl_net_unlock(net); + mutex_unlock(&ops->lock); notify_rule_change(RTM_DELRULE, rule, ops, nlh, NETLINK_CB(skb).portid); fib_rule_put(rule); @@ -1107,8 +1106,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh, return 0; errout_free: - if (!rtnl_held) - rtnl_net_unlock(net); + mutex_unlock(&ops->lock); kfree(nlrule); errout: rules_ops_put(ops); @@ -1402,24 +1400,30 @@ static int fib_rules_event(struct notifier_block *this, unsigned long event, struct net *net = dev_net(dev); struct fib_rules_ops *ops; - ASSERT_RTNL(); - switch (event) { case NETDEV_REGISTER: - list_for_each_entry(ops, &net->rules_ops, list) + list_for_each_entry(ops, &net->rules_ops, list) { + mutex_lock(&ops->lock); attach_rules(&ops->rules_list, dev); + mutex_unlock(&ops->lock); + } break; case NETDEV_CHANGENAME: list_for_each_entry(ops, &net->rules_ops, list) { + mutex_lock(&ops->lock); detach_rules(&ops->rules_list, dev); attach_rules(&ops->rules_list, dev); + mutex_unlock(&ops->lock); } break; case NETDEV_UNREGISTER: - list_for_each_entry(ops, &net->rules_ops, list) + list_for_each_entry(ops, &net->rules_ops, list) { + mutex_lock(&ops->lock); detach_rules(&ops->rules_list, dev); + mutex_unlock(&ops->lock); + } break; } diff --git a/net/core/filter.c b/net/core/filter.c index 2e96b4b847ce..4f5cbcac3e78 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2529,16 +2529,18 @@ int skb_do_redirect(struct sk_buff *skb) if (unlikely(!dev)) goto out_drop; if (flags & BPF_F_PEER) { - if (unlikely(!skb_at_tc_ingress(skb))) - goto out_drop; dev = skb_get_peer_dev(dev); if (unlikely(!dev || !(dev->flags & IFF_UP) || net_eq(net, dev_net(dev)))) goto out_drop; + skb_scrub_packet(skb, false); + if (flags & BPF_F_EGRESS) + return __bpf_redirect(skb, dev, 0); + if (unlikely(!skb_at_tc_ingress(skb))) + goto out_drop; skb->dev = dev; dev_sw_netstats_rx_add(dev, skb->len); - skb_scrub_packet(skb, false); return -EAGAIN; } return flags & BPF_F_NEIGH ? @@ -2575,10 +2577,10 @@ BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags) { struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); - if (unlikely(flags)) + if (unlikely(flags & ~BPF_F_EGRESS)) return TC_ACT_SHOT; - ri->flags = BPF_F_PEER; + ri->flags = BPF_F_PEER | flags; ri->tgt_index = ifindex; return TC_ACT_REDIRECT; @@ -4472,7 +4474,7 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) struct net_device *master, *slave; master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev); - if (unlikely(!(master->flags & IFF_UP))) + if (unlikely(!master || !(master->flags & IFF_UP))) return XDP_ABORTED; slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp); if (slave && slave != xdp->rxq->dev) { @@ -6221,7 +6223,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct in_device *in_dev; struct net_device *dev; struct fib_result res; - struct flowi4 fl4; + struct flowi4 fl4 = {}; u32 mtu = 0; int err; @@ -6361,7 +6363,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct neighbour *neigh; struct net_device *dev; struct inet6_dev *idev; - struct flowi6 fl6; + struct flowi6 fl6 = {}; int strict = 0; int oif, err; u32 mtu = 0; diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 2a98f5fa74eb..8aa4f9b4df81 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1173,13 +1173,21 @@ bool __skb_flow_dissect(const struct net *net, if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { - struct ethhdr *eth = eth_hdr(skb); struct flow_dissector_key_eth_addrs *key_eth_addrs; key_eth_addrs = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS, target_container); - memcpy(key_eth_addrs, eth, sizeof(*key_eth_addrs)); + /* TC filter blocks can be shared across devices with + * different link types, so we cannot validate this + * when the filter is installed -- check at dissect time. + */ + if (skb && skb->dev && + skb->dev->type == ARPHRD_ETHER && + skb_mac_header_was_set(skb)) + memcpy(key_eth_addrs, eth_hdr(skb), sizeof(*key_eth_addrs)); + else + memset(key_eth_addrs, 0, sizeof(*key_eth_addrs)); } if (dissector_uses_key(flow_dissector, diff --git a/net/core/lock_debug.c b/net/core/lock_debug.c index 8a81c5430705..c1a44594c492 100644 --- a/net/core/lock_debug.c +++ b/net/core/lock_debug.c @@ -29,6 +29,7 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event, case NETDEV_DOWN: case NETDEV_REBOOT: case NETDEV_UNREGISTER: + case NETDEV_DEBUG_UNREGISTER: case NETDEV_CHANGEMTU: case NETDEV_CHANGEADDR: case NETDEV_PRE_CHANGEADDR: diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c index f9d76d85d04f..b01a395d9a96 100644 --- a/net/core/lwtunnel.c +++ b/net/core/lwtunnel.c @@ -350,6 +350,8 @@ int lwtunnel_output(struct net *net, struct sock *sk, struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->output)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->output(net, sk, skb); dev_xmit_recursion_dec(); @@ -404,6 +406,8 @@ int lwtunnel_xmit(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->xmit)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->xmit(skb); dev_xmit_recursion_dec(); @@ -455,6 +459,8 @@ int lwtunnel_input(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->input)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->input(skb); dev_xmit_recursion_dec(); diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index 11b0b91683d7..c15d8d4ca1f8 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -189,7 +190,8 @@ netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, goto nla_put_failure; if (napi->thread) { - pid = task_pid_nr(napi->thread); + pid = task_pid_nr_ns(napi->thread, + task_active_pid_ns(current)); if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) goto nla_put_failure; } diff --git a/net/core/netdev_work.c b/net/core/netdev_work.c new file mode 100644 index 000000000000..3109fae132ad --- /dev/null +++ b/net/core/netdev_work.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include +#include +#include +#include + +#include "dev.h" + +static void netdev_work_proc(struct work_struct *work); + +/* @netdev_work_lock protects: + * - @netdev_work_list + * - within the list entries (struct net_device fields): + * - work_node + * - work_tracker + * - work_pending + * - work_core_pending + */ +static LIST_HEAD(netdev_work_list); +static DEFINE_SPINLOCK(netdev_work_lock); +static DECLARE_WORK(netdev_work, netdev_work_proc); + +static void netdev_work_enqueue(struct net_device *dev, unsigned long events, + unsigned long core) +{ + if (!events && !core) + return; + + spin_lock_bh(&netdev_work_lock); + if (list_empty(&dev->work_node)) { + list_add_tail(&dev->work_node, &netdev_work_list); + netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC); + } + dev->work_pending |= events; + dev->work_core_pending |= core; + spin_unlock_bh(&netdev_work_lock); + + schedule_work(&netdev_work); +} + +static unsigned long +netdev_work_dequeue(struct net_device *dev, unsigned long *pending, + unsigned long mask) +{ + unsigned long events; + + spin_lock_bh(&netdev_work_lock); + events = *pending & mask; + *pending &= ~events; + if (!list_empty(&dev->work_node) && + !dev->work_pending && !dev->work_core_pending) { + list_del_init(&dev->work_node); + netdev_put(dev, &dev->work_tracker); + } + spin_unlock_bh(&netdev_work_lock); + + return events; +} + +void netdev_work_sched(struct net_device *dev, unsigned long events) +{ + netdev_work_enqueue(dev, events, 0); +} +EXPORT_SYMBOL(netdev_work_sched); + +/** + * netdev_work_cancel() - cancel selected work for a netdev + * @dev: net_device + * @mask: events to cancel + * + * Clear @mask from the device's work pending mask. If no work is left pending + * the device is dequeued and its ndo_work won't be called. + * + * No expectations on locking, but also no guarantees provided. If the caller + * wants to touch @dev afterwards (e.g. call the work that got canceled) + * they have to ensure @dev does not get freed. + * + * Returns: the subset of @mask that was actually pending, so the caller can run + * those events inline. + */ +unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask) +{ + return netdev_work_dequeue(dev, &dev->work_pending, mask); +} +EXPORT_SYMBOL(netdev_work_cancel); + +void __netdev_work_core_sched(struct net_device *dev, unsigned long events) +{ + netdev_work_enqueue(dev, 0, events); +} + +unsigned long +__netdev_work_core_cancel(struct net_device *dev, unsigned long mask) +{ + return netdev_work_dequeue(dev, &dev->work_core_pending, mask); +} + +static void netdev_work_run(struct net_device *dev, unsigned long events, + unsigned long core) +{ + if (!netif_device_present(dev)) + return; + + if (core & NETDEV_WORK_RX_MODE) + netif_rx_mode_run(dev); + if (events && dev->netdev_ops->ndo_work) + dev->netdev_ops->ndo_work(dev, events); +} + +static void netdev_work_proc(struct work_struct *work) +{ + rtnl_lock(); + + while (true) { + unsigned long events = 0, core = 0; + netdevice_tracker tracker; + struct net_device *dev; + + spin_lock_bh(&netdev_work_lock); + if (list_empty(&netdev_work_list)) { + spin_unlock_bh(&netdev_work_lock); + break; + } + dev = list_first_entry(&netdev_work_list, struct net_device, + work_node); + /* Take a temporary reference so @dev can't be freed while we + * drop the lock to grab its ops lock; the work reference is + * only released once we claim the work below. + * The re-locking dance is to ensure that ops lock is enough + * to ensure canceling work is not racy with dequeue. + */ + netdev_hold(dev, &tracker, GFP_ATOMIC); + spin_unlock_bh(&netdev_work_lock); + + netdev_lock_ops(dev); + spin_lock_bh(&netdev_work_lock); + if (!list_empty(&dev->work_node)) { + list_del_init(&dev->work_node); + core = dev->work_core_pending; + dev->work_core_pending = 0; + events = dev->work_pending; + dev->work_pending = 0; + /* We took another ref above */ + netdev_put(dev, &dev->work_tracker); + + if (!dev_isalive(dev)) + core = events = 0; + } + spin_unlock_bh(&netdev_work_lock); + + netdev_work_run(dev, events, core); + netdev_unlock_ops(dev); + + netdev_put(dev, &tracker); + } + + rtnl_unlock(); +} diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 229dde818ab3..aed415d3cd74 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -221,7 +221,7 @@ static void refill_skbs(struct netpoll *np) skb_pool = &np->skb_pool; while (READ_ONCE(skb_pool->qlen) < MAX_SKBS) { - skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC); + skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC | __GFP_NOWARN); if (!skb) break; @@ -633,14 +633,6 @@ static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) container_of(rcu_head, struct netpoll_info, rcu); skb_queue_purge(&npinfo->txq); - - /* we can't call cancel_delayed_work_sync here, as we are in softirq */ - cancel_delayed_work(&npinfo->tx_work); - - /* clean after last, unfinished work */ - __skb_queue_purge(&npinfo->txq); - /* now cancel it again */ - cancel_delayed_work(&npinfo->tx_work); kfree(npinfo); } @@ -664,6 +656,7 @@ static void __netpoll_cleanup(struct netpoll *np) ops->ndo_netpoll_cleanup(np->dev); RCU_INIT_POINTER(np->dev->npinfo, NULL); + disable_delayed_work_sync(&npinfo->tx_work); call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 61d095ce1b3b..12aa3aa1688b 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2438,6 +2438,14 @@ struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid) } EXPORT_SYMBOL_GPL(rtnl_get_net_ns_capable); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net) +{ + return net_eq(link_net, dev_net(dev)) || + ns_capable(link_net->user_ns, CAP_NET_ADMIN); +} +EXPORT_SYMBOL_GPL(rtnl_dev_link_net_capable); + static int rtnl_valid_dump_ifinfo_req(const struct nlmsghdr *nlh, bool strict_check, struct nlattr **tb, struct netlink_ext_ack *extack) diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 99e3789492a0..c60ba6d292f9 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1515,6 +1515,17 @@ static int sock_map_prog_link_lookup(struct bpf_map *map, struct bpf_prog ***ppr return 0; } +static int sock_map_prog_attach_check(enum bpf_attach_type attach_type, + struct bpf_prog *prog) +{ + /* A stream parser must not modify the skb, only measure it. */ + if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER && + prog->aux->changes_pkt_data) + return -EINVAL; + + return 0; +} + /* Handle the following four cases: * prog_attach: prog != NULL, old == NULL, link == NULL * prog_detach: prog == NULL, old != NULL, link == NULL @@ -1533,6 +1544,10 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, if (ret) return ret; + ret = sock_map_prog_attach_check(which, prog); + if (ret) + return ret; + /* for prog_attach/prog_detach/link_attach, return error if a bpf_link * exists for that prog. */ @@ -1776,6 +1791,11 @@ static int sock_map_link_update_prog(struct bpf_link *link, ret = -EINVAL; goto out; } + + ret = sock_map_prog_attach_check(link->attach_type, prog); + if (ret) + goto out; + if (!sockmap_link->map) { ret = -ENOLINK; goto out; diff --git a/net/devlink/rate.c b/net/devlink/rate.c index 41be2d6c2954..533d21b028a7 100644 --- a/net/devlink/rate.c +++ b/net/devlink/rate.c @@ -486,16 +486,19 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate, devlink_rate->tx_weight = weight; } - nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]; - if (nla_parent) { - err = devlink_nl_rate_parent_node_set(devlink_rate, info, - nla_parent); + if (attrs[DEVLINK_ATTR_RATE_TC_BWS]) { + err = devlink_nl_rate_tc_bw_set(devlink_rate, info); if (err) return err; } - if (attrs[DEVLINK_ATTR_RATE_TC_BWS]) { - err = devlink_nl_rate_tc_bw_set(devlink_rate, info); + /* Keep parent setting last because it takes a reference. This function + * has no rollback, so failing after taking the ref would leak it. + */ + nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME]; + if (nla_parent) { + err = devlink_nl_rate_parent_node_set(devlink_rate, info, + nla_parent); if (err) return err; } @@ -725,11 +728,6 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name, if (!rate_node) return ERR_PTR(-ENOMEM); - if (parent) { - rate_node->parent = parent; - refcount_inc(&rate_node->parent->refcnt); - } - rate_node->type = DEVLINK_RATE_TYPE_NODE; rate_node->devlink = devlink; rate_node->priv = priv; @@ -740,6 +738,11 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name, return ERR_PTR(-ENOMEM); } + if (parent) { + rate_node->parent = parent; + refcount_inc(&rate_node->parent->refcnt); + } + refcount_set(&rate_node->refcnt, 1); list_add(&rate_node->list, &devlink->rate_list); devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW); diff --git a/net/dsa/tag.c b/net/dsa/tag.c index 79ad105902d9..991732d6eae2 100644 --- a/net/dsa/tag.c +++ b/net/dsa/tag.c @@ -79,15 +79,16 @@ static int dsa_switch_rcv(struct sk_buff *skb, struct net_device *dev, if (likely(skb->dev)) { dsa_default_offload_fwd_mark(skb); nskb = skb; + } else { + /* Just drop the skb if we can't find the user */ + kfree_skb(skb); } } else { nskb = cpu_dp->rcv(skb, dev); } - if (!nskb) { - kfree_skb(skb); + if (!nskb) return 0; - } skb = nskb; skb_push(skb, ETH_HLEN); @@ -107,11 +108,10 @@ static int dsa_switch_rcv(struct sk_buff *skb, struct net_device *dev, if (unlikely(cpu_dp->ds->untag_bridge_pvid || cpu_dp->ds->untag_vlan_aware_bridge_pvid)) { + /* dsa_software_vlan_untag() drops skb on failure */ nskb = dsa_software_vlan_untag(skb); - if (!nskb) { - kfree_skb(skb); + if (!nskb) return 0; - } skb = nskb; } diff --git a/net/dsa/tag_ar9331.c b/net/dsa/tag_ar9331.c index cbb588ca73aa..2e2388143b02 100644 --- a/net/dsa/tag_ar9331.c +++ b/net/dsa/tag_ar9331.c @@ -51,8 +51,10 @@ static struct sk_buff *ar9331_tag_rcv(struct sk_buff *skb, u8 ver, port; u16 hdr; - if (unlikely(!pskb_may_pull(skb, AR9331_HDR_LEN))) + if (unlikely(!pskb_may_pull(skb, AR9331_HDR_LEN))) { + kfree_skb(skb); return NULL; + } hdr = le16_to_cpu(*(__le16 *)skb_mac_header(skb)); @@ -60,12 +62,14 @@ static struct sk_buff *ar9331_tag_rcv(struct sk_buff *skb, if (unlikely(ver != AR9331_HDR_VERSION)) { netdev_warn_once(ndev, "%s:%i wrong header version 0x%2x\n", __func__, __LINE__, hdr); + kfree_skb(skb); return NULL; } if (unlikely(hdr & AR9331_HDR_FROM_CPU)) { netdev_warn_once(ndev, "%s:%i packet should not be from cpu 0x%2x\n", __func__, __LINE__, hdr); + kfree_skb(skb); return NULL; } @@ -75,8 +79,10 @@ static struct sk_buff *ar9331_tag_rcv(struct sk_buff *skb, port = FIELD_GET(AR9331_HDR_PORT_NUM_MASK, hdr); skb->dev = dsa_conduit_find_user(ndev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } return skb; } diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index cf9420439054..411e3b57d16a 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -102,9 +102,9 @@ static struct sk_buff *brcm_tag_xmit_ll(struct sk_buff *skb, * (including FCS and tag) because the length verification is done after * the Broadcom tag is stripped off the ingress packet. * - * Let dsa_user_xmit() free the SKB + * Free the SKB on error. */ - if (__skb_put_padto(skb, ETH_ZLEN + BRCM_TAG_LEN, false)) + if (skb_put_padto(skb, ETH_ZLEN + BRCM_TAG_LEN)) return NULL; skb_push(skb, BRCM_TAG_LEN); @@ -151,27 +151,35 @@ static struct sk_buff *brcm_tag_rcv_ll(struct sk_buff *skb, int source_port; u8 *brcm_tag; - if (unlikely(!pskb_may_pull(skb, BRCM_TAG_LEN))) + if (unlikely(!pskb_may_pull(skb, BRCM_TAG_LEN))) { + kfree_skb(skb); return NULL; + } brcm_tag = skb->data - offset; /* The opcode should never be different than 0b000 */ - if (unlikely((brcm_tag[0] >> BRCM_OPCODE_SHIFT) & BRCM_OPCODE_MASK)) + if (unlikely((brcm_tag[0] >> BRCM_OPCODE_SHIFT) & BRCM_OPCODE_MASK)) { + kfree_skb(skb); return NULL; + } /* We should never see a reserved reason code without knowing how to * handle it */ - if (unlikely(brcm_tag[2] & BRCM_EG_RC_RSVD)) + if (unlikely(brcm_tag[2] & BRCM_EG_RC_RSVD)) { + kfree_skb(skb); return NULL; + } /* Locate which port this is coming from */ source_port = brcm_tag[3] & BRCM_EG_PID_MASK; skb->dev = dsa_conduit_find_user(dev, 0, source_port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } /* Remove Broadcom tag and update checksum */ skb_pull_rcsum(skb, BRCM_TAG_LEN); @@ -228,8 +236,10 @@ static struct sk_buff *brcm_leg_tag_rcv(struct sk_buff *skb, __be16 *proto; u8 *brcm_tag; - if (unlikely(!pskb_may_pull(skb, BRCM_LEG_TAG_LEN + VLAN_HLEN))) + if (unlikely(!pskb_may_pull(skb, BRCM_LEG_TAG_LEN + VLAN_HLEN))) { + kfree_skb(skb); return NULL; + } brcm_tag = dsa_etype_header_pos_rx(skb); proto = (__be16 *)(brcm_tag + BRCM_LEG_TAG_LEN); @@ -237,8 +247,10 @@ static struct sk_buff *brcm_leg_tag_rcv(struct sk_buff *skb, source_port = brcm_tag[5] & BRCM_LEG_PORT_ID; skb->dev = dsa_conduit_find_user(dev, 0, source_port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } /* The internal switch in BCM63XX SoCs always tags on egress on the CPU * port. We use VID 0 internally for untagged traffic, so strip the tag @@ -273,10 +285,8 @@ static struct sk_buff *brcm_leg_tag_xmit(struct sk_buff *skb, * need to make sure that packets are at least 70 bytes * (including FCS and tag) because the length verification is done after * the Broadcom tag is stripped off the ingress packet. - * - * Let dsa_user_xmit() free the SKB */ - if (__skb_put_padto(skb, ETH_ZLEN + BRCM_LEG_TAG_LEN, false)) + if (skb_put_padto(skb, ETH_ZLEN + BRCM_LEG_TAG_LEN)) return NULL; skb_push(skb, BRCM_LEG_TAG_LEN); @@ -325,10 +335,8 @@ static struct sk_buff *brcm_leg_fcs_tag_xmit(struct sk_buff *skb, * need to make sure that packets are at least 70 bytes (including FCS * and tag) because the length verification is done after the Broadcom * tag is stripped off the ingress packet. - * - * Let dsa_user_xmit() free the SKB. */ - if (__skb_put_padto(skb, ETH_ZLEN + BRCM_LEG_TAG_LEN, false)) + if (skb_put_padto(skb, ETH_ZLEN + BRCM_LEG_TAG_LEN)) return NULL; fcs_len = skb->len; @@ -351,8 +359,9 @@ static struct sk_buff *brcm_leg_fcs_tag_xmit(struct sk_buff *skb, brcm_tag[5] = dp->index & BRCM_LEG_PORT_ID; /* Original FCS value */ - if (__skb_pad(skb, ETH_FCS_LEN, false)) + if (skb_pad(skb, ETH_FCS_LEN)) return NULL; + skb_put_data(skb, &fcs_val, ETH_FCS_LEN); return skb; diff --git a/net/dsa/tag_dsa.c b/net/dsa/tag_dsa.c index 2a2c4fb61a65..d5ffee35fbb5 100644 --- a/net/dsa/tag_dsa.c +++ b/net/dsa/tag_dsa.c @@ -224,6 +224,7 @@ static struct sk_buff *dsa_rcv_ll(struct sk_buff *skb, struct net_device *dev, /* Remote management is not implemented yet, * drop. */ + kfree_skb(skb); return NULL; case DSA_CODE_ARP_MIRROR: case DSA_CODE_POLICY_MIRROR: @@ -244,12 +245,14 @@ static struct sk_buff *dsa_rcv_ll(struct sk_buff *skb, struct net_device *dev, /* Reserved code, this could be anything. Drop * seems like the safest option. */ + kfree_skb(skb); return NULL; } break; default: + kfree_skb(skb); return NULL; } @@ -271,8 +274,10 @@ static struct sk_buff *dsa_rcv_ll(struct sk_buff *skb, struct net_device *dev, source_port); } - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } /* When using LAG offload, skb->dev is not a DSA user interface, * so we cannot call dsa_default_offload_fwd_mark and we need to @@ -335,8 +340,10 @@ static struct sk_buff *dsa_xmit(struct sk_buff *skb, struct net_device *dev) static struct sk_buff *dsa_rcv(struct sk_buff *skb, struct net_device *dev) { - if (unlikely(!pskb_may_pull(skb, DSA_HLEN))) + if (unlikely(!pskb_may_pull(skb, DSA_HLEN))) { + kfree_skb(skb); return NULL; + } return dsa_rcv_ll(skb, dev, 0); } @@ -375,8 +382,10 @@ static struct sk_buff *edsa_xmit(struct sk_buff *skb, struct net_device *dev) static struct sk_buff *edsa_rcv(struct sk_buff *skb, struct net_device *dev) { - if (unlikely(!pskb_may_pull(skb, EDSA_HLEN))) + if (unlikely(!pskb_may_pull(skb, EDSA_HLEN))) { + kfree_skb(skb); return NULL; + } skb_pull_rcsum(skb, EDSA_HLEN - DSA_HLEN); diff --git a/net/dsa/tag_gswip.c b/net/dsa/tag_gswip.c index 5fa436121087..5c407d448c9f 100644 --- a/net/dsa/tag_gswip.c +++ b/net/dsa/tag_gswip.c @@ -80,16 +80,20 @@ static struct sk_buff *gswip_tag_rcv(struct sk_buff *skb, int port; u8 *gswip_tag; - if (unlikely(!pskb_may_pull(skb, GSWIP_RX_HEADER_LEN))) + if (unlikely(!pskb_may_pull(skb, GSWIP_RX_HEADER_LEN))) { + kfree_skb(skb); return NULL; + } gswip_tag = skb->data - ETH_HLEN; /* Get source port information */ port = (gswip_tag[7] & GSWIP_RX_SPPID_MASK) >> GSWIP_RX_SPPID_SHIFT; skb->dev = dsa_conduit_find_user(dev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } /* remove GSWIP tag */ skb_pull_rcsum(skb, GSWIP_RX_HEADER_LEN); diff --git a/net/dsa/tag_hellcreek.c b/net/dsa/tag_hellcreek.c index 544ab15685a2..dd9f328f3182 100644 --- a/net/dsa/tag_hellcreek.c +++ b/net/dsa/tag_hellcreek.c @@ -27,8 +27,10 @@ static struct sk_buff *hellcreek_xmit(struct sk_buff *skb, * checksums after the switch strips the tag. */ if (skb->ip_summed == CHECKSUM_PARTIAL && - skb_checksum_help(skb)) + skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } /* Tag encoding */ tag = skb_put(skb, HELLCREEK_TAG_LEN); @@ -47,11 +49,14 @@ static struct sk_buff *hellcreek_rcv(struct sk_buff *skb, skb->dev = dsa_conduit_find_user(dev, 0, port); if (!skb->dev) { netdev_warn_once(dev, "Failed to get source port: %d\n", port); + kfree_skb(skb); return NULL; } - if (pskb_trim_rcsum(skb, skb->len - HELLCREEK_TAG_LEN)) + if (pskb_trim_rcsum(skb, skb->len - HELLCREEK_TAG_LEN)) { + kfree_skb(skb); return NULL; + } dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_ksz.c b/net/dsa/tag_ksz.c index d2475c3bbb7d..67fa89f102e0 100644 --- a/net/dsa/tag_ksz.c +++ b/net/dsa/tag_ksz.c @@ -88,11 +88,15 @@ static struct sk_buff *ksz_common_rcv(struct sk_buff *skb, unsigned int port, unsigned int len) { skb->dev = dsa_conduit_find_user(dev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } - if (pskb_trim_rcsum(skb, skb->len - len)) + if (pskb_trim_rcsum(skb, skb->len - len)) { + kfree_skb(skb); return NULL; + } dsa_default_offload_fwd_mark(skb); @@ -123,8 +127,10 @@ static struct sk_buff *ksz8795_xmit(struct sk_buff *skb, struct net_device *dev) struct ethhdr *hdr; u8 *tag; - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } /* Tag encoding */ tag = skb_put(skb, KSZ_INGRESS_TAG_LEN); @@ -141,8 +147,10 @@ static struct sk_buff *ksz8795_rcv(struct sk_buff *skb, struct net_device *dev) { u8 *tag; - if (skb_linearize(skb)) + if (skb_linearize(skb)) { + kfree_skb(skb); return NULL; + } tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; @@ -255,22 +263,24 @@ static struct sk_buff *ksz_defer_xmit(struct dsa_port *dp, struct sk_buff *skb) xmit_work_fn = tagger_data->xmit_work_fn; xmit_worker = priv->xmit_worker; - if (!xmit_work_fn || !xmit_worker) + if (!xmit_work_fn || !xmit_worker) { + kfree_skb(skb); return NULL; + } xmit_work = kzalloc_obj(*xmit_work, GFP_ATOMIC); - if (!xmit_work) + if (!xmit_work) { + kfree_skb(skb); return NULL; + } kthread_init_work(&xmit_work->work, xmit_work_fn); - /* Increase refcount so the kfree_skb in dsa_user_xmit - * won't really free the packet. - */ xmit_work->dp = dp; xmit_work->skb = skb_get(skb); kthread_queue_work(xmit_worker, &xmit_work->work); + kfree_skb(skb); return NULL; } @@ -284,8 +294,10 @@ static struct sk_buff *ksz9477_xmit(struct sk_buff *skb, __be16 *tag; u16 val; - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } /* Tag encoding */ ksz_xmit_timestamp(dp, skb); @@ -310,8 +322,10 @@ static struct sk_buff *ksz9477_rcv(struct sk_buff *skb, struct net_device *dev) unsigned int port; u8 *tag; - if (skb_linearize(skb)) + if (skb_linearize(skb)) { + kfree_skb(skb); return NULL; + } /* Tag decoding */ tag = skb_tail_pointer(skb) - KSZ_EGRESS_TAG_LEN; @@ -352,8 +366,10 @@ static struct sk_buff *ksz9893_xmit(struct sk_buff *skb, struct ethhdr *hdr; u8 *tag; - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } /* Tag encoding */ ksz_xmit_timestamp(dp, skb); @@ -418,8 +434,10 @@ static struct sk_buff *lan937x_xmit(struct sk_buff *skb, __be16 *tag; u16 val; - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } ksz_xmit_timestamp(dp, skb); diff --git a/net/dsa/tag_lan9303.c b/net/dsa/tag_lan9303.c index 258e5d7dc5ef..d1194696499a 100644 --- a/net/dsa/tag_lan9303.c +++ b/net/dsa/tag_lan9303.c @@ -85,6 +85,7 @@ static struct sk_buff *lan9303_rcv(struct sk_buff *skb, struct net_device *dev) if (unlikely(!pskb_may_pull(skb, LAN9303_TAG_LEN))) { dev_warn_ratelimited(&dev->dev, "Dropping packet, cannot pull\n"); + kfree_skb(skb); return NULL; } @@ -102,6 +103,7 @@ static struct sk_buff *lan9303_rcv(struct sk_buff *skb, struct net_device *dev) skb->dev = dsa_conduit_find_user(dev, 0, source_port); if (!skb->dev) { dev_warn_ratelimited(&dev->dev, "Dropping packet due to invalid source port\n"); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_mtk.c b/net/dsa/tag_mtk.c index dea3eecaf093..c7dc7731675e 100644 --- a/net/dsa/tag_mtk.c +++ b/net/dsa/tag_mtk.c @@ -72,8 +72,10 @@ static struct sk_buff *mtk_tag_rcv(struct sk_buff *skb, struct net_device *dev) int port; __be16 *phdr; - if (unlikely(!pskb_may_pull(skb, MTK_HDR_LEN))) + if (unlikely(!pskb_may_pull(skb, MTK_HDR_LEN))) { + kfree_skb(skb); return NULL; + } phdr = dsa_etype_header_pos_rx(skb); hdr = ntohs(*phdr); @@ -87,8 +89,10 @@ static struct sk_buff *mtk_tag_rcv(struct sk_buff *skb, struct net_device *dev) port = (hdr & MTK_HDR_RECV_SOURCE_PORT_MASK); skb->dev = dsa_conduit_find_user(dev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_mxl-gsw1xx.c b/net/dsa/tag_mxl-gsw1xx.c index 60f7c445e656..4b1b6ef94196 100644 --- a/net/dsa/tag_mxl-gsw1xx.c +++ b/net/dsa/tag_mxl-gsw1xx.c @@ -73,6 +73,7 @@ static struct sk_buff *gsw1xx_tag_rcv(struct sk_buff *skb, if (unlikely(!pskb_may_pull(skb, GSW1XX_HEADER_LEN))) { dev_warn_ratelimited(&dev->dev, "Dropping packet, cannot pull SKB\n"); + kfree_skb(skb); return NULL; } @@ -81,6 +82,7 @@ static struct sk_buff *gsw1xx_tag_rcv(struct sk_buff *skb, if (unlikely(ntohs(gsw1xx_tag[0]) != ETH_P_MXLGSW)) { dev_warn_ratelimited(&dev->dev, "Dropping packet due to invalid special tag\n"); dev_warn_ratelimited(&dev->dev, "Tag: %8ph\n", gsw1xx_tag); + kfree_skb(skb); return NULL; } @@ -90,6 +92,7 @@ static struct sk_buff *gsw1xx_tag_rcv(struct sk_buff *skb, if (!skb->dev) { dev_warn_ratelimited(&dev->dev, "Dropping packet due to invalid source port\n"); dev_warn_ratelimited(&dev->dev, "Tag: %8ph\n", gsw1xx_tag); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_mxl862xx.c b/net/dsa/tag_mxl862xx.c index 8daefeb8d49d..87b80ddf0946 100644 --- a/net/dsa/tag_mxl862xx.c +++ b/net/dsa/tag_mxl862xx.c @@ -64,6 +64,7 @@ static struct sk_buff *mxl862_tag_rcv(struct sk_buff *skb, if (unlikely(!pskb_may_pull(skb, MXL862_HEADER_LEN))) { dev_warn_ratelimited(&dev->dev, "Cannot pull SKB, packet dropped\n"); + kfree_skb(skb); return NULL; } @@ -73,6 +74,7 @@ static struct sk_buff *mxl862_tag_rcv(struct sk_buff *skb, dev_warn_ratelimited(&dev->dev, "Invalid special tag marker, packet dropped, tag: %8ph\n", mxl862_tag); + kfree_skb(skb); return NULL; } @@ -83,6 +85,7 @@ static struct sk_buff *mxl862_tag_rcv(struct sk_buff *skb, dev_warn_ratelimited(&dev->dev, "Invalid source port, packet dropped, tag: %8ph\n", mxl862_tag); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_netc.c b/net/dsa/tag_netc.c index ccedfe3a80b6..df72a61796ad 100644 --- a/net/dsa/tag_netc.c +++ b/net/dsa/tag_netc.c @@ -131,14 +131,13 @@ static struct sk_buff *netc_rcv(struct sk_buff *skb, int type, subtype; if (unlikely(!pskb_may_pull(skb, NETC_TAG_MAX_LEN))) - return NULL; + goto err_free_skb; tag_cmn = dsa_etype_header_pos_rx(skb); if (ntohs(tag_cmn->tpid) != ETH_P_NXP_NETC) { dev_warn_ratelimited(&ndev->dev, "Unknown TPID 0x%04x\n", ntohs(tag_cmn->tpid)); - - return NULL; + goto err_free_skb; } if (tag_cmn->qos & NETC_TAG_QV) @@ -149,14 +148,13 @@ static struct sk_buff *netc_rcv(struct sk_buff *skb, if (!sw_id) { dev_warn_ratelimited(&ndev->dev, "VEPA switch ID is not supported yet\n"); - - return NULL; + goto err_free_skb; } port = FIELD_GET(NETC_TAG_PORT, tag_cmn->switch_port); skb->dev = dsa_conduit_find_user(ndev, sw_id, port); if (!skb->dev) - return NULL; + goto err_free_skb; type = FIELD_GET(NETC_TAG_TYPE, tag_cmn->type); subtype = FIELD_GET(NETC_TAG_SUBTYPE, tag_cmn->type); @@ -165,11 +163,11 @@ static struct sk_buff *netc_rcv(struct sk_buff *skb, } else if (type == NETC_TAG_TO_HOST) { /* Currently only subtype0 supported */ if (subtype != NETC_TAG_TH_SUBTYPE0) - return NULL; + goto err_free_skb; } else { dev_warn_ratelimited(&ndev->dev, "Unexpected tag type %d\n", type); - return NULL; + goto err_free_skb; } /* Remove Switch tag from the frame */ @@ -178,6 +176,10 @@ static struct sk_buff *netc_rcv(struct sk_buff *skb, dsa_strip_etype_header(skb, tag_len); return skb; + +err_free_skb: + kfree_skb(skb); + return NULL; } static void netc_flow_dissect(const struct sk_buff *skb, __be16 *proto, diff --git a/net/dsa/tag_ocelot.c b/net/dsa/tag_ocelot.c index 3405def79c2d..d208c7322cd6 100644 --- a/net/dsa/tag_ocelot.c +++ b/net/dsa/tag_ocelot.c @@ -107,14 +107,16 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb, ocelot_xfh_get_rew_val(extraction, &rew_val); skb->dev = dsa_conduit_find_user(netdev, 0, src_port); - if (!skb->dev) + if (!skb->dev) { /* The switch will reflect back some frames sent through * sockets opened on the bare DSA conduit. These will come back * with src_port equal to the index of the CPU port, for which * there is no user registered. So don't print any error * message here (ignore and drop those frames). */ + kfree_skb(skb); return NULL; + } dsa_default_offload_fwd_mark(skb); skb->priority = qos_class; diff --git a/net/dsa/tag_ocelot_8021q.c b/net/dsa/tag_ocelot_8021q.c index e89d9254e90a..f50f1cd83f16 100644 --- a/net/dsa/tag_ocelot_8021q.c +++ b/net/dsa/tag_ocelot_8021q.c @@ -33,30 +33,34 @@ static struct sk_buff *ocelot_defer_xmit(struct dsa_port *dp, xmit_work_fn = data->xmit_work_fn; xmit_worker = priv->xmit_worker; - if (!xmit_work_fn || !xmit_worker) + if (!xmit_work_fn || !xmit_worker) { + kfree_skb(skb); return NULL; + } /* PTP over IP packets need UDP checksumming. We may have inherited * NETIF_F_HW_CSUM from the DSA conduit, but these packets are not sent * through the DSA conduit, so calculate the checksum here. */ - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } xmit_work = kzalloc_obj(*xmit_work, GFP_ATOMIC); - if (!xmit_work) + if (!xmit_work) { + kfree_skb(skb); return NULL; + } /* Calls felix_port_deferred_xmit in felix.c */ kthread_init_work(&xmit_work->work, xmit_work_fn); - /* Increase refcount so the kfree_skb in dsa_user_xmit - * won't really free the packet. - */ xmit_work->dp = dp; xmit_work->skb = skb_get(skb); kthread_queue_work(xmit_worker, &xmit_work->work); + kfree_skb(skb); return NULL; } @@ -84,8 +88,10 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb, dsa_8021q_rcv(skb, &src_port, &switch_id, NULL, NULL); skb->dev = dsa_conduit_find_user(netdev, switch_id, src_port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_qca.c b/net/dsa/tag_qca.c index 9e3b429e8b36..510792fbfa92 100644 --- a/net/dsa/tag_qca.c +++ b/net/dsa/tag_qca.c @@ -46,16 +46,20 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) tagger_data = ds->tagger_data; - if (unlikely(!pskb_may_pull(skb, QCA_HDR_LEN))) + if (unlikely(!pskb_may_pull(skb, QCA_HDR_LEN))) { + kfree_skb(skb); return NULL; + } phdr = dsa_etype_header_pos_rx(skb); hdr = ntohs(*phdr); /* Make sure the version is correct */ ver = FIELD_GET(QCA_HDR_RECV_VERSION, hdr); - if (unlikely(ver != QCA_HDR_VERSION)) + if (unlikely(ver != QCA_HDR_VERSION)) { + kfree_skb(skb); return NULL; + } /* Get pk type */ pk_type = FIELD_GET(QCA_HDR_RECV_TYPE, hdr); @@ -64,6 +68,7 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) if (pk_type == QCA_HDR_RECV_TYPE_RW_REG_ACK) { if (likely(tagger_data->rw_reg_ack_handler)) tagger_data->rw_reg_ack_handler(ds, skb); + kfree_skb(skb); return NULL; } @@ -71,6 +76,7 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) if (pk_type == QCA_HDR_RECV_TYPE_MIB) { if (likely(tagger_data->mib_autocast_handler)) tagger_data->mib_autocast_handler(ds, skb); + kfree_skb(skb); return NULL; } @@ -78,8 +84,10 @@ static struct sk_buff *qca_tag_rcv(struct sk_buff *skb, struct net_device *dev) port = FIELD_GET(QCA_HDR_RECV_SOURCE_PORT, hdr); skb->dev = dsa_conduit_find_user(dev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } /* Remove QCA tag and recalculate checksum */ skb_pull_rcsum(skb, QCA_HDR_LEN); diff --git a/net/dsa/tag_rtl4_a.c b/net/dsa/tag_rtl4_a.c index 3cc63eacfa03..590ea3b921c9 100644 --- a/net/dsa/tag_rtl4_a.c +++ b/net/dsa/tag_rtl4_a.c @@ -41,7 +41,7 @@ static struct sk_buff *rtl4a_tag_xmit(struct sk_buff *skb, u16 out; /* Pad out to at least 60 bytes */ - if (unlikely(__skb_put_padto(skb, ETH_ZLEN, false))) + if (unlikely(eth_skb_pad(skb))) return NULL; netdev_dbg(dev, "add realtek tag to package to port %d\n", @@ -75,8 +75,10 @@ static struct sk_buff *rtl4a_tag_rcv(struct sk_buff *skb, u8 prot; u8 port; - if (unlikely(!pskb_may_pull(skb, RTL4_A_HDR_LEN))) + if (unlikely(!pskb_may_pull(skb, RTL4_A_HDR_LEN))) { + kfree_skb(skb); return NULL; + } tag = dsa_etype_header_pos_rx(skb); p = (__be16 *)tag; @@ -92,6 +94,7 @@ static struct sk_buff *rtl4a_tag_rcv(struct sk_buff *skb, prot = (protport >> RTL4_A_PROTOCOL_SHIFT) & 0x0f; if (prot != RTL4_A_PROTOCOL_RTL8366RB) { netdev_err(dev, "unknown realtek protocol 0x%01x\n", prot); + kfree_skb(skb); return NULL; } port = protport & 0xff; @@ -99,6 +102,7 @@ static struct sk_buff *rtl4a_tag_rcv(struct sk_buff *skb, skb->dev = dsa_conduit_find_user(dev, 0, port); if (!skb->dev) { netdev_dbg(dev, "could not find user for port %d\n", port); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_rtl8_4.c b/net/dsa/tag_rtl8_4.c index 852c6b88079a..4da3beebef75 100644 --- a/net/dsa/tag_rtl8_4.c +++ b/net/dsa/tag_rtl8_4.c @@ -143,8 +143,10 @@ static struct sk_buff *rtl8_4t_tag_xmit(struct sk_buff *skb, /* Calculate the checksum here if not done yet as trailing tags will * break either software or hardware based checksum */ - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb)) { + kfree_skb(skb); return NULL; + } rtl8_4_write_tag(skb, dev, skb_put(skb, RTL8_4_TAG_LEN)); @@ -201,11 +203,15 @@ static int rtl8_4_read_tag(struct sk_buff *skb, struct net_device *dev, static struct sk_buff *rtl8_4_tag_rcv(struct sk_buff *skb, struct net_device *dev) { - if (unlikely(!pskb_may_pull(skb, RTL8_4_TAG_LEN))) + if (unlikely(!pskb_may_pull(skb, RTL8_4_TAG_LEN))) { + kfree_skb(skb); return NULL; + } - if (unlikely(rtl8_4_read_tag(skb, dev, dsa_etype_header_pos_rx(skb)))) + if (unlikely(rtl8_4_read_tag(skb, dev, dsa_etype_header_pos_rx(skb)))) { + kfree_skb(skb); return NULL; + } /* Remove tag and recalculate checksum */ skb_pull_rcsum(skb, RTL8_4_TAG_LEN); @@ -218,14 +224,20 @@ static struct sk_buff *rtl8_4_tag_rcv(struct sk_buff *skb, static struct sk_buff *rtl8_4t_tag_rcv(struct sk_buff *skb, struct net_device *dev) { - if (skb_linearize(skb)) + if (skb_linearize(skb)) { + kfree_skb(skb); return NULL; + } - if (unlikely(rtl8_4_read_tag(skb, dev, skb_tail_pointer(skb) - RTL8_4_TAG_LEN))) + if (unlikely(rtl8_4_read_tag(skb, dev, skb_tail_pointer(skb) - RTL8_4_TAG_LEN))) { + kfree_skb(skb); return NULL; + } - if (pskb_trim_rcsum(skb, skb->len - RTL8_4_TAG_LEN)) + if (pskb_trim_rcsum(skb, skb->len - RTL8_4_TAG_LEN)) { + kfree_skb(skb); return NULL; + } return skb; } diff --git a/net/dsa/tag_rzn1_a5psw.c b/net/dsa/tag_rzn1_a5psw.c index 10994b3470f6..734910156dc3 100644 --- a/net/dsa/tag_rzn1_a5psw.c +++ b/net/dsa/tag_rzn1_a5psw.c @@ -48,7 +48,7 @@ static struct sk_buff *a5psw_tag_xmit(struct sk_buff *skb, struct net_device *de * least 60 bytes otherwise they will be discarded when they enter the * switch port logic. */ - if (__skb_put_padto(skb, ETH_ZLEN, false)) + if (eth_skb_pad(skb)) return NULL; /* provide 'A5PSW_TAG_LEN' bytes additional space */ @@ -77,6 +77,7 @@ static struct sk_buff *a5psw_tag_rcv(struct sk_buff *skb, if (unlikely(!pskb_may_pull(skb, A5PSW_TAG_LEN))) { dev_warn_ratelimited(&dev->dev, "Dropping packet, cannot pull\n"); + kfree_skb(skb); return NULL; } @@ -84,14 +85,17 @@ static struct sk_buff *a5psw_tag_rcv(struct sk_buff *skb, if (tag->ctrl_tag != htons(ETH_P_DSA_A5PSW)) { dev_warn_ratelimited(&dev->dev, "Dropping packet due to invalid TAG marker\n"); + kfree_skb(skb); return NULL; } port = FIELD_GET(A5PSW_CTRL_DATA_PORT, ntohs(tag->ctrl_data)); skb->dev = dsa_conduit_find_user(dev, 0, port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } skb_pull_rcsum(skb, A5PSW_TAG_LEN); dsa_strip_etype_header(skb, A5PSW_TAG_LEN); diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index de6d4ce8668b..bfe1f746f55b 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -149,19 +149,20 @@ static struct sk_buff *sja1105_defer_xmit(struct dsa_port *dp, xmit_work_fn = tagger_data->xmit_work_fn; xmit_worker = priv->xmit_worker; - if (!xmit_work_fn || !xmit_worker) + if (!xmit_work_fn || !xmit_worker) { + kfree_skb(skb); return NULL; + } xmit_work = kzalloc_obj(*xmit_work, GFP_ATOMIC); - if (!xmit_work) + if (!xmit_work) { + kfree_skb(skb); return NULL; + } kthread_init_work(&xmit_work->work, xmit_work_fn); - /* Increase refcount so the kfree_skb in dsa_user_xmit - * won't really free the packet. - */ xmit_work->dp = dp; - xmit_work->skb = skb_get(skb); + xmit_work->skb = skb; kthread_queue_work(xmit_worker, &xmit_work->work); @@ -401,10 +402,7 @@ static struct sk_buff kfree_skb(priv->stampable_skb); } - /* Hold a reference to avoid dsa_switch_rcv - * from freeing the skb. - */ - priv->stampable_skb = skb_get(skb); + priv->stampable_skb = skb; spin_unlock(&priv->meta_lock); /* Tell DSA we got nothing */ @@ -436,6 +434,7 @@ static struct sk_buff dev_err_ratelimited(ds->dev, "Unexpected meta frame\n"); spin_unlock(&priv->meta_lock); + kfree_skb(skb); return NULL; } @@ -443,6 +442,7 @@ static struct sk_buff dev_err_ratelimited(ds->dev, "Meta frame on wrong port\n"); spin_unlock(&priv->meta_lock); + kfree_skb(skb); return NULL; } @@ -501,18 +501,21 @@ static struct sk_buff *sja1105_rcv(struct sk_buff *skb, /* Normal data plane traffic and link-local frames are tagged with * a tag_8021q VLAN which we have to strip */ - if (sja1105_skb_has_tag_8021q(skb)) + if (sja1105_skb_has_tag_8021q(skb)) { dsa_8021q_rcv(skb, &source_port, &switch_id, &vbid, &vid); - else if (source_port == -1 && switch_id == -1) + } else if (source_port == -1 && switch_id == -1) { /* Packets with no source information have no chance of * getting accepted, drop them straight away. */ + kfree_skb(skb); return NULL; + } skb->dev = dsa_tag_8021q_find_user(netdev, source_port, switch_id, vid, vbid); if (!skb->dev) { netdev_warn(netdev, "Couldn't decode source port\n"); + kfree_skb(skb); return NULL; } @@ -539,12 +542,15 @@ static struct sk_buff *sja1110_rcv_meta(struct sk_buff *skb, u16 rx_header) if (!ds) { net_err_ratelimited("%s: cannot find switch id %d\n", conduit->name, switch_id); + kfree_skb(skb); return NULL; } tagger_data = sja1105_tagger_data(ds); - if (!tagger_data->meta_tstamp_handler) + if (!tagger_data->meta_tstamp_handler) { + kfree_skb(skb); return NULL; + } for (i = 0; i <= n_ts; i++) { u8 ts_id, source_port, dir; @@ -562,6 +568,7 @@ static struct sk_buff *sja1110_rcv_meta(struct sk_buff *skb, u16 rx_header) } /* Discard the meta frame, we've consumed the timestamps it contained */ + kfree_skb(skb); return NULL; } @@ -572,8 +579,10 @@ static struct sk_buff *sja1110_rcv_inband_control_extension(struct sk_buff *skb, { u16 rx_header; - if (unlikely(!pskb_may_pull(skb, SJA1110_HEADER_LEN))) + if (unlikely(!pskb_may_pull(skb, SJA1110_HEADER_LEN))) { + kfree_skb(skb); return NULL; + } /* skb->data points to skb_mac_header(skb) + ETH_HLEN, which is exactly * what we need because the caller has checked the EtherType (which is @@ -609,8 +618,10 @@ static struct sk_buff *sja1110_rcv_inband_control_extension(struct sk_buff *skb, * padding and trailer we need to account for the fact that * skb->data points to skb_mac_header(skb) + ETH_HLEN. */ - if (pskb_trim_rcsum(skb, start_of_padding - ETH_HLEN)) + if (pskb_trim_rcsum(skb, start_of_padding - ETH_HLEN)) { + kfree_skb(skb); return NULL; + } /* Trap-to-host frame, no timestamp trailer */ } else { *source_port = SJA1110_RX_HEADER_SRC_PORT(rx_header); @@ -653,6 +664,7 @@ static struct sk_buff *sja1110_rcv(struct sk_buff *skb, if (!skb->dev) { netdev_warn(netdev, "Couldn't decode source port\n"); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_trailer.c b/net/dsa/tag_trailer.c index 4dce24cfe6a7..49c802c10ca6 100644 --- a/net/dsa/tag_trailer.c +++ b/net/dsa/tag_trailer.c @@ -30,22 +30,30 @@ static struct sk_buff *trailer_rcv(struct sk_buff *skb, struct net_device *dev) u8 *trailer; int source_port; - if (skb_linearize(skb)) + if (skb_linearize(skb)) { + kfree_skb(skb); return NULL; + } trailer = skb_tail_pointer(skb) - 4; if (trailer[0] != 0x80 || (trailer[1] & 0xf8) != 0x00 || - (trailer[2] & 0xef) != 0x00 || trailer[3] != 0x00) + (trailer[2] & 0xef) != 0x00 || trailer[3] != 0x00) { + kfree_skb(skb); return NULL; + } source_port = trailer[1] & 7; skb->dev = dsa_conduit_find_user(dev, 0, source_port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } - if (pskb_trim_rcsum(skb, skb->len - 4)) + if (pskb_trim_rcsum(skb, skb->len - 4)) { + kfree_skb(skb); return NULL; + } return skb; } diff --git a/net/dsa/tag_vsc73xx_8021q.c b/net/dsa/tag_vsc73xx_8021q.c index af121a9aff7f..f4736a1a7a0f 100644 --- a/net/dsa/tag_vsc73xx_8021q.c +++ b/net/dsa/tag_vsc73xx_8021q.c @@ -44,6 +44,7 @@ vsc73xx_rcv(struct sk_buff *skb, struct net_device *netdev) if (!skb->dev) { dev_warn_ratelimited(&netdev->dev, "Couldn't decode source port\n"); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/tag_xrs700x.c b/net/dsa/tag_xrs700x.c index a05219f702c6..bb268020ee86 100644 --- a/net/dsa/tag_xrs700x.c +++ b/net/dsa/tag_xrs700x.c @@ -30,15 +30,21 @@ static struct sk_buff *xrs700x_rcv(struct sk_buff *skb, struct net_device *dev) source_port = ffs((int)trailer[0]) - 1; - if (source_port < 0) + if (source_port < 0) { + kfree_skb(skb); return NULL; + } skb->dev = dsa_conduit_find_user(dev, 0, source_port); - if (!skb->dev) + if (!skb->dev) { + kfree_skb(skb); return NULL; + } - if (pskb_trim_rcsum(skb, skb->len - 1)) + if (pskb_trim_rcsum(skb, skb->len - 1)) { + kfree_skb(skb); return NULL; + } /* Frame is forwarded by hardware, don't forward in software. */ dsa_default_offload_fwd_mark(skb); diff --git a/net/dsa/tag_yt921x.c b/net/dsa/tag_yt921x.c index f3ced99b1c85..294784ab6694 100644 --- a/net/dsa/tag_yt921x.c +++ b/net/dsa/tag_yt921x.c @@ -87,8 +87,10 @@ yt921x_tag_rcv(struct sk_buff *skb, struct net_device *netdev) __be16 *tag; u16 rx; - if (unlikely(!pskb_may_pull(skb, YT921X_TAG_LEN))) + if (unlikely(!pskb_may_pull(skb, YT921X_TAG_LEN))) { + kfree_skb(skb); return NULL; + } tag = dsa_etype_header_pos_rx(skb); @@ -96,6 +98,7 @@ yt921x_tag_rcv(struct sk_buff *skb, struct net_device *netdev) dev_warn_ratelimited(&netdev->dev, "Unexpected EtherType 0x%04x\n", ntohs(tag[0])); + kfree_skb(skb); return NULL; } @@ -104,6 +107,7 @@ yt921x_tag_rcv(struct sk_buff *skb, struct net_device *netdev) if (unlikely((rx & YT921X_TAG_PORT_EN) == 0)) { dev_warn_ratelimited(&netdev->dev, "Unexpected rx tag 0x%04x\n", rx); + kfree_skb(skb); return NULL; } @@ -112,6 +116,7 @@ yt921x_tag_rcv(struct sk_buff *skb, struct net_device *netdev) if (unlikely(!skb->dev)) { dev_warn_ratelimited(&netdev->dev, "Couldn't decode source port %u\n", port); + kfree_skb(skb); return NULL; } diff --git a/net/dsa/user.c b/net/dsa/user.c index 8704c1a3a5b7..072fa76972cc 100644 --- a/net/dsa/user.c +++ b/net/dsa/user.c @@ -935,13 +935,12 @@ static netdev_tx_t dsa_user_xmit(struct sk_buff *skb, struct net_device *dev) eth_skb_pad(skb); /* Transmit function may have to reallocate the original SKB, - * in which case it must have freed it. Only free it here on error. + * in which case it must have freed it. Taggers will drop the + * passed skb on error. */ nskb = p->xmit(skb, dev); - if (!nskb) { - kfree_skb(skb); + if (!nskb) return NETDEV_TX_OK; - } return dsa_enqueue_skb(nskb, dev); } diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 2b3847f00801..4e5356e26f40 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -113,6 +113,8 @@ ethtool_nl_msg_needs_rtnl(const struct net_device *dev, u8 cmd) return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM; case ETHTOOL_MSG_RSS_SET: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; + case ETHTOOL_MSG_LINKSTATE_GET: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; case ETHTOOL_MSG_TSCONFIG_GET: case ETHTOOL_MSG_TSCONFIG_SET: /* tsconfig calls ndos (ndo_hwtstamp_set/get), not ethtool ops. @@ -159,6 +161,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd) case ETHTOOL_SRXFH: case ETHTOOL_SRXFHINDIR: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; + case ETHTOOL_GLINK: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; } return false; } diff --git a/net/ieee802154/core.c b/net/ieee802154/core.c index 89b671b12600..c0b8712018a1 100644 --- a/net/ieee802154/core.c +++ b/net/ieee802154/core.c @@ -228,36 +228,43 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev, continue; wpan_dev->netdev->netns_immutable = false; err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d"); - if (err) + if (err) { + WARN_ON(err && err != -ENOMEM); break; + } wpan_dev->netdev->netns_immutable = true; } - if (err) { - /* failed -- clean up to old netns */ - net = wpan_phy_net(&rdev->wpan_phy); - - list_for_each_entry_continue_reverse(wpan_dev, - &rdev->wpan_dev_list, - list) { - if (!wpan_dev->netdev) - continue; - wpan_dev->netdev->netns_immutable = false; - err = dev_change_net_namespace(wpan_dev->netdev, net, - "wpan%d"); - WARN_ON(err); - wpan_dev->netdev->netns_immutable = true; - } + if (err) + goto errout; - return err; - } + err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev)); + WARN_ON(err && err != -ENOMEM); - wpan_phy_net_set(&rdev->wpan_phy, net); + if (err) + goto errout; - err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev)); - WARN_ON(err); + wpan_phy_net_set(&rdev->wpan_phy, net); return 0; + +errout: + /* failed -- clean up to old netns */ + net = wpan_phy_net(&rdev->wpan_phy); + + list_for_each_entry_continue_reverse(wpan_dev, + &rdev->wpan_dev_list, + list) { + if (!wpan_dev->netdev) + continue; + wpan_dev->netdev->netns_immutable = false; + err = dev_change_net_namespace(wpan_dev->netdev, net, + "wpan%d"); + WARN_ON(err && err != -ENOMEM); + wpan_dev->netdev->netns_immutable = true; + } + + return err; } void cfg802154_dev_free(struct cfg802154_registered_device *rdev) @@ -351,7 +358,7 @@ static void __net_exit cfg802154_pernet_exit(struct net *net) rtnl_lock(); list_for_each_entry(rdev, &cfg802154_rdev_list, list) { if (net_eq(wpan_phy_net(&rdev->wpan_phy), net)) - WARN_ON(cfg802154_switch_netns(rdev, &init_net)); + cfg802154_switch_netns(rdev, &init_net); } rtnl_unlock(); } diff --git a/net/ieee802154/header_ops.c b/net/ieee802154/header_ops.c index 41a556be1017..a9f0c8df5ae4 100644 --- a/net/ieee802154/header_ops.c +++ b/net/ieee802154/header_ops.c @@ -173,10 +173,13 @@ ieee802154_hdr_get_addr(const u8 *buf, int mode, bool omit_pan, { int pos = 0; - addr->mode = mode; - - if (mode == IEEE802154_ADDR_NONE) + if (mode == IEEE802154_ADDR_NONE) { + memset(addr, 0, sizeof(*addr)); + addr->mode = IEEE802154_ADDR_NONE; return 0; + } + + addr->mode = mode; if (!omit_pan) { memcpy(&addr->pan_id, buf + pos, 2); diff --git a/net/ieee802154/ieee802154.h b/net/ieee802154/ieee802154.h index c5d91f78301a..e765adc4b88f 100644 --- a/net/ieee802154/ieee802154.h +++ b/net/ieee802154/ieee802154.h @@ -16,6 +16,15 @@ void ieee802154_nl_exit(void); .flags = GENL_ADMIN_PERM, \ } +#define IEEE802154_OP_RELAXED(_cmd, _func) \ + { \ + .cmd = _cmd, \ + .doit = _func, \ + .dumpit = NULL, \ + .flags = GENL_ADMIN_PERM, \ + .validate = GENL_DONT_VALIDATE_STRICT,\ + } + #define IEEE802154_DUMP(_cmd, _func, _dump) \ { \ .cmd = _cmd, \ @@ -23,6 +32,14 @@ void ieee802154_nl_exit(void); .dumpit = _dump, \ } +#define IEEE802154_DUMP_PRIV(_cmd, _func, _dump) \ + { \ + .cmd = _cmd, \ + .doit = _func, \ + .dumpit = _dump, \ + .flags = GENL_ADMIN_PERM, \ + } + struct genl_info; struct sk_buff *ieee802154_nl_create(int flags, u8 req); diff --git a/net/ieee802154/netlink.c b/net/ieee802154/netlink.c index 7d2de4ee6992..cacad21347ec 100644 --- a/net/ieee802154/netlink.c +++ b/net/ieee802154/netlink.c @@ -98,24 +98,24 @@ static const struct genl_small_ops ieee802154_ops[] = { IEEE802154_OP(IEEE802154_SET_MACPARAMS, ieee802154_set_macparams), IEEE802154_OP(IEEE802154_LLSEC_GETPARAMS, ieee802154_llsec_getparams), IEEE802154_OP(IEEE802154_LLSEC_SETPARAMS, ieee802154_llsec_setparams), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_KEY, NULL, - ieee802154_llsec_dump_keys), - IEEE802154_OP(IEEE802154_LLSEC_ADD_KEY, ieee802154_llsec_add_key), - IEEE802154_OP(IEEE802154_LLSEC_DEL_KEY, ieee802154_llsec_del_key), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_DEV, NULL, - ieee802154_llsec_dump_devs), - IEEE802154_OP(IEEE802154_LLSEC_ADD_DEV, ieee802154_llsec_add_dev), - IEEE802154_OP(IEEE802154_LLSEC_DEL_DEV, ieee802154_llsec_del_dev), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_DEVKEY, NULL, - ieee802154_llsec_dump_devkeys), - IEEE802154_OP(IEEE802154_LLSEC_ADD_DEVKEY, ieee802154_llsec_add_devkey), - IEEE802154_OP(IEEE802154_LLSEC_DEL_DEVKEY, ieee802154_llsec_del_devkey), - IEEE802154_DUMP(IEEE802154_LLSEC_LIST_SECLEVEL, NULL, - ieee802154_llsec_dump_seclevels), - IEEE802154_OP(IEEE802154_LLSEC_ADD_SECLEVEL, - ieee802154_llsec_add_seclevel), - IEEE802154_OP(IEEE802154_LLSEC_DEL_SECLEVEL, - ieee802154_llsec_del_seclevel), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_KEY, NULL, + ieee802154_llsec_dump_keys), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_KEY, ieee802154_llsec_add_key), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_KEY, ieee802154_llsec_del_key), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_DEV, NULL, + ieee802154_llsec_dump_devs), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_DEV, ieee802154_llsec_add_dev), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_DEV, ieee802154_llsec_del_dev), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_DEVKEY, NULL, + ieee802154_llsec_dump_devkeys), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_DEVKEY, ieee802154_llsec_add_devkey), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_DEVKEY, ieee802154_llsec_del_devkey), + IEEE802154_DUMP_PRIV(IEEE802154_LLSEC_LIST_SECLEVEL, NULL, + ieee802154_llsec_dump_seclevels), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_ADD_SECLEVEL, + ieee802154_llsec_add_seclevel), + IEEE802154_OP_RELAXED(IEEE802154_LLSEC_DEL_SECLEVEL, + ieee802154_llsec_del_seclevel), }; static const struct genl_multicast_group ieee802154_mcgrps[] = { diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 42212970d735..a5e739d32d59 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -76,7 +76,7 @@ static int __net_init fib4_rules_init(struct net *net) struct fib_table *fib_new_table(struct net *net, u32 id) { - struct fib_table *tb, *alias = NULL; + struct fib_table *tb, *new_tb, *alias = NULL; unsigned int h; if (id == 0) @@ -85,14 +85,27 @@ struct fib_table *fib_new_table(struct net *net, u32 id) if (tb) return tb; + if (!check_net(net)) + return NULL; + if (id == RT_TABLE_LOCAL && !net->ipv4.fib_has_custom_rules) alias = fib_new_table(net, RT_TABLE_MAIN); - if (check_net(net)) - tb = fib_trie_table(id, alias); - if (!tb) + new_tb = fib_trie_table(id, alias); + if (!new_tb) return NULL; + spin_lock(&net->ipv4.fib_table_hash_lock); + + tb = fib_get_table(net, id); + if (tb) { + spin_unlock(&net->ipv4.fib_table_hash_lock); + fib_free_table(new_tb); + return tb; + } + + tb = new_tb; + switch (id) { case RT_TABLE_MAIN: rcu_assign_pointer(net->ipv4.fib_main, tb); @@ -106,28 +119,35 @@ struct fib_table *fib_new_table(struct net *net, u32 id) h = id & (FIB_TABLE_HASHSZ - 1); hlist_add_head_rcu(&tb->tb_hlist, &net->ipv4.fib_table_hash[h]); + + spin_unlock(&net->ipv4.fib_table_hash_lock); + return tb; } EXPORT_SYMBOL_GPL(fib_new_table); -/* caller must hold either rtnl or rcu read lock */ struct fib_table *fib_get_table(struct net *net, u32 id) { - struct fib_table *tb; + struct fib_table *tb = NULL; struct hlist_head *head; unsigned int h; if (id == 0) id = RT_TABLE_MAIN; h = id & (FIB_TABLE_HASHSZ - 1); - head = &net->ipv4.fib_table_hash[h]; - hlist_for_each_entry_rcu(tb, head, tb_hlist, - lockdep_rtnl_is_held()) { + + /* fib_table is not destroyed until ip_fib_net_exit() + * except for the merged main/local table. + * fib_unmerge() is called under RTNL, so other readers + * under RTNL (e.g. fib_flush(), fib_info_notify_update()) + * can safely traverse the list with rcu_dereference_raw(). + */ + hlist_for_each_entry_rcu(tb, head, tb_hlist, true) if (tb->tb_id == id) - return tb; - } - return NULL; + break; + + return tb; } #endif /* CONFIG_IP_MULTIPLE_TABLES */ @@ -190,10 +210,9 @@ void fib_flush(struct net *net) for (h = 0; h < FIB_TABLE_HASHSZ; h++) { struct hlist_head *head = &net->ipv4.fib_table_hash[h]; - struct hlist_node *tmp; struct fib_table *tb; - hlist_for_each_entry_safe(tb, tmp, head, tb_hlist) + hlist_for_each_entry_rcu(tb, head, tb_hlist, true) flushed += fib_table_flush(net, tb, false); } @@ -1019,10 +1038,11 @@ static int inet_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) .dump_routes = true, .dump_exceptions = true, }; - unsigned int e = 0, s_e, h, s_h; struct hlist_head *head; int dumped = 0, err = 0; struct fib_table *tb; + unsigned int h, s_h; + u32 s_id; rcu_read_lock(); if (cb->strict_check) { @@ -1054,29 +1074,28 @@ static int inet_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) } s_h = cb->args[0]; - s_e = cb->args[1]; + s_id = cb->args[1]; err = 0; - for (h = s_h; h < FIB_TABLE_HASHSZ; h++, s_e = 0) { - e = 0; + for (h = s_h; h < FIB_TABLE_HASHSZ; h++, s_id = 0) { head = &net->ipv4.fib_table_hash[h]; hlist_for_each_entry_rcu(tb, head, tb_hlist) { - if (e < s_e) - goto next; + if (s_id && tb->tb_id != s_id) + continue; + + s_id = 0; if (dumped) memset(&cb->args[2], 0, sizeof(cb->args) - 2 * sizeof(cb->args[0])); + cb->args[1] = tb->tb_id; err = fib_table_dump(tb, skb, cb, &filter); if (err < 0) goto out; dumped = 1; -next: - e++; } } out: - cb->args[1] = e; cb->args[0] = h; unlock: @@ -1565,6 +1584,7 @@ static int __net_init ip_fib_net_init(struct net *net) net->ipv4.sysctl_fib_multipath_hash_fields = FIB_MULTIPATH_HASH_FIELD_DEFAULT_MASK; #endif + spin_lock_init(&net->ipv4.fib_table_hash_lock); /* Avoid false sharing : Use at least a full cache line */ size = max_t(size_t, size, L1_CACHE_BYTES); diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c index e068a5bace73..4edb0dca7be8 100644 --- a/net/ipv4/fib_rules.c +++ b/net/ipv4/fib_rules.c @@ -301,10 +301,12 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb, fib4_nl2rule_dscp_mask(tb[FRA_DSCP_MASK], rule4, extack) < 0) goto errout; - /* split local/main if they are not already split */ - err = fib_unmerge(net); - if (err) - goto errout; + if (!net->ipv4.fib_has_custom_rules) { + /* split local/main if they are not already split */ + err = fib_unmerge(net); + if (err) + goto errout; + } if (rule->table == RT_TABLE_UNSPEC && !rule->l3mdev) { if (rule->action == FR_ACT_TO_TBL) { @@ -349,7 +351,7 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb, return err; } -static int fib4_rule_delete(struct fib_rule *rule) +static void fib4_rule_delete(struct fib_rule *rule) { struct net *net = rule->fr_net; @@ -361,8 +363,6 @@ static int fib4_rule_delete(struct fib_rule *rule) if (net->ipv4.fib_rules_require_fldissect && fib_rule_requires_fldissect(rule)) net->ipv4.fib_rules_require_fldissect--; - - return 0; } static int fib4_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh, @@ -460,6 +460,11 @@ static void fib4_rule_flush_cache(struct fib_rules_ops *ops) rt_cache_flush(ops->fro_net); } +static bool fib4_rule_need_rtnl(struct net *net) +{ + return !net->ipv4.fib_has_custom_rules; +} + static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = { .family = AF_INET, .rule_size = sizeof(struct fib4_rule), @@ -473,6 +478,7 @@ static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = { .fill = fib4_rule_fill, .nlmsg_payload = fib4_rule_nlmsg_payload, .flush_cache = fib4_rule_flush_cache, + .need_rtnl = fib4_rule_need_rtnl, .nlgroup = RTNLGRP_IPV4_RULE, .owner = THIS_MODULE, }; diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index e11dc86ceda0..d1d342d7148e 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -2137,8 +2137,7 @@ void fib_info_notify_update(struct net *net, struct nl_info *info) struct hlist_head *head = &net->ipv4.fib_table_hash[h]; struct fib_table *tb; - hlist_for_each_entry_rcu(tb, head, tb_hlist, - lockdep_rtnl_is_held()) + hlist_for_each_entry_rcu(tb, head, tb_hlist, true) __fib_info_notify_update(net, tb, info); } } diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index b6337a47c141..116ce7cec80e 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -1473,6 +1473,7 @@ int inet_fill_ifmcaddr(struct sk_buff *skb, struct net_device *dev, ci.ifa_valid = INFINITY_LIFE_TIME; if (nla_put_in_addr(skb, IFA_MULTICAST, im->multiaddr) < 0 || + nla_put_u32(skb, IFA_MC_USERS, READ_ONCE(im->users)) < 0 || nla_put(skb, IFA_CACHEINFO, sizeof(ci), &ci) < 0) { nlmsg_cancel(skb, nlh); return -EMSGSIZE; @@ -1494,6 +1495,7 @@ static void inet_ifmcaddr_notify(struct net_device *dev, skb = nlmsg_new(NLMSG_ALIGN(sizeof(struct ifaddrmsg)) + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u32)) + nla_total_size(sizeof(struct ifa_cacheinfo)), GFP_KERNEL); if (!skb) diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 208dd48012d9..3efdfb4ffa21 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1457,6 +1457,9 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; @@ -1486,6 +1489,9 @@ static int erspan_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 3b4e9b8af044..e6dd1e5b8c32 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1116,8 +1116,8 @@ static int __ip_append_data(struct sock *sk, !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; else { - alloclen = fragheaderlen + transhdrlen; - pagedlen = datalen - transhdrlen; + alloclen = fragheaderlen + transhdrlen + fraggap; + pagedlen = datalen - transhdrlen - fraggap; } alloclen += alloc_extra; @@ -1164,9 +1164,6 @@ static int __ip_append_data(struct sock *sk, } copy = datalen - transhdrlen - fraggap - pagedlen; - /* [!] NOTE: copy will be negative if pagedlen>0 - * because then the equation reduces to -fraggap. - */ if (copy > 0 && INDIRECT_CALL_1(getfrag, ip_generic_getfrag, from, data + transhdrlen, offset, diff --git a/net/ipv4/ip_vti.c b/net/ipv4/ip_vti.c index 95b6bb78fcd2..3b80929994a0 100644 --- a/net/ipv4/ip_vti.c +++ b/net/ipv4/ip_vti.c @@ -596,6 +596,9 @@ static int vti_changelink(struct net_device *dev, struct nlattr *tb[], struct ip_tunnel_parm_kern p; __u32 fwmark = t->fwmark; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + vti_netlink_parms(data, &p, &fwmark); return ip_tunnel_changelink(dev, tb, &p, fwmark); } diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 4f89a03e0b49..b643194f57d2 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -494,6 +494,9 @@ static int ipip_changelink(struct net_device *dev, struct nlattr *tb[], bool collect_md; __u32 fwmark = t->fwmark; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { int err = ip_tunnel_encap_setup(t, &ipencap); diff --git a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c index 0ede138dfd29..e540b86bd15b 100644 --- a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c +++ b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c @@ -213,7 +213,7 @@ static int __init nf_nat_snmp_basic_init(void) RCU_INIT_POINTER(nf_nat_snmp_hook, help); nf_ct_helper_init(&snmp_trap_helper, AF_INET, IPPROTO_UDP, - "snmp_trap", SNMP_TRAP_PORT, SNMP_TRAP_PORT, SNMP_TRAP_PORT, + "snmp_trap", &snmp_exp_policy, 0, help, NULL, THIS_MODULE); err = nf_conntrack_helper_register(&snmp_trap_helper, &snmp_trap_helper_ptr); diff --git a/net/ipv4/netfilter/nf_reject_ipv4.c b/net/ipv4/netfilter/nf_reject_ipv4.c index fecf6621f679..4626dc46808f 100644 --- a/net/ipv4/netfilter/nf_reject_ipv4.c +++ b/net/ipv4/netfilter/nf_reject_ipv4.c @@ -89,7 +89,7 @@ static bool nf_skb_is_icmp_unreach(const struct sk_buff *skb) if (iph->protocol != IPPROTO_ICMP) return false; - thoff = skb_network_offset(skb) + sizeof(*iph); + thoff = skb_network_offset(skb) + ip_hdrlen(skb); tp = skb_header_pointer(skb, thoff + offsetof(struct icmphdr, type), diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index e9fbab6ad914..2aebaf8297e0 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -809,23 +809,18 @@ static int raw_seticmpfilter(struct sock *sk, sockptr_t optval, int optlen) return 0; } -static int raw_geticmpfilter(struct sock *sk, char __user *optval, int __user *optlen) +static int raw_geticmpfilter(struct sock *sk, sockopt_t *opt) { - int len, ret = -EFAULT; + int len = opt->optlen; - if (get_user(len, optlen)) - goto out; - ret = -EINVAL; if (len < 0) - goto out; + return -EINVAL; if (len > sizeof(struct icmp_filter)) len = sizeof(struct icmp_filter); - ret = -EFAULT; - if (put_user(len, optlen) || - copy_to_user(optval, &raw_sk(sk)->filter, len)) - goto out; - ret = 0; -out: return ret; + opt->optlen = len; + if (copy_to_iter(&raw_sk(sk)->filter, len, &opt->iter_out) != len) + return -EFAULT; + return 0; } static int do_raw_setsockopt(struct sock *sk, int optname, @@ -848,14 +843,13 @@ static int raw_setsockopt(struct sock *sk, int level, int optname, return do_raw_setsockopt(sk, optname, optval, optlen); } -static int do_raw_getsockopt(struct sock *sk, int optname, - char __user *optval, int __user *optlen) +static int do_raw_getsockopt(struct sock *sk, int optname, sockopt_t *opt) { if (optname == ICMP_FILTER) { if (inet_sk(sk)->inet_num != IPPROTO_ICMP) return -EOPNOTSUPP; else - return raw_geticmpfilter(sk, optval, optlen); + return raw_geticmpfilter(sk, opt); } return -ENOPROTOOPT; } @@ -863,9 +857,24 @@ static int do_raw_getsockopt(struct sock *sk, int optname, static int raw_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { + sockopt_t opt; + int err; + if (level != SOL_RAW) return ip_getsockopt(sk, level, optname, optval, optlen); - return do_raw_getsockopt(sk, optname, optval, optlen); + + err = sockopt_init_user(&opt, optval, optlen); + if (err) + return err; + + err = do_raw_getsockopt(sk, optname, &opt); + if (err) + return err; + + if (put_user(opt.optlen, optlen)) + return -EFAULT; + + return 0; } static int raw_ioctl(struct sock *sk, int cmd, int *karg) diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index c0e85cc171ae..ca1180dba1de 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -1058,7 +1058,9 @@ static struct ctl_table ipv4_net_table[] = { .data = &init_net.ipv4.sysctl_tcp_reordering, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &init_net.ipv4.sysctl_tcp_max_reordering, }, { .procname = "tcp_retries1", @@ -1293,7 +1295,8 @@ static struct ctl_table ipv4_net_table[] = { .data = &init_net.ipv4.sysctl_tcp_max_reordering, .maxlen = sizeof(int), .mode = 0644, - .proc_handler = proc_dointvec + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, }, { .procname = "tcp_dsack", @@ -1676,6 +1679,9 @@ static __net_init int ipv4_sysctl_init_net(struct net *net) */ table[i].mode &= ~0222; } + if (table[i].extra2 >= (void *)&init_net.ipv4 && + table[i].extra2 < (void *)(&init_net.ipv4 + 1)) + table[i].extra2 += (void *)net - (void *)&init_net; } } diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index 2f69bcecae78..e4ec60a33496 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -371,8 +371,9 @@ static void tcp_ao_key_free_rcu(struct rcu_head *head) kfree_sensitive(key); } -static void tcp_ao_info_free(struct tcp_ao_info *ao) +static void tcp_ao_info_free_rcu(struct rcu_head *head) { + struct tcp_ao_info *ao = container_of(head, struct tcp_ao_info, rcu); struct tcp_ao_key *key; struct hlist_node *n; @@ -411,7 +412,7 @@ void tcp_ao_destroy_sock(struct sock *sk, bool twsk) if (!twsk) tcp_ao_sk_omem_free(sk, ao); - tcp_ao_info_free(ao); + call_rcu(&ao->rcu, tcp_ao_info_free_rcu); } void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp) @@ -1747,6 +1748,10 @@ static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info, * them and we can just free all resources in RCU fashion. */ if (del_async) { + if (ao_info->current_key == key) + WRITE_ONCE(ao_info->current_key, NULL); + if (ao_info->rnext_key == key) + WRITE_ONCE(ao_info->rnext_key, NULL); atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc); call_rcu(&key->rcu, tcp_ao_key_free_rcu); return 0; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index ec09f97cc9e6..209ef7522508 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1467,9 +1467,9 @@ void tcp_clear_md5_list(struct sock *sk) md5sig = rcu_dereference_protected(tp->md5sig_info, 1); hlist_for_each_entry_safe(key, n, &md5sig->head, node) { - hlist_del(&key->node); + hlist_del_rcu(&key->node); atomic_sub(sizeof(*key), &sk->sk_omem_alloc); - kfree(key); + kfree_rcu(key, rcu); } } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 26dd751ec72a..d7c1444b5e30 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -2688,7 +2688,7 @@ static int tcp_mtu_probe(struct sock *sk) struct sk_buff *skb, *nskb, *next; struct net *net = sock_net(sk); int probe_size; - int size_needed; + u64 size_needed; int copy, len; int mss_now; int interval; @@ -2712,7 +2712,7 @@ static int tcp_mtu_probe(struct sock *sk) mss_now = tcp_current_mss(sk); probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high + icsk->icsk_mtup.search_low) >> 1); - size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache; + size_needed = probe_size + (tp->reordering + 1) * (u64)tp->mss_cache; interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low; /* When misfortune happens, we are reprobing actively, * and then reprobe timer has expired. We stick with current @@ -4329,9 +4329,13 @@ int tcp_connect(struct sock *sk) if (needs_md5) { tcp_ao_destroy_sock(sk, false); } else if (needs_ao) { + struct tcp_md5sig_info *md5sig; + tcp_clear_md5_list(sk); - kfree(rcu_replace_pointer(tp->md5sig_info, NULL, - lockdep_sock_is_held(sk))); + md5sig = rcu_replace_pointer(tp->md5sig_info, NULL, + lockdep_sock_is_held(sk)); + kfree_rcu(md5sig, rcu); + static_branch_slow_dec_deferred(&tcp_md5_needed); } } #endif diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 70f6cbd4ef73..59248a59358c 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -76,6 +76,7 @@ #include #include +#include #include #include #include @@ -2995,14 +2996,13 @@ static int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t opt } int udp_lib_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen) + sockopt_t *opt) { struct udp_sock *up = udp_sk(sk); int val, len; - if (get_user(len, optlen)) - return -EFAULT; - + len = opt->optlen; + /* keep the check so direct sockopt_t callers stay covered. */ if (len < 0) return -EINVAL; @@ -3037,9 +3037,8 @@ int udp_lib_getsockopt(struct sock *sk, int level, int optname, return -ENOPROTOOPT; } - if (put_user(len, optlen)) - return -EFAULT; - if (copy_to_user(optval, &val, len)) + opt->optlen = len; + if (copy_to_iter(&val, len, &opt->iter_out) != len) return -EFAULT; return 0; } @@ -3047,9 +3046,29 @@ int udp_lib_getsockopt(struct sock *sk, int level, int optname, static int udp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { - if (level == SOL_UDP) - return udp_lib_getsockopt(sk, level, optname, optval, optlen); - return ip_getsockopt(sk, level, optname, optval, optlen); + sockopt_t opt; + int err; + + /* + * keep the old __user pointers, until ip_getsockopt() moves + * to sockopt_t + */ + if (level != SOL_UDP) + return ip_getsockopt(sk, level, optname, optval, optlen); + + err = sockopt_init_user(&opt, optval, optlen); + if (err) + return err; + + err = udp_lib_getsockopt(sk, level, optname, &opt); + if (err) + return err; + + /* optval was written by copy_to_iter() in udp_lib_getsockopt() */ + if (put_user(opt.optlen, optlen)) + return -EFAULT; + + return 0; } /** diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index 9944ed923ddf..3b32a0afa979 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -301,7 +301,7 @@ __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) { - if (!utn->need_sync) + if (!utn->need_sync || utn->work_pending) return; queue_work(udp_tunnel_nic_workqueue, &utn->work); diff --git a/net/ipv4/xfrm4_input.c b/net/ipv4/xfrm4_input.c index c2eac844bcdb..f6f2a8ef3f88 100644 --- a/net/ipv4/xfrm4_input.c +++ b/net/ipv4/xfrm4_input.c @@ -76,8 +76,6 @@ int xfrm4_transport_finish(struct sk_buff *skb, int async) NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING, dev_net(dev), NULL, skb, dev, NULL, xfrm4_rcv_encap_finish); - if (async) - dev_put(dev); return 0; } diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 1f21ccb55caa..f1fe9ede1edb 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -913,7 +913,7 @@ static int addrconf_fixup_forwarding(const struct ctl_table *table, int *p, int if (newf) rt6_purge_dflt_routers(net); - return 1; + return 0; } static void addrconf_linkdown_change(struct net *net, __s32 newf) @@ -955,11 +955,7 @@ static int addrconf_fixup_linkdown(const struct ctl_table *table, int *p, int ne NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, NETCONFA_IFINDEX_DEFAULT, net->ipv6.devconf_dflt); - rtnl_net_unlock(net); - return 0; - } - - if (p == &net->ipv6.devconf_all->ignore_routes_with_linkdown) { + } else if (p == &net->ipv6.devconf_all->ignore_routes_with_linkdown) { WRITE_ONCE(net->ipv6.devconf_dflt->ignore_routes_with_linkdown, newf); addrconf_linkdown_change(net, newf); if ((!newf) ^ (!old)) @@ -968,11 +964,21 @@ static int addrconf_fixup_linkdown(const struct ctl_table *table, int *p, int ne NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, NETCONFA_IFINDEX_ALL, net->ipv6.devconf_all); + } else { + if (!newf ^ !old) { + struct inet6_dev *idev = table->extra1; + + inet6_netconf_notify_devconf(net, + RTM_NEWNETCONF, + NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN, + idev->dev->ifindex, + &idev->cnf); + } } rtnl_net_unlock(net); - return 1; + return 0; } #endif @@ -5258,6 +5264,7 @@ int inet6_fill_ifmcaddr(struct sk_buff *skb, put_ifaddrmsg(nlh, 128, IFA_F_PERMANENT, scope, ifindex); if (nla_put_in6_addr(skb, IFA_MULTICAST, &ifmca->mca_addr) < 0 || + nla_put_u32(skb, IFA_MC_USERS, READ_ONCE(ifmca->mca_users)) < 0 || put_cacheinfo(skb, ifmca->mca_cstamp, READ_ONCE(ifmca->mca_tstamp), INFINITY_LIFE_TIME, INFINITY_LIFE_TIME) < 0) { nlmsg_cancel(skb, nlh); @@ -6370,6 +6377,8 @@ static int addrconf_sysctl_forward(const struct ctl_table *ctl, int write, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_fixup_forwarding(ctl, valp, val); @@ -6467,6 +6476,8 @@ static int addrconf_sysctl_disable(const struct ctl_table *ctl, int write, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_disable_ipv6(ctl, valp, val); @@ -6478,20 +6489,19 @@ static int addrconf_sysctl_disable(const struct ctl_table *ctl, int write, static int addrconf_sysctl_proxy_ndp(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos) { + struct net *net = ctl->extra2; int *valp = ctl->data; - int ret; int old, new; + int ret; + + if (write && !rtnl_net_trylock(net)) + return restart_syscall(); old = *valp; ret = proc_dointvec(ctl, write, buffer, lenp, ppos); new = *valp; if (write && old != new) { - struct net *net = ctl->extra2; - - if (!rtnl_net_trylock(net)) - return restart_syscall(); - if (valp == &net->ipv6.devconf_dflt->proxy_ndp) { inet6_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_PROXY_NEIGH, @@ -6510,8 +6520,9 @@ static int addrconf_sysctl_proxy_ndp(const struct ctl_table *ctl, int write, idev->dev->ifindex, &idev->cnf); } - rtnl_net_unlock(net); } + if (write) + rtnl_net_unlock(net); return ret; } @@ -6669,6 +6680,8 @@ int addrconf_sysctl_ignore_routes_with_linkdown(const struct ctl_table *ctl, lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write) ret = addrconf_fixup_linkdown(ctl, valp, val); @@ -6763,6 +6776,8 @@ static int addrconf_sysctl_disable_policy(const struct ctl_table *ctl, int write lctl = *ctl; lctl.data = &val; ret = proc_dointvec(&lctl, write, buffer, lenp, ppos); + if (ret) + return ret; if (write && (*valp != val)) ret = addrconf_disable_policy(ctl, valp, val); diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c index e1b2b4fa6e18..04dab9329d0c 100644 --- a/net/ipv6/fib6_rules.c +++ b/net/ipv6/fib6_rules.c @@ -480,15 +480,13 @@ static int fib6_rule_configure(struct fib_rule *rule, struct sk_buff *skb, return err; } -static int fib6_rule_delete(struct fib_rule *rule) +static void fib6_rule_delete(struct fib_rule *rule) { struct net *net = rule->fr_net; if (net->ipv6.fib6_rules_require_fldissect && fib_rule_requires_fldissect(rule)) net->ipv6.fib6_rules_require_fldissect--; - - return 0; } static int fib6_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh, @@ -637,21 +635,14 @@ static int __net_init fib6_rules_net_init(struct net *net) goto out; } -static void __net_exit fib6_rules_net_exit_batch(struct list_head *net_list) +static void __net_exit fib6_rules_net_exit(struct net *net) { - struct net *net; - - rtnl_lock(); - list_for_each_entry(net, net_list, exit_list) { - fib_rules_unregister(net->ipv6.fib6_rules_ops); - cond_resched(); - } - rtnl_unlock(); + fib_rules_unregister(net->ipv6.fib6_rules_ops); } static struct pernet_operations fib6_rules_net_ops = { .init = fib6_rules_net_init, - .exit_batch = fib6_rules_net_exit_batch, + .exit = fib6_rules_net_exit, }; int __init fib6_rules_init(void) diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index b9f6d892a566..cfb2c41634a0 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -35,7 +35,7 @@ struct ioam6_lwt_freq { }; struct ioam6_lwt { - struct dst_entry null_dst; + struct rt6_info null_rt; struct dst_cache cache; struct ioam6_lwt_freq freq; atomic_t pkt_cnt; @@ -176,7 +176,7 @@ static int ioam6_build_state(struct net *net, struct nlattr *nla, * it is stored in the cache. Then, +1/-1 each time we read the cache * and release it. Long story short, we're fine. */ - dst_init(&ilwt->null_dst, NULL, NULL, DST_OBSOLETE_NONE, DST_NOCOUNT); + dst_init(&ilwt->null_rt.dst, NULL, NULL, DST_OBSOLETE_NONE, DST_NOCOUNT); atomic_set(&ilwt->pkt_cnt, 0); ilwt->freq.k = freq_k; @@ -360,7 +360,7 @@ static int ioam6_output(struct net *net, struct sock *sk, struct sk_buff *skb) /* This is how we notify that the destination does not change after * transformation and that we need to use orig_dst instead of the cache */ - if (dst == &ilwt->null_dst) { + if (dst == &ilwt->null_rt.dst) { dst_release(dst); dst = orig_dst; @@ -429,7 +429,7 @@ static int ioam6_output(struct net *net, struct sock *sk, struct sk_buff *skb) local_bh_disable(); if (orig_dst->lwtstate == dst->lwtstate) dst_cache_set_ip6(&ilwt->cache, - &ilwt->null_dst, &fl6.saddr); + &ilwt->null_rt.dst, &fl6.saddr); else dst_cache_set_ip6(&ilwt->cache, dst, &fl6.saddr); local_bh_enable(); diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index fc95738ded76..a130cdfaebfb 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -636,12 +636,12 @@ static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) }; const struct nlmsghdr *nlh = cb->nlh; struct net *net = sock_net(skb->sk); - unsigned int e = 0, s_e; struct hlist_head *head; struct fib6_walker *w; struct fib6_table *tb; unsigned int h, s_h; int err = 0; + u32 s_id; rcu_read_lock(); if (cb->strict_check) { @@ -701,23 +701,22 @@ static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb) } s_h = cb->args[0]; - s_e = cb->args[1]; + s_id = cb->args[1]; - for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_e = 0) { - e = 0; + for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_id = 0) { head = &net->ipv6.fib_table_hash[h]; hlist_for_each_entry_rcu(tb, head, tb6_hlist) { - if (e < s_e) - goto next; + if (s_id && tb->tb6_id != s_id) + continue; + + s_id = 0; + cb->args[1] = tb->tb6_id; err = fib6_dump_table(tb, skb, cb); if (err != 0) goto out; -next: - e++; } } out: - cb->args[1] = e; cb->args[0] = h; unlock: diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 795be59946f7..7c09a269b352 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -2047,6 +2047,9 @@ static int ip6gre_changelink(struct net_device *dev, struct nlattr *tb[], struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id); struct __ip6_tnl_parm p; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + t = ip6gre_changelink_common(dev, tb, data, &p, extack); if (IS_ERR(t)) return PTR_ERR(t); @@ -2266,6 +2269,9 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], struct __ip6_tnl_parm p; struct ip6gre_net *ign; + if (!rtnl_dev_link_net_capable(dev, t->net)) + return -EPERM; + ign = net_generic(t->net, ip6gre_net_id); t = ip6gre_changelink_common(dev, tb, data, &p, extack); if (IS_ERR(t)) diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 9f1e0e4f7464..368e4fa3b43c 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1667,8 +1667,8 @@ static int __ip6_append_data(struct sock *sk, !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; else { - alloclen = fragheaderlen + transhdrlen; - pagedlen = datalen - transhdrlen; + alloclen = fragheaderlen + transhdrlen + fraggap; + pagedlen = datalen - transhdrlen - fraggap; } alloclen += alloc_extra; @@ -1683,10 +1683,7 @@ static int __ip6_append_data(struct sock *sk, fraglen = datalen + fragheaderlen; copy = datalen - transhdrlen - fraggap - pagedlen; - /* [!] NOTE: copy may be negative if pagedlen>0 - * because then the equation may reduces to -fraggap. - */ - if (copy < 0 && !(flags & MSG_SPLICE_PAGES)) { + if (copy < 0) { err = -EINVAL; goto error; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 73fc5a0b8203..bf8e40af60b0 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1851,6 +1851,13 @@ static int ip6_tnl_fill_forward_path(struct net_device_path_ctx *ctx, struct dst_entry *dst; int err; + if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT)) { + /* encaplimit option is currently not supported is + * sw-acceleration path. + */ + return -EOPNOTSUPP; + } + dst = ip6_route_output(dev_net(ctx->dev), NULL, &fl6); if (!dst->error) { path->type = DEV_PATH_TUN; @@ -2103,6 +2110,9 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[], struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id); struct ip_tunnel_encap ipencap; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + if (dev == ip6n->fb_tnl_dev) { if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { /* iproute2 always sets TUNNEL_ENCAP_FLAG_CSUM6, so diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index d871cab6938d..ab94b3a4ba9c 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -1046,6 +1046,9 @@ static int vti6_changelink(struct net_device *dev, struct nlattr *tb[], struct __ip6_tnl_parm p; struct vti6_net *ip6n; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + ip6n = net_generic(net, vti6_net_id); if (dev == ip6n->fb_tnl_dev) return -EINVAL; diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 04b811b3be97..774f4c72a6fa 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -908,6 +908,7 @@ static void inet6_ifmcaddr_notify(struct net_device *dev, skb = nlmsg_new(NLMSG_ALIGN(sizeof(struct ifaddrmsg)) + nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(u32)) + nla_total_size(sizeof(struct ifa_cacheinfo)), GFP_KERNEL); if (!skb) diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index e7ad13c5bd26..f867ec8d3d90 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -967,10 +967,8 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb) return reason; } -static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr) +static int accept_untracked_na(struct inet6_dev *idev, struct in6_addr *saddr) { - struct inet6_dev *idev = __in6_dev_get(dev); - switch (READ_ONCE(idev->cnf.accept_untracked_na)) { case 0: /* Don't accept untracked na (absent in neighbor cache) */ return 0; @@ -980,7 +978,7 @@ static int accept_untracked_na(struct net_device *dev, struct in6_addr *saddr) * same subnet as an address configured on the interface that * received the na */ - return !!ipv6_chk_prefix(saddr, dev); + return !!ipv6_chk_prefix(saddr, idev->dev); default: return 0; } @@ -1078,7 +1076,7 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb) */ new_state = msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE; if (!neigh && lladdr && idev && READ_ONCE(idev->cnf.forwarding)) { - if (accept_untracked_na(dev, saddr)) { + if (accept_untracked_na(idev, saddr)) { neigh = neigh_create(&nd_tbl, &msg->target, dev); new_state = NUD_STALE; } diff --git a/net/ipv6/netfilter/ip6t_ah.c b/net/ipv6/netfilter/ip6t_ah.c index 70da2f2ce064..1258783ed876 100644 --- a/net/ipv6/netfilter/ip6t_ah.c +++ b/net/ipv6/netfilter/ip6t_ah.c @@ -56,6 +56,11 @@ static bool ah_mt6(const struct sk_buff *skb, struct xt_action_param *par) } hdrlen = ipv6_authlen(ah); + if (skb->len - ptr < hdrlen) { + /* Packet smaller than its length field */ + par->hotdrop = true; + return false; + } pr_debug("IPv6 AH LEN %u %u ", hdrlen, ah->hdrlen); pr_debug("RES %04X ", ah->reserved); diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c index 450dd53846a2..6d1a5d2026a6 100644 --- a/net/ipv6/netfilter/ip6t_hbh.c +++ b/net/ipv6/netfilter/ip6t_hbh.c @@ -75,6 +75,7 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(oh); if (skb->len - ptr < hdrlen) { /* Packet smaller than it's length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 5561bd9cea81..278b52752f36 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -56,7 +56,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(rh); if (skb->len - ptr < hdrlen) { - /* Pcket smaller than its length field */ + /* Packet smaller than its length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 6361ad2fcf77..a1301334da48 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -5055,6 +5055,9 @@ static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg) struct inet6_dev *idev = __in6_dev_get(arg->dev); u32 mtu = f6i->fib6_pmtu; + if (!idev) + return 0; + if (mtu >= arg->mtu || (mtu < arg->mtu && mtu == idev->cnf.mtu6)) fib6_metric_set(f6i, RTAX_MTU, arg->mtu); diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 1c3ad25700c4..62a7eb779202 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -29,6 +29,9 @@ bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced) int max_last_entry; int trailing; + if (len < sizeof(*srh)) + return false; + if (srh->type != IPV6_SRCRT_TYPE_4) return false; diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 64f0d1b622d3..a38b24fb8384 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1613,6 +1613,9 @@ static int ipip6_changelink(struct net_device *dev, struct nlattr *tb[], __u32 fwmark = t->fwmark; int err; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + if (dev == sitn->fb_tunnel_dev) return -EINVAL; diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 15e032194ecc..392e18b97045 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -1826,9 +1826,22 @@ static int udpv6_setsockopt(struct sock *sk, int level, int optname, static int udpv6_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { - if (level == SOL_UDP) - return udp_lib_getsockopt(sk, level, optname, optval, optlen); - return ipv6_getsockopt(sk, level, optname, optval, optlen); + sockopt_t opt; + int err; + + if (level != SOL_UDP) + return ipv6_getsockopt(sk, level, optname, optval, optlen); + + err = sockopt_init_user(&opt, optval, optlen); + if (err) + return err; + + err = udp_lib_getsockopt(sk, level, optname, &opt); + if (err) + return err; + if (put_user(opt.optlen, optlen)) + return -EFAULT; + return 0; } diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 699a001ac166..89d0443b5307 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -71,8 +71,6 @@ int xfrm6_transport_finish(struct sk_buff *skb, int async) NF_HOOK(NFPROTO_IPV6, NF_INET_PRE_ROUTING, dev_net(dev), NULL, skb, dev, NULL, xfrm6_transport_finish2); - if (async) - dev_put(dev); return 0; } diff --git a/net/key/af_key.c b/net/key/af_key.c index a714c997aab2..1d8965d7f4f3 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -1218,6 +1218,7 @@ static struct xfrm_state * pfkey_msg2xfrm_state(struct net *net, goto out; } strcpy(x->calg->alg_name, a->name); + x->calg->alg_key_len = 0; x->props.calgo = sa->sadb_sa_encrypt; } else { int keysize = 0; diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 8ed1be1ecccc..b0447c33dbf0 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -312,6 +312,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr) /* assign new connection to its SAP */ llc_sap_add_socket(sap, sk); sock_reset_flag(sk, SOCK_ZAPPED); + llc_sap_put(sap); rc = 0; out: dev_put(dev); diff --git a/net/llc/sysctl_net_llc.c b/net/llc/sysctl_net_llc.c index c8d88e2508fc..15f1e5d88f20 100644 --- a/net/llc/sysctl_net_llc.c +++ b/net/llc/sysctl_net_llc.c @@ -47,7 +47,7 @@ static struct ctl_table_header *llc_station_header; int __init llc_sysctl_init(void) { - struct ctl_table empty[1] = {}; + static struct ctl_table empty[1] = {}; llc2_timeout_header = register_net_sysctl(&init_net, "net/llc/llc2/timeout", llc2_timeout_table); llc_station_header = register_net_sysctl_sz(&init_net, "net/llc/station", empty, 0); diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 3b58af59f7e4..11943f14936f 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1146,9 +1146,6 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, fd->max_interval = params->max_interval; old = sdata_dereference(link->u.ap.fils_discovery, sdata); - if (old) - kfree_rcu(old, rcu_head); - if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); if (!new) @@ -1160,6 +1157,9 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.fils_discovery, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_FILS_DISCOVERY; return 0; } @@ -1179,8 +1179,6 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, link_conf->unsol_bcast_probe_resp_interval = params->interval; old = sdata_dereference(link->u.ap.unsol_bcast_probe_resp, sdata); - if (old) - kfree_rcu(old, rcu_head); if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); @@ -1193,6 +1191,9 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.unsol_bcast_probe_resp, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_UNSOL_BCAST_PROBE_RESP; return 0; } @@ -4949,8 +4950,8 @@ static int ieee80211_set_rekey_data(struct wiphy *wiphy, return 0; } -static int ieee80211_probe_client(struct wiphy *wiphy, struct net_device *dev, - const u8 *peer, u64 *cookie) +static int ieee80211_probe_peer(struct wiphy *wiphy, struct net_device *dev, + const u8 *peer, u64 *cookie) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); struct ieee80211_local *local = sdata->local; @@ -6060,7 +6061,7 @@ const struct cfg80211_ops mac80211_config_ops = { .tdls_mgmt = ieee80211_tdls_mgmt, .tdls_channel_switch = ieee80211_tdls_channel_switch, .tdls_cancel_channel_switch = ieee80211_tdls_cancel_channel_switch, - .probe_client = ieee80211_probe_client, + .probe_peer = ieee80211_probe_peer, .set_noack_map = ieee80211_set_noack_map, #ifdef CONFIG_PM .set_wakeup = ieee80211_set_wakeup, diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 9e92337bb6f9..fa773f3b0541 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -5641,13 +5641,15 @@ static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, size_t len) { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; - u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + u16 reason_code; lockdep_assert_wiphy(sdata->local->hw.wiphy); - if (len < 24 + 2) + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) return; + reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) { ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code); return; @@ -7138,7 +7140,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data; - u16 capab_info, status_code, aid; + u16 capab_info, status_code, aid = 0; struct ieee80211_elems_parse_params parse_params = { .bss = NULL, .link_id = -1, @@ -7217,8 +7219,10 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, if (elems->aid_resp) aid = le16_to_cpu(elems->aid_resp->aid); - else + else if (!assoc_data->s1g) aid = le16_to_cpu(mgmt->u.assoc_resp.aid); + else if (status_code == WLAN_STATUS_SUCCESS) + goto abandon_assoc; /* * The 5 MSB of the AID field are reserved for a non-S1G STA. For diff --git a/net/mac80211/nan.c b/net/mac80211/nan.c index 1800bb96dd29..19e08661be43 100644 --- a/net/mac80211/nan.c +++ b/net/mac80211/nan.c @@ -253,9 +253,12 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, { struct ieee80211_nan_channel *sched_idx_to_chan[IEEE80211_NAN_MAX_CHANNELS] = {}; struct ieee80211_nan_sched_cfg *sched_cfg = &sdata->vif.cfg.nan_sched; - struct ieee80211_nan_sched_cfg backup_sched; + struct ieee80211_nan_sched_cfg *backup_sched __free(kfree) = kmalloc_obj(*backup_sched); int ret; + if (!backup_sched) + return -ENOMEM; + if (sched->n_channels > IEEE80211_NAN_MAX_CHANNELS) return -EOPNOTSUPP; @@ -275,13 +278,13 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS); - memcpy(backup_sched.schedule, sched_cfg->schedule, - sizeof(backup_sched.schedule)); - memcpy(backup_sched.channels, sched_cfg->channels, - sizeof(backup_sched.channels)); - memcpy(backup_sched.avail_blob, sched_cfg->avail_blob, - sizeof(backup_sched.avail_blob)); - backup_sched.avail_blob_len = sched_cfg->avail_blob_len; + memcpy(backup_sched->schedule, sched_cfg->schedule, + sizeof(backup_sched->schedule)); + memcpy(backup_sched->channels, sched_cfg->channels, + sizeof(backup_sched->channels)); + memcpy(backup_sched->avail_blob, sched_cfg->avail_blob, + sizeof(backup_sched->avail_blob)); + backup_sched->avail_blob_len = sched_cfg->avail_blob_len; memcpy(sched_cfg->avail_blob, sched->nan_avail_blob, sched->nan_avail_blob_len); @@ -380,17 +383,17 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, if (!chan_def->chan) continue; - if (!cfg80211_chandef_identical(&backup_sched.channels[i].chanreq.oper, + if (!cfg80211_chandef_identical(&backup_sched->channels[i].chanreq.oper, chan_def)) ieee80211_nan_remove_channel(sdata, &sched_cfg->channels[i]); } /* Re-add all backed up channels */ - for (int i = 0; i < ARRAY_SIZE(backup_sched.channels); i++) { + for (int i = 0; i < ARRAY_SIZE(backup_sched->channels); i++) { struct ieee80211_nan_channel *chan = &sched_cfg->channels[i]; - *chan = backup_sched.channels[i]; + *chan = backup_sched->channels[i]; /* * For deferred update, no channels were removed and the channel @@ -421,11 +424,11 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, } } - memcpy(sched_cfg->schedule, backup_sched.schedule, - sizeof(backup_sched.schedule)); - memcpy(sched_cfg->avail_blob, backup_sched.avail_blob, - sizeof(backup_sched.avail_blob)); - sched_cfg->avail_blob_len = backup_sched.avail_blob_len; + memcpy(sched_cfg->schedule, backup_sched->schedule, + sizeof(backup_sched->schedule)); + memcpy(sched_cfg->avail_blob, backup_sched->avail_blob, + sizeof(backup_sched->avail_blob)); + sched_cfg->avail_blob_len = backup_sched->avail_blob_len; sched_cfg->deferred = false; bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS); diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index fb9a3574afe9..d9ea19be075d 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -1526,6 +1526,9 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx) if (status->flag & RX_FLAG_DUP_VALIDATED) return RX_CONTINUE; + if (ieee80211_is_ext(hdr->frame_control)) + return RX_CONTINUE; + /* * Drop duplicate 802.11 retransmissions * (IEEE 802.11-2012: 9.3.2.10 "Duplicate detection and recovery") @@ -4510,12 +4513,16 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx) struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); u8 *bssid = ieee80211_get_bssid(hdr, skb->len, sdata->vif.type); - bool multicast = is_multicast_ether_addr(hdr->addr1) || - ieee80211_is_s1g_beacon(hdr->frame_control); + bool multicast; static const u8 nan_network_id[ETH_ALEN] __aligned(2) = { 0x51, 0x6F, 0x9A, 0x01, 0x00, 0x00 }; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) + return sdata->vif.type == NL80211_IFTYPE_STATION && bssid; + + multicast = is_multicast_ether_addr(hdr->addr1); + switch (sdata->vif.type) { case NL80211_IFTYPE_STATION: if (!bssid && !sdata->u.mgd.use_4addr) @@ -5212,6 +5219,11 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx, hdr = (struct ieee80211_hdr *)rx->skb->data; } + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + ieee80211_invoke_rx_handlers(rx); + return true; + } + /* Store a copy of the pre-translated link addresses for SW crypto */ if (unlikely(is_unicast_ether_addr(hdr->addr1) && !ieee80211_is_data(hdr->frame_control))) @@ -5301,6 +5313,13 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx, struct sta_info *sta; int link_id = -1; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + if (!ieee80211_rx_data_set_sta(rx, NULL, -1)) + return false; + + return ieee80211_prepare_and_rx_handle(rx, skb, consume); + } + /* * Look up link station first, in case there's a * chance that they might have a link address that @@ -5376,6 +5395,17 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, err = -ENOBUFS; else err = skb_linearize(skb); + } else if (ieee80211_is_s1g_beacon(fc)) { + size_t s1g_hdr_len = offsetof(struct ieee80211_ext, + u.s1g_beacon.variable) + + ieee80211_s1g_optional_len(fc); + + if (skb->len < s1g_hdr_len) + err = -ENOBUFS; + else + err = skb_linearize(skb); + } else if (ieee80211_is_ext(fc)) { + err = -EINVAL; } else { err = !pskb_may_pull(skb, ieee80211_hdrlen(fc)); } diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 02b587ff8504..22eba0e6e54c 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -355,6 +355,15 @@ static void sta_info_free_link(struct link_sta_info *link_sta) free_percpu(link_sta->pcpu_rx_stats); } +static void sta_link_free_rcu(struct rcu_head *head) +{ + struct sta_link_alloc *alloc = + container_of(head, struct sta_link_alloc, rcu_head); + + sta_info_free_link(&alloc->info); + kfree(alloc); +} + static void sta_accumulate_removed_link_stats(struct sta_info *sta, int link_id) { struct link_sta_info *link_sta = wiphy_dereference(sta->local->hw.wiphy, @@ -439,10 +448,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id, RCU_INIT_POINTER(sta->link[link_id], NULL); RCU_INIT_POINTER(sta->sta.link[link_id], NULL); - if (alloc) { - sta_info_free_link(&alloc->info); - kfree_rcu(alloc, rcu_head); - } + if (alloc) + call_rcu(&alloc->rcu_head, sta_link_free_rcu); ieee80211_sta_recalc_aggregates(&sta->sta); } diff --git a/net/mac80211/status.c b/net/mac80211/status.c index dd1dbba06838..c3d29aed93fe 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -655,7 +655,7 @@ static void ieee80211_report_ack_skb(struct ieee80211_local *local, GFP_ATOMIC); else if (ieee80211_is_any_nullfunc(hdr->frame_control)) cfg80211_probe_status(sdata->dev, hdr->addr1, - cookie, acked, + cookie, -1, acked, info->status.ack_signal, is_valid_ack_signal, GFP_ATOMIC); diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index c13b209fad47..91b14112e24f 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2607,6 +2607,18 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local, return info_id; } +static void ieee80211_remove_ack_skb(struct ieee80211_local *local, u16 info_id) +{ + struct sk_buff *ack_skb; + unsigned long flags; + + spin_lock_irqsave(&local->ack_status_lock, flags); + ack_skb = idr_remove(&local->ack_status_frames, info_id); + spin_unlock_irqrestore(&local->ack_status_lock, flags); + + kfree_skb(ack_skb); +} + /** * ieee80211_build_hdr - build 802.11 header in the given frame * @sdata: virtual interface to build the header for @@ -2982,7 +2994,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) { ieee80211_free_txskb(&local->hw, skb); skb = NULL; - return ERR_PTR(-ENOMEM); + ret = -ENOMEM; + goto free; } } @@ -3050,6 +3063,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, return skb; free: + if (info_id) + ieee80211_remove_ack_skb(local, info_id); kfree_skb(skb); return ERR_PTR(ret); } diff --git a/net/mac80211/util.c b/net/mac80211/util.c index f6d4ae4127c8..59f73dabe6e0 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -73,6 +73,9 @@ u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, if (ieee80211_is_s1g_beacon(fc)) { struct ieee80211_ext *ext = (void *) hdr; + if (len < offsetofend(struct ieee80211_ext, u.s1g_beacon.sa)) + return NULL; + return ext->u.s1g_beacon.sa; } diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index 000be60d9580..b823720630e7 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -703,7 +703,7 @@ void ieee802154_remove_interfaces(struct ieee802154_local *local) mutex_lock(&local->iflist_mtx); list_for_each_entry_safe(sdata, tmp, &local->interfaces, list) { - list_del(&sdata->list); + list_del_rcu(&sdata->list); unregister_netdevice(sdata->dev); } diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c index e8512578398e..5e7cc11fab3a 100644 --- a/net/mac802154/llsec.c +++ b/net/mac802154/llsec.c @@ -710,6 +710,7 @@ int mac802154_llsec_encrypt(struct mac802154_llsec *sec, struct sk_buff *skb) { struct ieee802154_hdr hdr; int rc, authlen, hlen; + struct sk_buff *trailer; struct mac802154_llsec_key *key; u32 frame_ctr; @@ -769,6 +770,12 @@ int mac802154_llsec_encrypt(struct mac802154_llsec *sec, struct sk_buff *skb) skb->mac_len = ieee802154_hdr_push(skb, &hdr); skb_reset_mac_header(skb); + rc = skb_cow_data(skb, 0, &trailer); + if (rc < 0) { + llsec_key_put(key); + return rc; + } + rc = llsec_do_encrypt(skb, sec, &hdr, key); llsec_key_put(key); @@ -908,6 +915,13 @@ llsec_do_decrypt(struct sk_buff *skb, const struct mac802154_llsec *sec, const struct ieee802154_hdr *hdr, struct mac802154_llsec_key *key, __le64 dev_addr) { + struct sk_buff *trailer; + int err; + + err = skb_cow_data(skb, 0, &trailer); + if (err < 0) + return err; + if (hdr->sec.level == IEEE802154_SCF_SECLEVEL_ENC) return llsec_do_decrypt_unauth(skb, sec, hdr, key, dev_addr); else diff --git a/net/mac802154/scan.c b/net/mac802154/scan.c index 0a31ac8d8415..300d4584533e 100644 --- a/net/mac802154/scan.c +++ b/net/mac802154/scan.c @@ -594,6 +594,7 @@ int mac802154_perform_association(struct ieee802154_sub_if_data *sdata, "Negative ASSOC RESP received from %8phC: %s\n", &ceaddr, local->assoc_status == IEEE802154_PAN_AT_CAPACITY ? "PAN at capacity" : "access denied"); + goto clear_assoc; } ret = 0; diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig index 665f8008cc4b..4c04cd8d40a2 100644 --- a/net/netfilter/Kconfig +++ b/net/netfilter/Kconfig @@ -256,8 +256,7 @@ config NF_CONNTRACK_H323 To compile it as a module, choose M here. If unsure, say N. config NF_CONNTRACK_IRC - tristate "IRC protocol support" - default m if NETFILTER_ADVANCED=n + tristate "IRC DCC protocol support (obsolete)" help There is a commonly-used extension to IRC called Direct Client-to-Client Protocol (DCC). This enables users to send @@ -267,6 +266,8 @@ config NF_CONNTRACK_IRC using NAT, this extension will enable you to send files and initiate chats. Note that you do NOT need this extension to get files or have others initiate chats, or everything else in IRC. + DCC tracking behind NAT requires plaintext (unencrypted) IRC, so + this helper is of limited use these days. To compile it as a module, choose M here. If unsure, say N. @@ -308,17 +309,17 @@ config NF_CONNTRACK_SNMP To compile it as a module, choose M here. If unsure, say N. config NF_CONNTRACK_PPTP - tristate "PPtP protocol support" + tristate "PPtP protocol support (deprecated)" depends on NETFILTER_ADVANCED select NF_CT_PROTO_GRE help This module adds support for PPTP (Point to Point Tunnelling Protocol, RFC2637) connection tracking and NAT. - If you are running PPTP sessions over a stateful firewall or NAT + If you are still running PPTP sessions over a stateful firewall or NAT box, you may want to enable this feature. - Please note that not all PPTP modes of operation are supported yet. + Please note that not all PPTP modes of operation are supported. Specifically these limitations exist: - Blindly assumes that control connections are always established in PNS->PAC direction. This is a violation of RFC2637. diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index 798c7993635e..bb9b5bed10e1 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -165,6 +165,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, ip_set_init_skbinfo(ext_skbinfo(x, set), ext); /* Activate element */ + smp_mb__before_atomic(); set_bit(e->id, map->members); set->elements++; @@ -219,7 +220,7 @@ mtype_list(const struct ip_set *set, cond_resched_rcu(); id = cb->args[IPSET_CB_ARG0]; x = get_ext(set, map, id); - if (!test_bit(id, map->members) || + if (!test_bit_acquire(id, map->members) || (SET_WITH_TIMEOUT(set) && #ifdef IP_SET_BITMAP_STORED_TIMEOUT mtype_is_filled(x) && @@ -278,6 +279,7 @@ mtype_gc(struct timer_list *t) x = get_ext(set, map, id); if (ip_set_timeout_expired(ext_timeout(x, set))) { clear_bit(id, map->members); + smp_mb__after_atomic(); ip_set_ext_destroy(set, x); set->elements--; } diff --git a/net/netfilter/ipset/ip_set_bitmap_ip.c b/net/netfilter/ipset/ip_set_bitmap_ip.c index 5988b9bb9029..ac7febce074f 100644 --- a/net/netfilter/ipset/ip_set_bitmap_ip.c +++ b/net/netfilter/ipset/ip_set_bitmap_ip.c @@ -67,7 +67,7 @@ static int bitmap_ip_do_test(const struct bitmap_ip_adt_elem *e, struct bitmap_ip *map, size_t dsize) { - return !!test_bit(e->id, map->members); + return !!test_bit_acquire(e->id, map->members); } static int diff --git a/net/netfilter/ipset/ip_set_bitmap_ipmac.c b/net/netfilter/ipset/ip_set_bitmap_ipmac.c index 752f59ef8744..5921fd9d2dca 100644 --- a/net/netfilter/ipset/ip_set_bitmap_ipmac.c +++ b/net/netfilter/ipset/ip_set_bitmap_ipmac.c @@ -86,7 +86,7 @@ bitmap_ipmac_do_test(const struct bitmap_ipmac_adt_elem *e, { const struct bitmap_ipmac_elem *elem; - if (!test_bit(e->id, map->members)) + if (!test_bit_acquire(e->id, map->members)) return 0; elem = get_const_elem(map->extensions, e->id, dsize); if (e->add_mac && elem->filled == MAC_FILLED) diff --git a/net/netfilter/ipset/ip_set_bitmap_port.c b/net/netfilter/ipset/ip_set_bitmap_port.c index 7138e080def4..ca875c982424 100644 --- a/net/netfilter/ipset/ip_set_bitmap_port.c +++ b/net/netfilter/ipset/ip_set_bitmap_port.c @@ -58,7 +58,7 @@ static int bitmap_port_do_test(const struct bitmap_port_adt_elem *e, const struct bitmap_port *map, size_t dsize) { - return !!test_bit(e->id, map->members); + return !!test_bit_acquire(e->id, map->members); } static int diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index 3706b4a85a0f..6cfad152d7d1 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -351,8 +351,8 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, if (unlikely(c)) { set->ext_size -= sizeof(*c) + strlen(c->str) + 1; - kfree_rcu(c, rcu); rcu_assign_pointer(comment->c, NULL); + kfree_rcu(c, rcu); } if (!len) return; @@ -393,8 +393,8 @@ ip_set_comment_free(struct ip_set *set, void *ptr) if (unlikely(!c)) return; set->ext_size -= sizeof(*c) + strlen(c->str) + 1; - kfree_rcu(c, rcu); rcu_assign_pointer(comment->c, NULL); + kfree_rcu(c, rcu); } typedef void (*destroyer)(struct ip_set *, void *); @@ -1480,7 +1480,11 @@ ip_set_dump_done(struct netlink_callback *cb) struct ip_set_net *inst = (struct ip_set_net *)cb->args[IPSET_CB_NET]; ip_set_id_t index = (ip_set_id_t)cb->args[IPSET_CB_INDEX]; - struct ip_set *set = ip_set_ref_netlink(inst, index); + struct ip_set *set; + + rcu_read_lock(); + set = ip_set_ref_netlink(inst, index); + rcu_read_unlock(); if (set->variant->uref) set->variant->uref(set, cb, false); @@ -1686,7 +1690,9 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb) release_refcount: /* If there was an error or set is done, release set */ if (ret || !cb->args[IPSET_CB_ARG0]) { + rcu_read_lock(); set = ip_set_ref_netlink(inst, index); + rcu_read_unlock(); if (set->variant->uref) set->variant->uref(set, cb, false); pr_debug("release set %s\n", set->name); diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 04e4627ddfc1..dedf59b661dd 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -606,7 +606,7 @@ mtype_cancel_gc(struct ip_set *set) struct htype *h = set->data; if (SET_WITH_TIMEOUT(set)) - cancel_delayed_work_sync(&h->gc.dwork); + disable_delayed_work_sync(&h->gc.dwork); } static int @@ -689,7 +689,7 @@ mtype_resize(struct ip_set *set, bool retried) continue; pos = smp_load_acquire(&n->pos); for (j = 0; j < pos; j++) { - if (!test_bit(j, n->used)) + if (!test_bit_acquire(j, n->used)) continue; data = ahash_data(n, j, dsize); if (SET_ELEM_EXPIRED(set, data)) @@ -826,7 +826,7 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size) continue; pos = smp_load_acquire(&n->pos); for (j = 0; j < pos; j++) { - if (!test_bit(j, n->used)) + if (!test_bit_acquire(j, n->used)) continue; data = ahash_data(n, j, set->dsize); if (!SET_ELEM_EXPIRED(set, data)) @@ -1201,7 +1201,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, continue; pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; data = ahash_data(n, i, set->dsize); if (!mtype_data_equal(data, d, &multi)) @@ -1259,7 +1259,7 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext, } pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; data = ahash_data(n, i, set->dsize); if (!mtype_data_equal(data, d, &multi)) @@ -1396,7 +1396,7 @@ mtype_list(const struct ip_set *set, continue; pos = smp_load_acquire(&n->pos); for (i = 0; i < pos; i++) { - if (!test_bit(i, n->used)) + if (!test_bit_acquire(i, n->used)) continue; e = ahash_data(n, i, set->dsize); if (SET_ELEM_EXPIRED(set, e)) diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index cb36641f8d1c..6ed2622363f0 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1420,8 +1420,8 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, cp->app = NULL; cp->app_data = NULL; /* reset struct ip_vs_seq */ - cp->in_seq.delta = 0; - cp->out_seq.delta = 0; + memset(&cp->in_seq, 0, sizeof(cp->in_seq)); + memset(&cp->out_seq, 0, sizeof(cp->out_seq)); if (unlikely(flags & IP_VS_CONN_F_NO_CPORT)) { int af_id = ip_vs_af_index(cp->af); diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index d40b404c1bf6..906f2c361676 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1767,6 +1767,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, bool tunnel, new_cp = false; union nf_inet_addr *raddr; char *outer_proto = "IPIP"; + int ulen = 0; *related = 1; @@ -1831,7 +1832,6 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Error for our tunnel must arrive at LOCAL_IN */ (skb_rtable(skb)->rt_flags & RTCF_LOCAL)) { __u8 iproto; - int ulen; /* Non-first fragment has no UDP/GRE header */ if (unlikely(cih->frag_off & htons(IP_OFFSET))) @@ -1936,8 +1936,8 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (dest_dst) mtu = dst_mtu(dest_dst->dst_cache); } - if (mtu > 68 + sizeof(struct iphdr)) - mtu -= sizeof(struct iphdr); + if (mtu > 68 + sizeof(struct iphdr) + ulen) + mtu -= sizeof(struct iphdr) + ulen; info = htonl(mtu); } /* Strip outer IP, ICMP and IPIP/UDP/GRE, go to IP header of diff --git a/net/netfilter/ipvs/ip_vs_nfct.c b/net/netfilter/ipvs/ip_vs_nfct.c index 81974f69e5bb..347185fd0c8c 100644 --- a/net/netfilter/ipvs/ip_vs_nfct.c +++ b/net/netfilter/ipvs/ip_vs_nfct.c @@ -208,7 +208,7 @@ static void ip_vs_nfct_expect_callback(struct nf_conn *ct, * Use port 0 to expect connection from any port. */ void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, - struct ip_vs_conn *cp, u_int8_t proto, + struct ip_vs_conn *cp, u8 proto, const __be16 port, int from_rs) { struct nf_conntrack_expect *exp; diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index dd67004a5cc0..91582069f6d2 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -183,17 +183,16 @@ static int __nf_conncount_add(struct net *net, return -ENOENT; if (ct && nf_ct_is_confirmed(ct)) { - /* local connections are confirmed in postrouting so confirmation - * might have happened before hitting connlimit + /* Connection is confirmed but might still be in the setup phase. + * Only skip the tracking if it is fully assured. This guarantees + * that setup packets or retransmissions are properly counted and + * deduplicated. */ - if (skb->skb_iif != LOOPBACK_IFINDEX) { + if (test_bit(IPS_ASSURED_BIT, &ct->status)) { err = -EEXIST; goto out_put; } - /* this is likely a local connection, skip optimization to avoid - * adding duplicates from a 'packet train' - */ goto check_connections; } diff --git a/net/netfilter/nf_conntrack_amanda.c b/net/netfilter/nf_conntrack_amanda.c index ddafbdfc96dc..06d6ec12c86d 100644 --- a/net/netfilter/nf_conntrack_amanda.c +++ b/net/netfilter/nf_conntrack_amanda.c @@ -89,7 +89,7 @@ static int amanda_help(struct sk_buff *skb, struct nf_conntrack_tuple *tuple; unsigned int dataoff, start, stop, off, i; char pbuf[sizeof("65535")], *tmp; - u_int16_t len; + u16 len; __be16 port; int ret = NF_ACCEPT; nf_nat_amanda_hook_fn *nf_nat_amanda; @@ -199,10 +199,10 @@ static int __init nf_conntrack_amanda_init(void) } nf_ct_helper_init(&amanda_helper[0], AF_INET, IPPROTO_UDP, - HELPER_NAME, 10080, 10080, 10080, + HELPER_NAME, &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE); nf_ct_helper_init(&amanda_helper[1], AF_INET6, IPPROTO_UDP, - HELPER_NAME, 10080, 10080, 10080, + HELPER_NAME, &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE); ret = nf_conntrack_helpers_register(amanda_helper, diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index 40c261cd0af3..f98d1d4b42c3 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -65,6 +65,15 @@ enum { NF_BPF_CT_OPTS_SZ = 16, }; +static void *bpf_ct_opts_result(struct bpf_ct_opts *opts, u32 opts__sz, void *ret) +{ + if (!IS_ERR(ret)) + return ret; + if (opts__sz >= offsetofend(struct bpf_ct_opts, error)) + opts->error = PTR_ERR(ret); + return NULL; +} + static int bpf_nf_ct_tuple_parse(struct bpf_sock_tuple *bpf_tuple, u32 tuple_len, u8 protonum, u8 dir, struct nf_conntrack_tuple *tuple) @@ -297,12 +306,7 @@ bpf_xdp_ct_alloc(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, nfct = __bpf_nf_ct_alloc_entry(dev_net(ctx->rxq->dev), bpf_tuple, tuple__sz, opts, opts__sz, 10); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - - return (struct nf_conn___init *)nfct; + return (struct nf_conn___init *)bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_xdp_ct_lookup - Lookup CT entry for the given tuple, and acquire a @@ -331,11 +335,7 @@ bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple, caller_net = dev_net(ctx->rxq->dev); nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - return nfct; + return bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_skb_ct_alloc - Allocate a new CT entry @@ -363,12 +363,7 @@ bpf_skb_ct_alloc(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); nfct = __bpf_nf_ct_alloc_entry(net, bpf_tuple, tuple__sz, opts, opts__sz, 10); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - - return (struct nf_conn___init *)nfct; + return (struct nf_conn___init *)bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_skb_ct_lookup - Lookup CT entry for the given tuple, and acquire a @@ -397,11 +392,7 @@ bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple, caller_net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk); nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz); - if (IS_ERR(nfct)) { - opts->error = PTR_ERR(nfct); - return NULL; - } - return nfct; + return bpf_ct_opts_result(opts, opts__sz, nfct); } /* bpf_ct_insert_entry - Add the provided entry into a CT map diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index 400119b6320e..6ff954f1bfb8 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -62,11 +62,10 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, if (exp == NULL) goto out; + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; exp->tuple = ct->tuplehash[IP_CT_DIR_REPLY].tuple; helper = rcu_dereference(help->helper); - if (helper) - exp->tuple.src.u.udp.port = helper->tuple.src.u.udp.port; exp->mask.src.u3.ip = mask; exp->mask.src.u.udp.port = htons(0xFFFF); diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 4fb3a2d18631..784bd1d7a9bf 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -1471,6 +1471,31 @@ static bool gc_worker_can_early_drop(const struct nf_conn *ct) return false; } +static void nf_ct_help_gc(struct nf_conn *ct) +{ + struct nf_conn_help *help; + + if (!refcount_inc_not_zero(&ct->ct_general.use)) + return; + + /* load ->status after refcount increase */ + smp_acquire__after_ctrl_dep(); + + if (!nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct)) { + nf_ct_put(ct); + return; + } + + /* re-check helper due to SLAB_TYPESAFE_BY_RCU */ + if (test_bit(IPS_HELPER_BIT, &ct->status)) { + help = nfct_help(ct); + if (help) + nf_ct_expectation_gc(help); + } + + nf_ct_put(ct); +} + static void gc_worker(struct work_struct *work) { unsigned int i, hashsz, nf_conntrack_max95 = 0; @@ -1543,7 +1568,13 @@ static void gc_worker(struct work_struct *work) expires = (expires - (long)next_run) / ++count; next_run += expires; - if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp)) + if (gc_worker_skip_ct(tmp)) + continue; + + if (test_bit(IPS_HELPER_BIT, &tmp->status)) + nf_ct_help_gc(tmp); + + if (nf_conntrack_max95 == 0) continue; net = nf_ct_net(tmp); diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 5c9b17835c28..7ae68d60586a 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -43,6 +43,24 @@ unsigned int nf_ct_expect_max __read_mostly; static struct kmem_cache *nf_ct_expect_cachep __read_mostly; static siphash_aligned_key_t nf_ct_expect_hashrnd; +void nf_ct_expectation_gc(struct nf_conn_help *master_help) +{ + struct nf_conntrack_expect *exp; + struct hlist_node *next; + + if (hlist_empty(&master_help->expectations)) + return; + + spin_lock_bh(&nf_conntrack_expect_lock); + hlist_for_each_entry_safe(exp, next, &master_help->expectations, lnode) { + if (!nf_ct_exp_is_expired(exp)) + continue; + + nf_ct_unlink_expect(exp); + } + spin_unlock_bh(&nf_conntrack_expect_lock); +} + /* nf_conntrack_expect helper functions */ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, u32 portid, int report) @@ -52,7 +70,6 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, struct nf_conntrack_net *cnet; lockdep_nfct_expect_lock_held(); - WARN_ON_ONCE(timer_pending(&exp->timeout)); hlist_del_rcu(&exp->hnode); @@ -70,16 +87,6 @@ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, } EXPORT_SYMBOL_GPL(nf_ct_unlink_expect_report); -static void nf_ct_expectation_timed_out(struct timer_list *t) -{ - struct nf_conntrack_expect *exp = timer_container_of(exp, t, timeout); - - spin_lock_bh(&nf_conntrack_expect_lock); - nf_ct_unlink_expect(exp); - spin_unlock_bh(&nf_conntrack_expect_lock); - nf_ct_expect_put(exp); -} - static unsigned int nf_ct_expect_dst_hash(const struct net *n, const struct nf_conntrack_tuple *tuple) { struct { @@ -117,19 +124,6 @@ nf_ct_exp_equal(const struct nf_conntrack_tuple *tuple, nf_ct_exp_zone_equal_any(i, zone); } -bool nf_ct_remove_expect(struct nf_conntrack_expect *exp) -{ - lockdep_nfct_expect_lock_held(); - - if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect(exp); - nf_ct_expect_put(exp); - return true; - } - return false; -} -EXPORT_SYMBOL_GPL(nf_ct_remove_expect); - struct nf_conntrack_expect * __nf_ct_expect_find(struct net *net, const struct nf_conntrack_zone *zone, @@ -144,6 +138,8 @@ __nf_ct_expect_find(struct net *net, h = nf_ct_expect_dst_hash(net, tuple); hlist_for_each_entry_rcu(i, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) + continue; if (nf_ct_exp_equal(tuple, i, zone, net)) return i; } @@ -178,6 +174,7 @@ nf_ct_find_expectation(struct net *net, { struct nf_conntrack_net *cnet = nf_ct_pernet(net); struct nf_conntrack_expect *i, *exp = NULL; + struct hlist_node *next; unsigned int h; lockdep_nfct_expect_lock_held(); @@ -186,7 +183,11 @@ nf_ct_find_expectation(struct net *net, return NULL; h = nf_ct_expect_dst_hash(net, tuple); - hlist_for_each_entry(i, &nf_ct_expect_hash[h], hnode) { + hlist_for_each_entry_safe(i, next, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) { + nf_ct_unlink_expect(i); + continue; + } if (!(i->flags & NF_CT_EXPECT_INACTIVE) && nf_ct_exp_equal(tuple, i, zone, net)) { exp = i; @@ -196,13 +197,16 @@ nf_ct_find_expectation(struct net *net, if (!exp) return NULL; + if (!refcount_inc_not_zero(&exp->use)) + return NULL; + /* If master is not in hash table yet (ie. packet hasn't left this machine yet), how can other end know about expected? Hence these are not the droids you are looking for (if master ct never got confirmed, we'd hold a reference to it and weird things would happen to future packets). */ if (!nf_ct_is_confirmed(exp->master)) - return NULL; + goto err_release_exp; /* Avoid race with other CPUs, that for exp->master ct, is * about to invoke ->destroy(), or nf_ct_delete() via timeout @@ -214,18 +218,17 @@ nf_ct_find_expectation(struct net *net, */ if (unlikely(nf_ct_is_dying(exp->master) || !refcount_inc_not_zero(&exp->master->ct_general.use))) - return NULL; + goto err_release_exp; - if (exp->flags & NF_CT_EXPECT_PERMANENT || !unlink) { - refcount_inc(&exp->use); + if (exp->flags & NF_CT_EXPECT_PERMANENT || !unlink) return exp; - } else if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect(exp); - return exp; - } - /* Undo exp->master refcnt increase, if timer_delete() failed */ - nf_ct_put(exp->master); + nf_ct_unlink_expect(exp); + + return exp; + +err_release_exp: + nf_ct_expect_put(exp); return NULL; } @@ -241,9 +244,8 @@ void nf_ct_remove_expectations(struct nf_conn *ct) return; spin_lock_bh(&nf_conntrack_expect_lock); - hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) { - nf_ct_remove_expect(exp); - } + hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) + nf_ct_unlink_expect(exp); spin_unlock_bh(&nf_conntrack_expect_lock); } EXPORT_SYMBOL_GPL(nf_ct_remove_expectations); @@ -292,7 +294,7 @@ static bool master_matches(const struct nf_conntrack_expect *a, void nf_ct_unexpect_related(struct nf_conntrack_expect *exp) { spin_lock_bh(&nf_conntrack_expect_lock); - nf_ct_remove_expect(exp); + WRITE_ONCE(exp->flags, exp->flags | NF_CT_EXPECT_DEAD); spin_unlock_bh(&nf_conntrack_expect_lock); } EXPORT_SYMBOL_GPL(nf_ct_unexpect_related); @@ -304,10 +306,11 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) { struct nf_conntrack_expect *new; - new = kmem_cache_alloc(nf_ct_expect_cachep, GFP_ATOMIC); + new = kmem_cache_zalloc(nf_ct_expect_cachep, GFP_ATOMIC); if (!new) return NULL; + new->timeout = nfct_time_stamp; new->master = me; refcount_set(&new->use, 1); return new; @@ -352,6 +355,8 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, exp->tuple.src.l3num = family; exp->tuple.dst.protonum = proto; + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; + if (saddr) { memcpy(&exp->tuple.src.u3, saddr, len); if (sizeof(exp->tuple.src.u3) > len) @@ -386,6 +391,7 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, #if IS_ENABLED(CONFIG_NF_NAT) memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); + exp->dir = 0; #endif } EXPORT_SYMBOL_GPL(nf_ct_expect_init); @@ -413,17 +419,12 @@ static void nf_ct_expect_insert(struct nf_conntrack_expect *exp, struct net *net = nf_ct_exp_net(exp); unsigned int h = nf_ct_expect_dst_hash(net, &exp->tuple); - /* two references : one for hash insert, one for the timer */ - refcount_add(2, &exp->use); + refcount_inc(&exp->use); - timer_setup(&exp->timeout, nf_ct_expectation_timed_out, 0); helper = rcu_dereference_protected(master_help->helper, lockdep_is_held(&nf_conntrack_expect_lock)); - if (helper) { - exp->timeout.expires = jiffies + - helper->expect_policy[exp->class].timeout * HZ; - } - add_timer(&exp->timeout); + if (helper) + exp->timeout += helper->expect_policy[exp->class].timeout * HZ; hlist_add_head_rcu(&exp->lnode, &master_help->expectations); master_help->expecting[exp->class]++; @@ -435,19 +436,26 @@ static void nf_ct_expect_insert(struct nf_conntrack_expect *exp, NF_CT_STAT_INC(net, expect_create); } -/* Race with expectations being used means we could have none to find; OK. */ static void evict_oldest_expect(struct nf_conn_help *master_help, - struct nf_conntrack_expect *new) + struct nf_conntrack_expect *new, + const struct nf_conntrack_expect_policy *p) { struct nf_conntrack_expect *exp, *last = NULL; + struct hlist_node *next; - hlist_for_each_entry(exp, &master_help->expectations, lnode) { + hlist_for_each_entry_safe(exp, next, &master_help->expectations, lnode) { + if (nf_ct_exp_is_expired(exp)) { + nf_ct_unlink_expect(exp); + continue; + } if (exp->class == new->class) last = exp; } - if (last) - nf_ct_remove_expect(last); + /* Still worth to evict oldest expectation after garbage collection? */ + if (last && + master_help->expecting[last->class] >= p->max_expected) + nf_ct_unlink_expect(last); } static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, @@ -467,14 +475,18 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, h = nf_ct_expect_dst_hash(net, &expect->tuple); hlist_for_each_entry_safe(i, next, &nf_ct_expect_hash[h], hnode) { + if (nf_ct_exp_is_expired(i)) { + nf_ct_unlink_expect(i); + continue; + } if (master_matches(i, expect, flags) && expect_matches(i, expect)) { if (i->class != expect->class || i->master != expect->master) return -EALREADY; - if (nf_ct_remove_expect(i)) - break; + nf_ct_unlink_expect(i); + break; } else if (expect_clash(i, expect)) { ret = -EBUSY; goto out; @@ -485,15 +497,15 @@ static inline int __nf_ct_expect_check(struct nf_conntrack_expect *expect, lockdep_is_held(&nf_conntrack_expect_lock)); if (helper) { p = &helper->expect_policy[expect->class]; - if (p->max_expected && - master_help->expecting[expect->class] >= p->max_expected) { - evict_oldest_expect(master_help, expect); - if (master_help->expecting[expect->class] - >= p->max_expected) { - ret = -EMFILE; - goto out; - } - } + if (master_help->expecting[expect->class] >= p->max_expected) + evict_oldest_expect(master_help, expect, p); + } else { + const struct nf_conntrack_expect_policy default_exp_policy = { + .max_expected = NF_CT_EXPECT_MAX_CNT, + }; + + if (master_help->expecting[expect->class] >= default_exp_policy.max_expected) + evict_oldest_expect(master_help, expect, &default_exp_policy); } cnet = nf_ct_pernet(net); @@ -547,10 +559,8 @@ void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, vo hlist_for_each_entry_safe(exp, next, &nf_ct_expect_hash[i], hnode) { - if (iter(exp, data) && timer_delete(&exp->timeout)) { + if (iter(exp, data)) nf_ct_unlink_expect(exp); - nf_ct_expect_put(exp); - } } } @@ -577,10 +587,8 @@ void nf_ct_expect_iterate_net(struct net *net, if (!net_eq(nf_ct_exp_net(exp), net)) continue; - if (iter(exp, data) && timer_delete(&exp->timeout)) { + if (iter(exp, data)) nf_ct_unlink_expect_report(exp, portid, report); - nf_ct_expect_put(exp); - } } } @@ -657,17 +665,17 @@ static int exp_seq_show(struct seq_file *s, void *v) struct net *net = seq_file_net(s); struct hlist_node *n = v; char *delim = ""; + __s32 timeout; expect = hlist_entry(n, struct nf_conntrack_expect, hnode); if (!net_eq(nf_ct_exp_net(expect), net)) return 0; + if (nf_ct_exp_is_expired(expect)) + return 0; - if (expect->timeout.function) - seq_printf(s, "%ld ", timer_pending(&expect->timeout) - ? (long)(expect->timeout.expires - jiffies)/HZ : 0); - else - seq_puts(s, "- "); + timeout = (__s32)(READ_ONCE(expect->timeout) - nfct_time_stamp) / HZ; + seq_printf(s, "%d ", timeout > 0 ? timeout : 0); seq_printf(s, "l3proto = %u proto=%u ", expect->tuple.src.l3num, expect->tuple.dst.protonum); diff --git a/net/netfilter/nf_conntrack_ftp.c b/net/netfilter/nf_conntrack_ftp.c index 0847f845613d..f3944598c172 100644 --- a/net/netfilter/nf_conntrack_ftp.c +++ b/net/netfilter/nf_conntrack_ftp.c @@ -35,11 +35,6 @@ MODULE_ALIAS("ip_conntrack_ftp"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); static DEFINE_SPINLOCK(nf_ftp_lock); -#define MAX_PORTS 8 -static u_int16_t ports[MAX_PORTS]; -static unsigned int ports_c; -module_param_array(ports, ushort, &ports_c, 0400); - static bool loose; module_param(loose, bool, 0600); @@ -560,8 +555,8 @@ static int nf_ct_ftp_from_nlattr(struct nlattr *attr, struct nf_conn *ct) return 0; } -static struct nf_conntrack_helper ftp[MAX_PORTS * 2] __read_mostly; -static struct nf_conntrack_helper *ftp_ptr[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper ftp __read_mostly; +static struct nf_conntrack_helper *ftp_ptr __read_mostly; static const struct nf_conntrack_expect_policy ftp_exp_policy = { .max_expected = 1, @@ -570,32 +565,23 @@ static const struct nf_conntrack_expect_policy ftp_exp_policy = { static void __exit nf_conntrack_ftp_fini(void) { - nf_conntrack_helpers_unregister(ftp_ptr, ports_c * 2); + nf_conntrack_helper_unregister(ftp_ptr); } static int __init nf_conntrack_ftp_init(void) { - int i, ret = 0; + int ret = 0; NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_ftp_master)); - if (ports_c == 0) - ports[ports_c++] = FTP_PORT; - /* FIXME should be configurable whether IPv4 and IPv6 FTP connections are tracked or not - YK */ - for (i = 0; i < ports_c; i++) { - nf_ct_helper_init(&ftp[2 * i], AF_INET, IPPROTO_TCP, - HELPER_NAME, FTP_PORT, ports[i], ports[i], - &ftp_exp_policy, 0, help, - nf_ct_ftp_from_nlattr, THIS_MODULE); - nf_ct_helper_init(&ftp[2 * i + 1], AF_INET6, IPPROTO_TCP, - HELPER_NAME, FTP_PORT, ports[i], ports[i], - &ftp_exp_policy, 0, help, - nf_ct_ftp_from_nlattr, THIS_MODULE); - } + nf_ct_helper_init(&ftp, NFPROTO_UNSPEC, IPPROTO_TCP, + HELPER_NAME, + &ftp_exp_policy, 0, help, + nf_ct_ftp_from_nlattr, THIS_MODULE); - ret = nf_conntrack_helpers_register(ftp, ports_c * 2, ftp_ptr); + ret = nf_conntrack_helper_register(&ftp, &ftp_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 7f189dceb3c4..4cb1665bba02 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -671,7 +671,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, static int callforward_do_filter(struct net *net, const union nf_inet_addr *src, const union nf_inet_addr *dst, - u_int8_t family) + u8 family) { int ret = 0; @@ -1388,8 +1388,8 @@ static int process_rcf(struct sk_buff *skb, struct nf_conn *ct, "timeout to %u seconds for", info->timeout); nf_ct_dump_tuple(&exp->tuple); - mod_timer_pending(&exp->timeout, - jiffies + info->timeout * HZ); + WRITE_ONCE(exp->timeout, + nfct_time_stamp + (info->timeout * HZ)); } spin_unlock_bh(&nf_conntrack_expect_lock); } @@ -1713,19 +1713,19 @@ static int __init h323_helper_init(void) int ret; nf_ct_helper_init(&nf_conntrack_helper_ras[0], AF_INET, IPPROTO_UDP, - "RAS", RAS_PORT, RAS_PORT, RAS_PORT, + "RAS", &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE); nf_ct_helper_init(&nf_conntrack_helper_ras[1], AF_INET6, IPPROTO_UDP, - "RAS", RAS_PORT, RAS_PORT, RAS_PORT, + "RAS", &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE); nf_ct_helper_init(&nf_conntrack_helper_h245, AF_UNSPEC, IPPROTO_UDP, - "H.245", 0, 0, 0, + "H.245", &h245_exp_policy, 0, h245_help, NULL, THIS_MODULE); nf_ct_helper_init(&nf_conntrack_helper_q931[0], AF_INET, IPPROTO_TCP, - "Q.931", Q931_PORT, Q931_PORT, Q931_PORT, + "Q.931", &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE); nf_ct_helper_init(&nf_conntrack_helper_q931[1], AF_INET6, IPPROTO_TCP, - "Q.931", Q931_PORT, Q931_PORT, Q931_PORT, + "Q.931", &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE); ret = nf_conntrack_helper_register(&nf_conntrack_helper_h245, diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index 2f35bdd0d7d7..506c58034761 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -40,12 +40,16 @@ static unsigned int nf_ct_helper_count __read_mostly; static DEFINE_MUTEX(nf_ct_nat_helpers_mutex); static struct list_head nf_ct_nat_helpers __read_mostly; -/* Stupid hash, but collision free for the default registrations of the - * helpers currently in the kernel. */ -static unsigned int helper_hash(const struct nf_conntrack_tuple *tuple) +static unsigned int helper_hash(const char *name, u8 protonum) { - return (((tuple->src.l3num << 8) | tuple->dst.protonum) ^ - (__force __u16)tuple->src.u.all) % nf_ct_helper_hsize; + static u32 seed; + u32 initval; + + get_random_once(&seed, sizeof(seed)); + + initval = seed ^ protonum; + + return jhash(name, strlen(name), initval) % nf_ct_helper_hsize; } struct nf_conntrack_helper * @@ -54,18 +58,18 @@ __nf_conntrack_helper_find(const char *name, u16 l3num, u8 protonum) struct nf_conntrack_helper *h; unsigned int i; - for (i = 0; i < nf_ct_helper_hsize; i++) { - hlist_for_each_entry_rcu(h, &nf_ct_helper_hash[i], hnode) { - if (strcmp(h->name, name)) - continue; + if (!nf_ct_helper_hash) + return NULL; - if (h->tuple.src.l3num != NFPROTO_UNSPEC && - h->tuple.src.l3num != l3num) - continue; + i = helper_hash(name, protonum); - if (h->tuple.dst.protonum == protonum) - return h; - } + hlist_for_each_entry_rcu(h, &nf_ct_helper_hash[i], hnode) { + if (strcmp(h->name, name)) + continue; + if (h->nfproto != NFPROTO_UNSPEC && h->nfproto != l3num) + continue; + if (h->l4proto == protonum) + return h; } return NULL; } @@ -181,10 +185,10 @@ nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp) struct nf_conn_help *help; help = nf_ct_ext_add(ct, NF_CT_EXT_HELPER, gfp); - if (help) + if (help) { + __set_bit(IPS_HELPER_BIT, &ct->status); INIT_HLIST_HEAD(&help->expectations); - else - pr_debug("failed to add helper extension area"); + } return help; } EXPORT_SYMBOL_GPL(nf_ct_helper_ext_add); @@ -203,10 +207,8 @@ int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, return 0; help = nfct_help(tmpl); - if (help != NULL) { + if (help) helper = rcu_dereference(help->helper); - set_bit(IPS_HELPER_BIT, &ct->status); - } help = nfct_help(ct); @@ -365,9 +367,8 @@ EXPORT_SYMBOL_GPL(nf_ct_helper_log); int __nf_conntrack_helper_register(struct nf_conntrack_helper *me) { - struct nf_conntrack_tuple_mask mask = { .src.u.all = htons(0xFFFF) }; - unsigned int h = helper_hash(&me->tuple); struct nf_conntrack_helper *cur; + unsigned int h; int ret = 0, i; BUG_ON(me->expect_class_max >= NF_CT_MAX_EXPECT_CLASSES); @@ -376,32 +377,26 @@ int __nf_conntrack_helper_register(struct nf_conntrack_helper *me) if (!nf_ct_helper_hash) return -ENOENT; - if (me->expect_policy->max_expected > NF_CT_EXPECT_MAX_CNT) - return -EINVAL; + for (i = 0; i <= me->expect_class_max; i++) { + if (!me->expect_policy[i].max_expected) + me->expect_policy[i].max_expected = NF_CT_EXPECT_MAX_CNT; - mutex_lock(&nf_ct_helper_mutex); - for (i = 0; i < nf_ct_helper_hsize; i++) { - hlist_for_each_entry(cur, &nf_ct_helper_hash[i], hnode) { - if (!strcmp(cur->name, me->name) && - (cur->tuple.src.l3num == NFPROTO_UNSPEC || - cur->tuple.src.l3num == me->tuple.src.l3num) && - cur->tuple.dst.protonum == me->tuple.dst.protonum) { - ret = -EBUSY; - goto out; - } - } + if (me->expect_policy[i].max_expected > NF_CT_EXPECT_MAX_CNT) + return -EINVAL; } - /* avoid unpredictable behaviour for auto_assign_helper */ - if (!(me->flags & NF_CT_HELPER_F_USERSPACE)) { - hlist_for_each_entry(cur, &nf_ct_helper_hash[h], hnode) { - if (nf_ct_tuple_src_mask_cmp(&cur->tuple, &me->tuple, - &mask)) { - ret = -EBUSY; - goto out; - } + h = helper_hash(me->name, me->l4proto); + mutex_lock(&nf_ct_helper_mutex); + hlist_for_each_entry(cur, &nf_ct_helper_hash[h], hnode) { + if (!strcmp(cur->name, me->name) && + (cur->nfproto == NFPROTO_UNSPEC || + cur->nfproto == me->nfproto) && + cur->l4proto == me->l4proto) { + ret = -EBUSY; + goto out; } } + refcount_set(&me->ct_refcnt, 1); hlist_add_head_rcu(&me->hnode, &nf_ct_helper_hash[h]); nf_ct_helper_count++; @@ -476,8 +471,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me) EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister); void nf_ct_helper_init(struct nf_conntrack_helper *helper, - u16 l3num, u16 protonum, const char *name, - u16 default_port, u16 spec_port, u32 id, + u8 l3num, u16 protonum, const char *name, const struct nf_conntrack_expect_policy *exp_pol, u32 expect_class_max, int (*help)(struct sk_buff *skb, unsigned int protoff, @@ -489,9 +483,8 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, { memset(helper, 0, sizeof(*helper)); - helper->tuple.src.l3num = l3num; - helper->tuple.dst.protonum = protonum; - helper->tuple.src.u.all = htons(spec_port); + helper->nfproto = l3num; + helper->l4proto = protonum; rcu_assign_pointer(helper->help, help); helper->from_nlattr = from_nlattr; @@ -499,10 +492,7 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, snprintf(helper->nat_mod_name, sizeof(helper->nat_mod_name), NF_NAT_HELPER_PREFIX "%s", name); - if (spec_port == default_port) - snprintf(helper->name, sizeof(helper->name), "%s", name); - else - snprintf(helper->name, sizeof(helper->name), "%s-%u", name, id); + snprintf(helper->name, sizeof(helper->name), "%s", name); if (WARN_ON_ONCE(expect_class_max >= NF_CT_MAX_EXPECT_CLASSES)) return; diff --git a/net/netfilter/nf_conntrack_irc.c b/net/netfilter/nf_conntrack_irc.c index 0c117b8492e9..4e6bafe41437 100644 --- a/net/netfilter/nf_conntrack_irc.c +++ b/net/netfilter/nf_conntrack_irc.c @@ -21,9 +21,6 @@ #include #include -#define MAX_PORTS 8 -static unsigned short ports[MAX_PORTS]; -static unsigned int ports_c; static unsigned int max_dcc_channels = 8; static unsigned int dcc_timeout __read_mostly = 300; /* This is slow, but it's simple. --RR */ @@ -42,8 +39,6 @@ MODULE_LICENSE("GPL"); MODULE_ALIAS("ip_conntrack_irc"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); -module_param_array(ports, ushort, &ports_c, 0400); -MODULE_PARM_DESC(ports, "port numbers of IRC servers"); module_param(max_dcc_channels, uint, 0400); MODULE_PARM_DESC(max_dcc_channels, "max number of expected DCC channels per " "IRC session"); @@ -254,13 +249,15 @@ static int help(struct sk_buff *skb, unsigned int protoff, return ret; } -static struct nf_conntrack_helper irc[MAX_PORTS] __read_mostly; -static struct nf_conntrack_helper *irc_ptr[MAX_PORTS] __read_mostly; +static struct nf_conntrack_helper irc __read_mostly; +static struct nf_conntrack_helper *irc_ptr __read_mostly; static struct nf_conntrack_expect_policy irc_exp_policy; static int __init nf_conntrack_irc_init(void) { - int i, ret; + int ret; + + nf_conntrack_helper_deprecated(HELPER_NAME); if (max_dcc_channels < 1) { pr_err("max_dcc_channels must not be zero\n"); @@ -280,17 +277,11 @@ static int __init nf_conntrack_irc_init(void) if (!irc_buffer) return -ENOMEM; - /* If no port given, default to standard irc port */ - if (ports_c == 0) - ports[ports_c++] = IRC_PORT; - - for (i = 0; i < ports_c; i++) { - nf_ct_helper_init(&irc[i], AF_INET, IPPROTO_TCP, HELPER_NAME, - IRC_PORT, ports[i], i, &irc_exp_policy, - 0, help, NULL, THIS_MODULE); - } + nf_ct_helper_init(&irc, AF_INET, IPPROTO_TCP, HELPER_NAME, + &irc_exp_policy, + 0, help, NULL, THIS_MODULE); - ret = nf_conntrack_helpers_register(&irc[0], ports_c, irc_ptr); + ret = nf_conntrack_helper_register(&irc, &irc_ptr); if (ret) { pr_err("failed to register helpers\n"); kfree(irc_buffer); @@ -302,7 +293,7 @@ static int __init nf_conntrack_irc_init(void) static void __exit nf_conntrack_irc_fini(void) { - nf_conntrack_helpers_unregister(irc_ptr, ports_c); + nf_conntrack_helper_unregister(irc_ptr); kfree(irc_buffer); } diff --git a/net/netfilter/nf_conntrack_netbios_ns.c b/net/netfilter/nf_conntrack_netbios_ns.c index 89d1cf7d6512..caa2b101fa9e 100644 --- a/net/netfilter/nf_conntrack_netbios_ns.c +++ b/net/netfilter/nf_conntrack_netbios_ns.c @@ -21,7 +21,6 @@ #include #define HELPER_NAME "netbios-ns" -#define NMBD_PORT 137 MODULE_AUTHOR("Patrick McHardy "); MODULE_DESCRIPTION("NetBIOS name service broadcast connection tracking helper"); @@ -54,7 +53,6 @@ static int __init nf_conntrack_netbios_ns_init(void) exp_policy.timeout = timeout; nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP, HELPER_NAME, - NMBD_PORT, NMBD_PORT, NMBD_PORT, &exp_policy, 0, netbios_ns_help, NULL, THIS_MODULE); return nf_conntrack_helper_register(&helper, &helper_ptr); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index b429e648f06c..31cbb1b55b9e 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1953,19 +1953,6 @@ static int ctnetlink_change_helper(struct nf_conn *ct, return err; } - if (!strcmp(helpname, "") && help) { - helper = rcu_dereference(help->helper); - if (helper) { - /* we had a helper before ... */ - nf_ct_remove_expectations(ct); - RCU_INIT_POINTER(help->helper, NULL); - if (refcount_dec_and_test(&helper->ct_refcnt)) - kfree_rcu(helper, rcu); - } - rcu_read_unlock(); - return 0; - } - helper = __nf_conntrack_helper_find(helpname, nf_ct_l3num(ct), nf_ct_protonum(ct)); if (helper == NULL) { @@ -3014,8 +3001,7 @@ static int ctnetlink_exp_dump_expect(struct sk_buff *skb, const struct nf_conntrack_expect *exp) { - struct nf_conn *master = exp->master; - long timeout = ((long)exp->timeout.expires - (long)jiffies) / HZ; + __s32 timeout = (__s32)(READ_ONCE(exp->timeout) - nfct_time_stamp) / HZ; struct nf_conntrack_helper *helper; #if IS_ENABLED(CONFIG_NF_NAT) struct nlattr *nest_parms; @@ -3030,9 +3016,7 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, goto nla_put_failure; if (ctnetlink_exp_dump_mask(skb, &exp->tuple, &exp->mask) < 0) goto nla_put_failure; - if (ctnetlink_exp_dump_tuple(skb, - &master->tuplehash[IP_CT_DIR_ORIGINAL].tuple, - CTA_EXPECT_MASTER) < 0) + if (ctnetlink_exp_dump_tuple(skb, &exp->master_tuple, CTA_EXPECT_MASTER) < 0) goto nla_put_failure; #if IS_ENABLED(CONFIG_NF_NAT) @@ -3045,9 +3029,9 @@ ctnetlink_exp_dump_expect(struct sk_buff *skb, if (nla_put_be32(skb, CTA_EXPECT_NAT_DIR, htonl(exp->dir))) goto nla_put_failure; - nat_tuple.src.l3num = nf_ct_l3num(master); + nat_tuple.src.l3num = exp->master_tuple.src.l3num; nat_tuple.src.u3 = exp->saved_addr; - nat_tuple.dst.protonum = nf_ct_protonum(master); + nat_tuple.dst.protonum = exp->master_tuple.dst.protonum; nat_tuple.src.u = exp->saved_proto; if (ctnetlink_exp_dump_tuple(skb, &nat_tuple, @@ -3178,6 +3162,9 @@ ctnetlink_exp_dump_table(struct sk_buff *skb, struct netlink_callback *cb) restart: hlist_for_each_entry_rcu(exp, &nf_ct_expect_hash[cb->args[0]], hnode) { + if (nf_ct_exp_is_expired(exp)) + continue; + if (l3proto && exp->tuple.src.l3num != l3proto) continue; @@ -3456,11 +3443,8 @@ static int ctnetlink_del_expect(struct sk_buff *skb, } /* after list removal, usage count == 1 */ - if (timer_delete(&exp->timeout)) { - nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, - nlmsg_report(info->nlh)); - nf_ct_expect_put(exp); - } + nf_ct_unlink_expect_report(exp, NETLINK_CB(skb).portid, + nlmsg_report(info->nlh)); spin_unlock_bh(&nf_conntrack_expect_lock); /* have to put what we 'get' above. * after this line usage count == 0 */ @@ -3484,14 +3468,10 @@ static int ctnetlink_change_expect(struct nf_conntrack_expect *x, const struct nlattr * const cda[]) { - if (cda[CTA_EXPECT_TIMEOUT]) { - if (!timer_delete(&x->timeout)) - return -ETIME; + if (cda[CTA_EXPECT_TIMEOUT]) + WRITE_ONCE(x->timeout, nfct_time_stamp + + ntohl(nla_get_be32(cda[CTA_EXPECT_TIMEOUT])) * HZ); - x->timeout.expires = jiffies + - ntohl(nla_get_be32(cda[CTA_EXPECT_TIMEOUT])) * HZ; - add_timer(&x->timeout); - } return 0; } @@ -3569,8 +3549,6 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, if (cda[CTA_EXPECT_FLAGS]) { exp->flags = ntohl(nla_get_be32(cda[CTA_EXPECT_FLAGS])); exp->flags &= ~NF_CT_EXPECT_USERSPACE; - } else { - exp->flags = 0; } if (cda[CTA_EXPECT_FN]) { const char *name = nla_data(cda[CTA_EXPECT_FN]); @@ -3582,8 +3560,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, goto err_out; } exp->expectfn = expfn->expectfn; - } else - exp->expectfn = NULL; + } exp->class = class; exp->master = ct; @@ -3593,6 +3570,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, #endif rcu_assign_pointer(exp->helper, helper); rcu_assign_pointer(exp->assign_helper, assign_helper); + exp->master_tuple = ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple; exp->tuple = *tuple; exp->mask.src.u3 = mask->src.u3; exp->mask.src.u.all = mask->src.u.all; @@ -3602,12 +3580,6 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp, nf_ct_l3num(ct)); if (err < 0) goto err_out; -#if IS_ENABLED(CONFIG_NF_NAT) - } else { - memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); - memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); - exp->dir = 0; -#endif } return exp; err_out: diff --git a/net/netfilter/nf_conntrack_ovs.c b/net/netfilter/nf_conntrack_ovs.c index 49d1511e9921..b4085af3ad1c 100644 --- a/net/netfilter/nf_conntrack_ovs.c +++ b/net/netfilter/nf_conntrack_ovs.c @@ -31,8 +31,8 @@ int nf_ct_helper(struct sk_buff *skb, struct nf_conn *ct, if (!helper) return NF_ACCEPT; - if (helper->tuple.src.l3num != NFPROTO_UNSPEC && - helper->tuple.src.l3num != proto) + if (helper->nfproto != NFPROTO_UNSPEC && + helper->nfproto != proto) return NF_ACCEPT; switch (proto) { @@ -60,7 +60,7 @@ int nf_ct_helper(struct sk_buff *skb, struct nf_conn *ct, return NF_DROP; } - if (helper->tuple.dst.protonum != proto) + if (helper->l4proto != proto) return NF_ACCEPT; helper_cb = rcu_dereference(helper->help); diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c index 776505a78e64..cbf32a3cb1f6 100644 --- a/net/netfilter/nf_conntrack_pptp.c +++ b/net/netfilter/nf_conntrack_pptp.c @@ -540,11 +540,13 @@ static int __init nf_conntrack_pptp_init(void) NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_pptp_master)); nf_ct_helper_init(&pptp, AF_INET, IPPROTO_TCP, - "pptp", PPTP_CONTROL_PORT, PPTP_CONTROL_PORT, PPTP_CONTROL_PORT, + "pptp", &pptp_exp_policy, 0, conntrack_pptp_help, NULL, THIS_MODULE); pptp.destroy = gre_pptp_destroy_siblings; + nf_conntrack_helper_deprecated(pptp.name); + return nf_conntrack_helper_register(&pptp, &pptp_ptr); } diff --git a/net/netfilter/nf_conntrack_sane.c b/net/netfilter/nf_conntrack_sane.c index 39085acf7a71..a0658f69d78f 100644 --- a/net/netfilter/nf_conntrack_sane.c +++ b/net/netfilter/nf_conntrack_sane.c @@ -34,11 +34,6 @@ MODULE_AUTHOR("Michal Schmidt "); MODULE_DESCRIPTION("SANE connection tracking helper"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); -#define MAX_PORTS 8 -static u_int16_t ports[MAX_PORTS]; -static unsigned int ports_c; -module_param_array(ports, ushort, &ports_c, 0400); - struct sane_request { __be32 RPC_code; #define SANE_NET_START 7 /* RPC code */ @@ -169,8 +164,8 @@ static int help(struct sk_buff *skb, return ret; } -static struct nf_conntrack_helper sane[MAX_PORTS * 2] __read_mostly; -static struct nf_conntrack_helper *sane_ptr[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper sane __read_mostly; +static struct nf_conntrack_helper *sane_ptr __read_mostly; static const struct nf_conntrack_expect_policy sane_exp_policy = { .max_expected = 1, @@ -179,32 +174,21 @@ static const struct nf_conntrack_expect_policy sane_exp_policy = { static void __exit nf_conntrack_sane_fini(void) { - nf_conntrack_helpers_unregister(sane_ptr, ports_c * 2); + nf_conntrack_helper_unregister(sane_ptr); } static int __init nf_conntrack_sane_init(void) { - int i, ret = 0; + int ret = 0; NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_sane_master)); - if (ports_c == 0) - ports[ports_c++] = SANE_PORT; - - /* FIXME should be configurable whether IPv4 and IPv6 connections - are tracked or not - YK */ - for (i = 0; i < ports_c; i++) { - nf_ct_helper_init(&sane[2 * i], AF_INET, IPPROTO_TCP, - HELPER_NAME, SANE_PORT, ports[i], ports[i], - &sane_exp_policy, 0, help, NULL, - THIS_MODULE); - nf_ct_helper_init(&sane[2 * i + 1], AF_INET6, IPPROTO_TCP, - HELPER_NAME, SANE_PORT, ports[i], ports[i], - &sane_exp_policy, 0, help, NULL, - THIS_MODULE); - } + nf_ct_helper_init(&sane, NFPROTO_UNSPEC, IPPROTO_TCP, + HELPER_NAME, + &sane_exp_policy, 0, help, NULL, + THIS_MODULE); - ret = nf_conntrack_helpers_register(sane, ports_c * 2, sane_ptr); + ret = nf_conntrack_helper_register(&sane, &sane_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index c606d1f60b58..0ff089e03891 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -35,12 +35,6 @@ MODULE_DESCRIPTION("SIP connection tracking helper"); MODULE_ALIAS("ip_conntrack_sip"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); -#define MAX_PORTS 8 -static unsigned short ports[MAX_PORTS]; -static unsigned int ports_c; -module_param_array(ports, ushort, &ports_c, 0400); -MODULE_PARM_DESC(ports, "port numbers of SIP servers"); - static unsigned int sip_timeout __read_mostly = SIP_TIMEOUT; module_param(sip_timeout, uint, 0600); MODULE_PARM_DESC(sip_timeout, "timeout for the master SIP session"); @@ -897,11 +891,10 @@ static int refresh_signalling_expectation(struct nf_conn *ct, exp->tuple.dst.protonum != proto || exp->tuple.dst.u.udp.port != port) continue; - if (mod_timer_pending(&exp->timeout, jiffies + expires * HZ)) { - exp->flags &= ~NF_CT_EXPECT_INACTIVE; - found = 1; - break; - } + WRITE_ONCE(exp->timeout, nfct_time_stamp + (expires * HZ)); + WRITE_ONCE(exp->flags, exp->flags & ~NF_CT_EXPECT_INACTIVE); + found = 1; + break; } spin_unlock_bh(&nf_conntrack_expect_lock); return found; @@ -920,8 +913,7 @@ static void flush_expectations(struct nf_conn *ct, bool media) hlist_for_each_entry_safe(exp, next, &help->expectations, lnode) { if ((exp->class != SIP_EXPECT_SIGNALLING) ^ media) continue; - if (!nf_ct_remove_expect(exp)) - continue; + nf_ct_unlink_expect(exp); if (!media) break; } @@ -958,7 +950,6 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, return NF_ACCEPT; saddr = &ct->tuplehash[!dir].tuple.src.u3; } else if (sip_external_media) { - struct net_device *dev = skb_dst(skb)->dev; struct dst_entry *dst = NULL; struct flowi fl; @@ -980,7 +971,11 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, * through the same interface as the signalling peer. */ if (dst) { - bool external_media = (dst->dev == dev); + const struct dst_entry *this_dst = skb_dst(skb); + bool external_media = false; + + if (this_dst && dst->dev == this_dst->dev) + external_media = true; dst_release(dst); if (external_media) @@ -1413,7 +1408,6 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); - exp->timeout.expires = sip_timeout * HZ; rcu_assign_pointer(exp->assign_helper, helper); exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; @@ -1767,8 +1761,8 @@ static int sip_help_udp(struct sk_buff *skb, unsigned int protoff, return process_sip_msg(skb, ct, protoff, dataoff, &dptr, &datalen); } -static struct nf_conntrack_helper sip[MAX_PORTS * 4] __read_mostly; -static struct nf_conntrack_helper *sip_ptr[MAX_PORTS * 4] __read_mostly; +static struct nf_conntrack_helper sip[2] __read_mostly; +static struct nf_conntrack_helper *sip_ptr[2] __read_mostly; static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1] = { [SIP_EXPECT_SIGNALLING] = { @@ -1795,38 +1789,25 @@ static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1 static void __exit nf_conntrack_sip_fini(void) { - nf_conntrack_helpers_unregister(sip_ptr, ports_c * 4); + nf_conntrack_helpers_unregister(sip_ptr, 2); } static int __init nf_conntrack_sip_init(void) { - int i, ret; + int ret; NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_sip_master)); - if (ports_c == 0) - ports[ports_c++] = SIP_PORT; - - for (i = 0; i < ports_c; i++) { - nf_ct_helper_init(&sip[4 * i], AF_INET, IPPROTO_UDP, - HELPER_NAME, SIP_PORT, ports[i], i, - sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp, - NULL, THIS_MODULE); - nf_ct_helper_init(&sip[4 * i + 1], AF_INET, IPPROTO_TCP, - HELPER_NAME, SIP_PORT, ports[i], i, - sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp, - NULL, THIS_MODULE); - nf_ct_helper_init(&sip[4 * i + 2], AF_INET6, IPPROTO_UDP, - HELPER_NAME, SIP_PORT, ports[i], i, - sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp, - NULL, THIS_MODULE); - nf_ct_helper_init(&sip[4 * i + 3], AF_INET6, IPPROTO_TCP, - HELPER_NAME, SIP_PORT, ports[i], i, - sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp, - NULL, THIS_MODULE); - } + nf_ct_helper_init(&sip[0], NFPROTO_UNSPEC, IPPROTO_UDP, + HELPER_NAME, + sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp, + NULL, THIS_MODULE); + nf_ct_helper_init(&sip[1], NFPROTO_UNSPEC, IPPROTO_TCP, + HELPER_NAME, + sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp, + NULL, THIS_MODULE); - ret = nf_conntrack_helpers_register(sip, ports_c * 4, sip_ptr); + ret = nf_conntrack_helpers_register(sip, 2, sip_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_conntrack_snmp.c b/net/netfilter/nf_conntrack_snmp.c index b6fce5703fce..109986d5d55e 100644 --- a/net/netfilter/nf_conntrack_snmp.c +++ b/net/netfilter/nf_conntrack_snmp.c @@ -14,8 +14,6 @@ #include #include -#define SNMP_PORT 161 - MODULE_AUTHOR("Jiri Olsa "); MODULE_DESCRIPTION("SNMP service broadcast connection tracking helper"); MODULE_LICENSE("GPL"); @@ -55,7 +53,7 @@ static int __init nf_conntrack_snmp_init(void) exp_policy.timeout = timeout; nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP, - "snmp", SNMP_PORT, SNMP_PORT, SNMP_PORT, + "snmp", &exp_policy, 0, snmp_conntrack_help, NULL, THIS_MODULE); diff --git a/net/netfilter/nf_conntrack_tftp.c b/net/netfilter/nf_conntrack_tftp.c index 4393c435aa35..a69559edf9b3 100644 --- a/net/netfilter/nf_conntrack_tftp.c +++ b/net/netfilter/nf_conntrack_tftp.c @@ -26,12 +26,6 @@ MODULE_LICENSE("GPL"); MODULE_ALIAS("ip_conntrack_tftp"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); -#define MAX_PORTS 8 -static unsigned short ports[MAX_PORTS]; -static unsigned int ports_c; -module_param_array(ports, ushort, &ports_c, 0400); -MODULE_PARM_DESC(ports, "Port numbers of TFTP servers"); - nf_nat_tftp_hook_fn __rcu *nf_nat_tftp_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_nat_tftp_hook); @@ -95,8 +89,8 @@ static int tftp_help(struct sk_buff *skb, return ret; } -static struct nf_conntrack_helper tftp[MAX_PORTS * 2] __read_mostly; -static struct nf_conntrack_helper *tftp_ptr[MAX_PORTS * 2] __read_mostly; +static struct nf_conntrack_helper tftp __read_mostly; +static struct nf_conntrack_helper *tftp_ptr __read_mostly; static const struct nf_conntrack_expect_policy tftp_exp_policy = { .max_expected = 1, @@ -105,30 +99,21 @@ static const struct nf_conntrack_expect_policy tftp_exp_policy = { static void __exit nf_conntrack_tftp_fini(void) { - nf_conntrack_helpers_unregister(tftp_ptr, ports_c * 2); + nf_conntrack_helper_unregister(tftp_ptr); } static int __init nf_conntrack_tftp_init(void) { - int i, ret; + int ret; NF_CT_HELPER_BUILD_BUG_ON(0); - if (ports_c == 0) - ports[ports_c++] = TFTP_PORT; - - for (i = 0; i < ports_c; i++) { - nf_ct_helper_init(&tftp[2 * i], AF_INET, IPPROTO_UDP, - HELPER_NAME, TFTP_PORT, ports[i], i, - &tftp_exp_policy, 0, tftp_help, NULL, - THIS_MODULE); - nf_ct_helper_init(&tftp[2 * i + 1], AF_INET6, IPPROTO_UDP, - HELPER_NAME, TFTP_PORT, ports[i], i, - &tftp_exp_policy, 0, tftp_help, NULL, - THIS_MODULE); - } + nf_ct_helper_init(&tftp, NFPROTO_UNSPEC, IPPROTO_UDP, + HELPER_NAME, + &tftp_exp_policy, 0, tftp_help, NULL, + THIS_MODULE); - ret = nf_conntrack_helpers_register(tftp, ports_c * 2, tftp_ptr); + ret = nf_conntrack_helper_register(&tftp, &tftp_ptr); if (ret < 0) { pr_err("failed to register helpers\n"); return ret; diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 785d8c244a77..99c5b9d671a0 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -505,8 +505,13 @@ static u32 nf_flow_table_tcp_timeout(const struct nf_conn *ct) */ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) { - static const u32 min_timeout = 5 * 60 * HZ; - u32 expires = nf_ct_expires(ct); + static const s32 min_timeout = 5 * 60 * HZ; + u32 ct_timeout = READ_ONCE(ct->timeout); + s32 expires; + + expires = ct_timeout - nfct_time_stamp; + if (expires <= 0) /* already expired */ + return; /* normal case: large enough timeout, nothing to do. */ if (likely(expires >= min_timeout)) @@ -524,7 +529,7 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) if (nf_ct_is_confirmed(ct) && test_bit(IPS_OFFLOAD_BIT, &ct->status)) { u8 l4proto = nf_ct_protonum(ct); - u32 new_timeout = true; + u32 new_timeout = 1; switch (l4proto) { case IPPROTO_UDP: @@ -549,7 +554,7 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) */ if (new_timeout) { new_timeout += nfct_time_stamp; - cmpxchg(&ct->timeout, expires, new_timeout); + cmpxchg(&ct->timeout, ct_timeout, new_timeout); } } diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 9c05a50d6013..29e93ac1e2e4 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -326,8 +326,10 @@ static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx, return false; iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset); - size = iph->ihl << 2; + if (iph->ihl < 5) + return false; + size = iph->ihl << 2; if (ip_is_fragment(iph) || unlikely(ip_has_options(size))) return false; @@ -335,9 +337,9 @@ static bool nf_flow_ip4_tunnel_proto(struct nf_flowtable_ctx *ctx, return false; if (iph->protocol == IPPROTO_IPIP) { - ctx->tun.proto = IPPROTO_IPIP; + ctx->tun.proto = iph->protocol; ctx->tun.hdr_size = size; - ctx->offset += size; + ctx->offset += ctx->tun.hdr_size; } return true; @@ -347,29 +349,23 @@ static bool nf_flow_ip6_tunnel_proto(struct nf_flowtable_ctx *ctx, struct sk_buff *skb) { #if IS_ENABLED(CONFIG_IPV6) - struct ipv6hdr *ip6h, _ip6h; - __be16 frag_off; - u8 nexthdr; - int hdrlen; + struct ipv6hdr *ip6h; - ip6h = skb_header_pointer(skb, ctx->offset, sizeof(*ip6h), &_ip6h); - if (!ip6h) + if (!pskb_may_pull(skb, sizeof(*ip6h) + ctx->offset)) return false; + ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset); if (ip6h->hop_limit <= 1) return false; - nexthdr = ip6h->nexthdr; - hdrlen = ipv6_skip_exthdr(skb, sizeof(*ip6h) + ctx->offset, &nexthdr, - &frag_off); - if (hdrlen < 0) + if (ipv6_ext_hdr(ip6h->nexthdr)) return false; - if (nexthdr == IPPROTO_IPV6) { - ctx->tun.hdr_size = hdrlen; - ctx->tun.proto = IPPROTO_IPV6; + if (ip6h->nexthdr == IPPROTO_IPV6) { + ctx->tun.proto = ip6h->nexthdr; + ctx->tun.hdr_size = sizeof(*ip6h); + ctx->offset += ctx->tun.hdr_size; } - ctx->offset += ctx->tun.hdr_size; return true; #else @@ -648,25 +644,19 @@ static int nf_flow_tunnel_v4_push(struct net *net, struct sk_buff *skb, return 0; } -struct ipv6_tel_txoption { - struct ipv6_txoptions ops; - __u8 dst_opt[8]; -}; - static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - struct in6_addr **ip6_daddr, - int encap_limit) + struct in6_addr **ip6_daddr) { struct ipv6hdr *ip6h = (struct ipv6hdr *)skb_network_header(skb); - u8 hop_limit = ip6h->hop_limit, proto = IPPROTO_IPV6; struct rtable *rt = dst_rtable(tuple->dst_cache); __u8 dsfield = ipv6_get_dsfield(ip6h); struct flowi6 fl6 = { .daddr = tuple->tun.src_v6, .saddr = tuple->tun.dst_v6, - .flowi6_proto = proto, + .flowi6_proto = IPPROTO_IPV6, }; + u8 hop_limit = ip6h->hop_limit; int err, mtu; u32 headroom; @@ -674,41 +664,18 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, if (err) return err; - skb_set_inner_ipproto(skb, proto); + skb_set_inner_ipproto(skb, IPPROTO_IPV6); headroom = sizeof(*ip6h) + LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len; - if (encap_limit) - headroom += 8; err = skb_cow_head(skb, headroom); if (err) return err; skb_scrub_packet(skb, true); mtu = dst_mtu(&rt->dst) - sizeof(*ip6h); - if (encap_limit) - mtu -= 8; mtu = max(mtu, IPV6_MIN_MTU); skb_dst_update_pmtu_no_confirm(skb, mtu); - if (encap_limit > 0) { - struct ipv6_tel_txoption opt = { - .dst_opt[2] = IPV6_TLV_TNL_ENCAP_LIMIT, - .dst_opt[3] = 1, - .dst_opt[4] = encap_limit, - .dst_opt[5] = IPV6_TLV_PADN, - .dst_opt[6] = 1, - }; - struct ipv6_opt_hdr *hopt; - - opt.ops.dst1opt = (struct ipv6_opt_hdr *)opt.dst_opt; - opt.ops.opt_nflen = 8; - - hopt = skb_push(skb, ipv6_optlen(opt.ops.dst1opt)); - memcpy(hopt, opt.ops.dst1opt, ipv6_optlen(opt.ops.dst1opt)); - hopt->nexthdr = IPPROTO_IPV6; - proto = NEXTHDR_DEST; - } - skb_push(skb, sizeof(*ip6h)); skb_reset_network_header(skb); @@ -716,7 +683,7 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, ip6_flow_hdr(ip6h, dsfield, ip6_make_flowlabel(net, skb, fl6.flowlabel, true, &fl6)); ip6h->hop_limit = hop_limit; - ip6h->nexthdr = proto; + ip6h->nexthdr = IPPROTO_IPV6; ip6h->daddr = tuple->tun.src_v6; ip6h->saddr = tuple->tun.dst_v6; ipv6_hdr(skb)->payload_len = htons(skb->len - sizeof(*ip6h)); @@ -729,12 +696,10 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - struct in6_addr **ip6_daddr, - int encap_limit) + struct in6_addr **ip6_daddr) { if (tuple->tun_num) - return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, ip6_daddr, - encap_limit); + return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, ip6_daddr); return 0; } @@ -1089,7 +1054,7 @@ static int nf_flow_tuple_ipv6(struct nf_flowtable_ctx *ctx, struct sk_buff *skb, static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx, struct nf_flowtable *flow_table, struct flow_offload_tuple_rhash *tuplehash, - struct sk_buff *skb, int encap_limit) + struct sk_buff *skb) { enum flow_offload_tuple_dir dir; struct flow_offload *flow; @@ -1100,11 +1065,8 @@ static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx, flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset; - if (flow->tuplehash[!dir].tuple.tun_num) { + if (flow->tuplehash[!dir].tuple.tun_num) mtu -= sizeof(*ip6h); - if (encap_limit > 0) - mtu -= 8; /* encap limit option */ - } if (unlikely(nf_flow_exceeds_mtu(skb, mtu))) return 0; @@ -1158,7 +1120,6 @@ unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { - int encap_limit = IPV6_DEFAULT_TNL_ENCAP_LIMIT; struct flow_offload_tuple_rhash *tuplehash; struct nf_flowtable *flow_table = priv; struct flow_offload_tuple *other_tuple; @@ -1177,8 +1138,7 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, if (tuplehash == NULL) return NF_ACCEPT; - ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb, - encap_limit); + ret = nf_flow_offload_ipv6_forward(&ctx, flow_table, tuplehash, skb); if (ret < 0) return NF_DROP; else if (ret == 0) @@ -1198,7 +1158,7 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, ip6_daddr = &other_tuple->src_v6; if (nf_flow_tunnel_v6_push(state->net, skb, other_tuple, - &ip6_daddr, encap_limit) < 0) + &ip6_daddr) < 0) return NF_DROP; switch (tuplehash->tuple.xmit_type) { diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 1e7e216b9f89..98c03b487f52 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -53,8 +53,10 @@ static int nft_dev_fill_forward_path(const struct nf_flow_route *route, struct neighbour *n; u8 nud_state; - if (!nft_is_valid_ether_device(dev)) + if (!nft_is_valid_ether_device(dev)) { + eth_zero_addr(ha); goto out; + } n = dst_neigh_lookup(dst_cache, daddr); if (!n) diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 2bbf5163c0e2..8ac326e1eb5b 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -39,12 +39,6 @@ static struct hlist_head *nf_nat_bysource __read_mostly; static unsigned int nf_nat_htable_size __read_mostly; static siphash_aligned_key_t nf_nat_hash_rnd; -struct nf_nat_lookup_hook_priv { - struct nf_hook_entries __rcu *entries; - - struct rcu_head rcu_head; -}; - struct nf_nat_hooks_net { struct nf_hook_ops *nat_hook_ops; unsigned int users; @@ -1181,6 +1175,16 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops, struct nf_hook_ops *nat_ops; int i, ret; +#ifndef MODULE + /* If nf_nat_core is built-in and nf_nat_init() fails, dependent + * modules like nft_chain_nat.ko may still call this function. + * However, nat_net would be invalid, likely pointing to some other + * per-net structure. + */ + if (WARN_ON_ONCE(!nf_nat_hook)) + return -EOPNOTSUPP; +#endif + if (WARN_ON_ONCE(pf >= ARRAY_SIZE(nat_net->nat_proto_net))) return -EINVAL; diff --git a/net/netfilter/nf_nat_proto.c b/net/netfilter/nf_nat_proto.c index 07f51fe75fbe..64b9bac228ea 100644 --- a/net/netfilter/nf_nat_proto.c +++ b/net/netfilter/nf_nat_proto.c @@ -770,6 +770,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = { .pf = NFPROTO_IPV4, .hooknum = NF_INET_PRE_ROUTING, .priority = NF_IP_PRI_NAT_DST, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* After packet filtering, change source */ { @@ -777,6 +778,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = { .pf = NFPROTO_IPV4, .hooknum = NF_INET_POST_ROUTING, .priority = NF_IP_PRI_NAT_SRC, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* Before packet filtering, change destination */ { @@ -784,6 +786,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = { .pf = NFPROTO_IPV4, .hooknum = NF_INET_LOCAL_OUT, .priority = NF_IP_PRI_NAT_DST, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* After packet filtering, change source */ { @@ -791,6 +794,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = { .pf = NFPROTO_IPV4, .hooknum = NF_INET_LOCAL_IN, .priority = NF_IP_PRI_NAT_SRC, + .hook_ops_type = NF_HOOK_OP_NAT, }, }; @@ -1031,6 +1035,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = { .pf = NFPROTO_IPV6, .hooknum = NF_INET_PRE_ROUTING, .priority = NF_IP6_PRI_NAT_DST, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* After packet filtering, change source */ { @@ -1038,6 +1043,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = { .pf = NFPROTO_IPV6, .hooknum = NF_INET_POST_ROUTING, .priority = NF_IP6_PRI_NAT_SRC, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* Before packet filtering, change destination */ { @@ -1045,6 +1051,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = { .pf = NFPROTO_IPV6, .hooknum = NF_INET_LOCAL_OUT, .priority = NF_IP6_PRI_NAT_DST, + .hook_ops_type = NF_HOOK_OP_NAT, }, /* After packet filtering, change source */ { @@ -1052,6 +1059,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = { .pf = NFPROTO_IPV6, .hooknum = NF_INET_LOCAL_IN, .priority = NF_IP6_PRI_NAT_SRC, + .hook_ops_type = NF_HOOK_OP_NAT, }, }; diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index 67c04d8143ab..aea02f6aff09 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -289,13 +289,24 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, /* Mangle destination port for Cisco phones, then fix up checksums */ if (dir == IP_CT_DIR_REPLY && ct_sip_info->forced_dport) { + int doff = *dptr - (const char *)skb->data; struct udphdr *uh; + if (doff <= 0) { + DEBUG_NET_WARN_ON_ONCE(1); + return NF_DROP; + } + + /* ct_sip_info->forced_dport only expected with UDP */ + if (nf_ct_protonum(ct) != IPPROTO_UDP) + return NF_DROP; + if (skb_ensure_writable(skb, skb->len)) { nf_ct_helper_log(skb, ct, "cannot mangle packet"); return NF_DROP; } + *dptr = skb->data + doff; uh = (void *)skb->data + protoff; uh->dest = ct_sip_info->forced_dport; diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c index 57b450024a99..73363ceedebe 100644 --- a/net/netfilter/nf_queue.c +++ b/net/netfilter/nf_queue.c @@ -68,6 +68,7 @@ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry) nf_queue_sock_put(state->sk); #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + dev_put(entry->bridge_dev); dev_put(entry->physin); dev_put(entry->physout); #endif @@ -84,6 +85,8 @@ static void __nf_queue_entry_init_physdevs(struct nf_queue_entry *entry) { #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) const struct sk_buff *skb = entry->skb; + struct dst_entry *dst = skb_dst(skb); + struct net_device *dev = NULL; if (nf_bridge_info_exists(skb)) { entry->physin = nf_bridge_get_physindev(skb, entry->state.net); @@ -92,6 +95,16 @@ static void __nf_queue_entry_init_physdevs(struct nf_queue_entry *entry) entry->physin = NULL; entry->physout = NULL; } + + if (entry->state.pf == NFPROTO_BRIDGE && + dst && (dst->flags & DST_FAKE_RTABLE)) + dev = dst_dev_rcu(dst); + + /* Must hold a reference on the bridge device: dst_hold() protects + * the dst itself, but the fake rtable is embedded in bridge-private + * storage that netdevice teardown can free independently. + */ + entry->bridge_dev = dev; #endif } @@ -108,6 +121,7 @@ bool nf_queue_entry_get_refs(struct nf_queue_entry *entry) dev_hold(state->out); #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + dev_hold(entry->bridge_dev); dev_hold(entry->physin); dev_hold(entry->physout); #endif diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 4884f7f7aaee..a9eaf9455c77 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -6563,6 +6563,9 @@ static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set, if (err < 0) return err; + if (!elem.priv) + return 0; + err = -ENOMEM; skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC); if (skb == NULL) diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index f1460b683d7a..5b6a09f46903 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -67,7 +67,7 @@ nfnl_userspace_cthelper(struct sk_buff *skb, unsigned int protoff, } static const struct nla_policy nfnl_cthelper_tuple_pol[NFCTH_TUPLE_MAX+1] = { - [NFCTH_TUPLE_L3PROTONUM] = { .type = NLA_U16, }, + [NFCTH_TUPLE_L3PROTONUM] = NLA_POLICY_MAX(NLA_BE16, NFPROTO_IPV6), [NFCTH_TUPLE_L4PROTONUM] = { .type = NLA_U8, }, }; @@ -163,6 +163,8 @@ nfnl_cthelper_expect_policy(struct nf_conntrack_expect_policy *expect_policy, tb[NFCTH_POLICY_NAME], NF_CT_HELPER_NAME_LEN); expect_policy->max_expected = ntohl(nla_get_be32(tb[NFCTH_POLICY_EXPECT_MAX])); + if (!expect_policy->max_expected) + expect_policy->max_expected = NF_CT_EXPECT_MAX_CNT; if (expect_policy->max_expected > NF_CT_EXPECT_MAX_CNT) return -EINVAL; @@ -254,7 +256,8 @@ nfnl_cthelper_create(const struct nlattr * const tb[], helper->data_len = size; helper->flags |= NF_CT_HELPER_F_USERSPACE; - memcpy(&helper->tuple, tuple, sizeof(struct nf_conntrack_tuple)); + helper->nfproto = tuple->src.l3num; + helper->l4proto = tuple->dst.protonum; helper->me = THIS_MODULE; helper->help = nfnl_userspace_cthelper; @@ -314,6 +317,8 @@ nfnl_cthelper_update_policy_one(const struct nf_conntrack_expect_policy *policy, new_policy->max_expected = ntohl(nla_get_be32(tb[NFCTH_POLICY_EXPECT_MAX])); + if (!new_policy->max_expected) + new_policy->max_expected = NF_CT_EXPECT_MAX_CNT; if (new_policy->max_expected > NF_CT_EXPECT_MAX_CNT) return -EINVAL; @@ -449,8 +454,8 @@ static int nfnl_cthelper_new(struct sk_buff *skb, const struct nfnl_info *info, if (strncmp(cur->name, helper_name, NF_CT_HELPER_NAME_LEN)) continue; - if ((tuple.src.l3num != cur->tuple.src.l3num || - tuple.dst.protonum != cur->tuple.dst.protonum)) + if ((tuple.src.l3num != cur->nfproto || + tuple.dst.protonum != cur->l4proto)) continue; if (info->nlh->nlmsg_flags & NLM_F_EXCL) @@ -479,10 +484,10 @@ nfnl_cthelper_dump_tuple(struct sk_buff *skb, goto nla_put_failure; if (nla_put_be16(skb, NFCTH_TUPLE_L3PROTONUM, - htons(helper->tuple.src.l3num))) + htons(helper->nfproto))) goto nla_put_failure; - if (nla_put_u8(skb, NFCTH_TUPLE_L4PROTONUM, helper->tuple.dst.protonum)) + if (nla_put_u8(skb, NFCTH_TUPLE_L4PROTONUM, helper->l4proto)) goto nla_put_failure; nla_nest_end(skb, nest_parms); @@ -661,8 +666,8 @@ static int nfnl_cthelper_get(struct sk_buff *skb, const struct nfnl_info *info, continue; if (tuple_set && - (tuple.src.l3num != cur->tuple.src.l3num || - tuple.dst.protonum != cur->tuple.dst.protonum)) + (tuple.src.l3num != cur->nfproto || + tuple.dst.protonum != cur->l4proto)) continue; skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); @@ -721,8 +726,8 @@ static int nfnl_cthelper_del(struct sk_buff *skb, const struct nfnl_info *info, continue; if (tuple_set && - (tuple.src.l3num != cur->tuple.src.l3num || - tuple.dst.protonum != cur->tuple.dst.protonum)) + (tuple.src.l3num != cur->nfproto || + tuple.dst.protonum != cur->l4proto)) continue; found = true; diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c index 170d3db860c5..66c2016f6049 100644 --- a/net/netfilter/nfnetlink_cttimeout.c +++ b/net/netfilter/nfnetlink_cttimeout.c @@ -168,7 +168,7 @@ static int cttimeout_new_timeout(struct sk_buff *skb, if (ret < 0) goto err_free_timeout_policy; - strcpy(timeout->name, nla_data(cda[CTA_TIMEOUT_NAME])); + nla_strscpy(timeout->name, cda[CTA_TIMEOUT_NAME], sizeof(timeout->name)); timeout->timeout->l3num = l3num; timeout->timeout->l4proto = l4proto; refcount_set(&timeout->timeout->refcnt, 1); diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c index 5623c18fcd12..95005e9a6066 100644 --- a/net/netfilter/nfnetlink_hook.c +++ b/net/netfilter/nfnetlink_hook.c @@ -190,7 +190,7 @@ static int nfnl_hook_put_nft_ft_info(struct sk_buff *nlskb, static int nfnl_hook_dump_one(struct sk_buff *nlskb, const struct nfnl_dump_hook_data *ctx, - const struct nf_hook_ops *ops, + const struct nf_hook_ops *ops, int priority, int family, unsigned int seq) { u16 event = nfnl_msg_type(NFNL_SUBSYS_HOOK, NFNL_MSG_HOOK_GET); @@ -244,7 +244,7 @@ static int nfnl_hook_dump_one(struct sk_buff *nlskb, if (ret) goto nla_put_failure; - ret = nla_put_be32(nlskb, NFNLA_HOOK_PRIORITY, htonl(ops->priority)); + ret = nla_put_be32(nlskb, NFNLA_HOOK_PRIORITY, htonl(priority)); if (ret) goto nla_put_failure; @@ -337,6 +337,30 @@ nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *de return hook_head; } +static int nfnl_hook_dump_nat(struct sk_buff *nlskb, + const struct nfnl_dump_hook_data *ctx, + const struct nf_hook_ops *ops, + int family, unsigned int seq) +{ + struct nf_nat_lookup_hook_priv *priv = ops->priv; + struct nf_hook_entries *e = rcu_dereference(priv->entries); + struct nf_hook_ops **nat_ops; + int i, err; + + if (!e) + return 0; + + nat_ops = nf_hook_entries_get_hook_ops(e); + + for (i = 0; i < e->num_hook_entries; i++) { + err = nfnl_hook_dump_one(nlskb, ctx, nat_ops[i], + ops->priority, family, seq); + if (err) + return err; + } + return 0; +} + static int nfnl_hook_dump(struct sk_buff *nlskb, struct netlink_callback *cb) { @@ -365,8 +389,13 @@ static int nfnl_hook_dump(struct sk_buff *nlskb, ops = nf_hook_entries_get_hook_ops(e); for (; i < e->num_hook_entries; i++) { - err = nfnl_hook_dump_one(nlskb, ctx, ops[i], family, - cb->nlh->nlmsg_seq); + if (ops[i]->hook_ops_type == NF_HOOK_OP_NAT) + err = nfnl_hook_dump_nat(nlskb, ctx, ops[i], family, + cb->nlh->nlmsg_seq); + else + err = nfnl_hook_dump_one(nlskb, ctx, ops[i], + ops[i]->priority, family, + cb->nlh->nlmsg_seq); if (err) break; } diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index c5e29fec419b..35d4c6c628ff 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -1184,6 +1184,173 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) return err; } +static bool nfqnl_validate_ipopts(const struct iphdr *iph_new, + const struct nf_queue_entry *e) +{ + const struct iphdr *iph_orig = ip_hdr(e->skb); + unsigned int ihl = iph_new->ihl * 4; + + if (iph_new->ihl != iph_orig->ihl) + return false; + if (ihl == sizeof(*iph_orig)) + return true; + + return memcmp(iph_new + 1, ip_hdr(e->skb) + 1, ihl - sizeof(*iph_orig)) == 0; +} + +static bool nfqnl_validate_ip4(const struct iphdr *iph, unsigned int data_len, + const struct nf_queue_entry *e) +{ + unsigned int ihl; + + if (data_len < sizeof(*iph)) + return false; + + ihl = iph->ihl * 4u; + if (ihl < sizeof(*iph) || data_len < ihl) + return false; + + if (iph->version != 4 || + ((iph->frag_off ^ ip_hdr(e->skb)->frag_off) & ~htons(IP_DF)) != 0) + return false; + + /* BIG TCP won't work; netlink attr len is u16 */ + if (ntohs(iph->tot_len) != data_len) + return false; + + /* support for ipopts mangling would require + * recompile + skb transport header update. + */ + return nfqnl_validate_ipopts(iph, e); +} + +static bool nfqnl_validate_one_exthdr(const u8 *data, + unsigned int data_len, + const struct nf_queue_entry *e, + int start, int hdrlen) +{ + u16 octets; + + if (data_len < hdrlen || hdrlen < 2) + return false; + + while (hdrlen > 0) { + if (data_len < sizeof(octets)) + return false; + data_len -= sizeof(octets); + + if (skb_copy_bits(e->skb, start, &octets, sizeof(octets))) + return false; + + if (hdrlen < sizeof(octets)) + return false; + + hdrlen -= sizeof(octets); + if (memcmp(data, &octets, sizeof(octets))) + return false; + + start += sizeof(octets); + data += sizeof(octets); + } + + return true; +} + +static bool nfqnl_validate_exthdr(const struct ipv6hdr *ip6_new, + unsigned int data_len, + const struct nf_queue_entry *e) +{ + const struct ipv6hdr *ip6_orig = ipv6_hdr(e->skb); + int exthdr_cnt = 0, start = sizeof(*ip6_orig); + const u8 *data = (const u8 *)ip6_new; + u8 orig_nexthdr = ip6_orig->nexthdr; + u8 new_nexthdr = ip6_new->nexthdr; + + if (new_nexthdr != orig_nexthdr) + return false; + + data += sizeof(*ip6_new); + data_len -= sizeof(*ip6_new); + + while (ipv6_ext_hdr(orig_nexthdr)) { + const struct ipv6_opt_hdr *hp; + struct ipv6_opt_hdr _hdr; + int hdrlen; + + if (orig_nexthdr == NEXTHDR_NONE) + return true; + + if (unlikely(exthdr_cnt++ >= IP6_MAX_EXT_HDRS_CNT)) + return false; + + hp = skb_header_pointer(e->skb, start, sizeof(_hdr), &_hdr); + if (!hp) + return false; + + switch (orig_nexthdr) { + case NEXTHDR_FRAGMENT: + hdrlen = sizeof(struct frag_hdr); + break; + case NEXTHDR_AUTH: + hdrlen = ipv6_authlen(hp); + break; + default: + hdrlen = ipv6_optlen(hp); + break; + } + + if (!nfqnl_validate_one_exthdr(data, data_len, e, + start, hdrlen)) + return false; + + orig_nexthdr = hp->nexthdr; + hp = (const void *)data; + new_nexthdr = hp->nexthdr; + + if (new_nexthdr != orig_nexthdr) + return false; + + data_len -= hdrlen; + start += hdrlen; + data += hdrlen; + } + + return true; +} + +static bool nfqnl_validate_ip6(const struct ipv6hdr *ip6, unsigned int data_len, + const struct nf_queue_entry *e) +{ + if (data_len < sizeof(*ip6)) + return false; + + /* BIG TCP/jumbograms won't work; netlink attr len is u16 */ + if (ntohs(ip6->payload_len) != data_len - sizeof(*ip6)) + return false; + + if (ip6->version != 6) + return false; + + return nfqnl_validate_exthdr(ip6, data_len, e); +} + +static bool nfqnl_validate_write(const void *data, unsigned int data_len, + const struct nf_queue_entry *e) +{ + switch (e->state.pf) { + case NFPROTO_IPV4: + return nfqnl_validate_ip4(data, data_len, e); + case NFPROTO_IPV6: + return nfqnl_validate_ip6(data, data_len, e) && + !(IP6CB(e->skb)->flags & IP6SKB_JUMBOGRAM); + case NFPROTO_BRIDGE: + /* No write support. Bridge is dubious: userspace doesn't even see L2 header */ + return false; + } + + return false; +} + static int nfqnl_mangle(void *data, unsigned int data_len, struct nf_queue_entry *e, int diff) { @@ -1192,6 +1359,9 @@ nfqnl_mangle(void *data, unsigned int data_len, struct nf_queue_entry *e, int di if (e->state.net->user_ns != &init_user_ns) return -EPERM; + if (!nfqnl_validate_write(data, data_len, e)) + return -EINVAL; + if (diff < 0) { unsigned int min_len = skb_transport_offset(e->skb); @@ -1262,6 +1432,9 @@ dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex) if (physinif == ifindex || physoutif == ifindex) return 1; + + if (entry->bridge_dev && entry->bridge_dev->ifindex == ifindex) + return 1; #endif if (entry->skb_dev && entry->skb_dev->ifindex == ifindex) return 1; diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 0caa9304d2d0..63864b928259 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -397,6 +397,22 @@ static int nft_target_validate(const struct nft_ctx *ctx, return 0; } +static int nft_target_bridge_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + struct xt_target *target = expr->ops->data; + + /* Do not allow UNSPEC to stand-in for NFPROTO_BRIDGE + * targets: they are incompatible. ebtables targets return + * EBT_ACCEPT, DROP and so on which are not compatible with + * NF_ACCEPT, NF_DROP and so on. + */ + if (target->family != NFPROTO_BRIDGE) + return -ENOENT; + + return nft_target_validate(ctx, expr); +} + static void __nft_match_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt, @@ -932,13 +948,15 @@ nft_target_select_ops(const struct nft_ctx *ctx, ops->init = nft_target_init; ops->destroy = nft_target_destroy; ops->dump = nft_target_dump; - ops->validate = nft_target_validate; ops->data = target; - if (family == NFPROTO_BRIDGE) + if (family == NFPROTO_BRIDGE) { ops->eval = nft_target_eval_bridge; - else + ops->validate = nft_target_bridge_validate; + } else { ops->eval = nft_target_eval_xt; + ops->validate = nft_target_validate; + } return ops; err: diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 25934c6f01fb..358b9287e12e 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -1145,7 +1145,6 @@ static void nft_ct_helper_obj_eval(struct nft_object *obj, help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); if (help && refcount_inc_not_zero(&to_assign->ct_refcnt)) { rcu_assign_pointer(help->helper, to_assign); - set_bit(IPS_HELPER_BIT, &ct->status); if ((ct->status & IPS_NAT_MASK) && !nfct_seqadj(ct)) if (!nfct_seqadj_ext_add(ct)) @@ -1216,11 +1215,23 @@ struct nft_ct_expect_obj { u32 timeout; }; +static int nft_ct_expect_timeout_get(const struct nlattr *attr, u32 *val) +{ + unsigned long jiffies_val = msecs_to_jiffies(nla_get_u32(attr)); + + if (jiffies_val > UINT_MAX) + return -ERANGE; + + *val = jiffies_val; + return 0; +} + static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_object *obj) { struct nft_ct_expect_obj *priv = nft_obj_data(obj); + int err; if (!tb[NFTA_CT_EXPECT_L4PROTO] || !tb[NFTA_CT_EXPECT_DPORT] || @@ -1255,8 +1266,11 @@ static int nft_ct_expect_obj_init(const struct nft_ctx *ctx, return -EOPNOTSUPP; } + err = nft_ct_expect_timeout_get(tb[NFTA_CT_EXPECT_TIMEOUT], &priv->timeout); + if (err) + return err; + priv->dport = nla_get_be16(tb[NFTA_CT_EXPECT_DPORT]); - priv->timeout = nla_get_u32(tb[NFTA_CT_EXPECT_TIMEOUT]); priv->size = nla_get_u8(tb[NFTA_CT_EXPECT_SIZE]); return nf_ct_netns_get(ctx->net, ctx->family); @@ -1276,13 +1290,24 @@ static int nft_ct_expect_obj_dump(struct sk_buff *skb, if (nla_put_be16(skb, NFTA_CT_EXPECT_L3PROTO, htons(priv->l3num)) || nla_put_u8(skb, NFTA_CT_EXPECT_L4PROTO, priv->l4proto) || nla_put_be16(skb, NFTA_CT_EXPECT_DPORT, priv->dport) || - nla_put_u32(skb, NFTA_CT_EXPECT_TIMEOUT, priv->timeout) || + nla_put_u32(skb, NFTA_CT_EXPECT_TIMEOUT, jiffies_to_msecs(priv->timeout)) || nla_put_u8(skb, NFTA_CT_EXPECT_SIZE, priv->size)) return -1; return 0; } +#if IS_ENABLED(CONFIG_NF_NAT) +static void nft_ct_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this) +{ + const struct nf_ct_helper_expectfn *expfn; + + expfn = nf_ct_helper_expectfn_find_by_name("nat-follow-master"); + if (expfn) + expfn->expectfn(ct, this); +} +#endif + static void nft_ct_expect_obj_eval(struct nft_object *obj, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -1326,8 +1351,15 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, priv->l4proto, NULL, &priv->dport); - exp->timeout.expires = jiffies + priv->timeout * HZ; + exp->timeout += priv->timeout; +#if IS_ENABLED(CONFIG_NF_NAT) + if (ct->status & IPS_NAT_MASK) { + exp->saved_proto.tcp.port = priv->dport; + exp->dir = !dir; + exp->expectfn = nft_ct_nat_follow_master; + } +#endif if (nf_ct_expect_related(exp, 0) != 0) regs->verdict.code = NF_DROP; @@ -1361,6 +1393,13 @@ static struct nft_object_type nft_ct_expect_obj_type __read_mostly = { .owner = THIS_MODULE, }; +#if IS_ENABLED(CONFIG_NF_NAT) +static struct nf_ct_helper_expectfn nft_ct_nat __read_mostly = { + .name = "nft_ct-follow-master", + .expectfn = nft_ct_nat_follow_master, +}; +#endif + static int __init nft_ct_module_init(void) { int err; @@ -1386,6 +1425,9 @@ static int __init nft_ct_module_init(void) err = nft_register_obj(&nft_ct_timeout_obj_type); if (err < 0) goto err4; +#endif +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_register(&nft_ct_nat); #endif return 0; @@ -1411,6 +1453,13 @@ static void __exit nft_ct_module_exit(void) nft_unregister_obj(&nft_ct_helper_obj_type); nft_unregister_expr(&nft_notrack_type); nft_unregister_expr(&nft_ct_type); + +#if IS_ENABLED(CONFIG_NF_NAT) + nf_ct_helper_expectfn_unregister(&nft_ct_nat); + synchronize_rcu(); + nf_ct_helper_expectfn_destroy(&nft_ct_nat); + synchronize_rcu(); +#endif } module_init(nft_ct_module_init); diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index e048f05694cd..89555380f1c5 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -31,6 +31,15 @@ int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) const struct nft_fib *priv = nft_expr_priv(expr); unsigned int hooks; + switch (ctx->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + break; + default: + return -EOPNOTSUPP; + } + switch (priv->result) { case NFT_FIB_RESULT_OIF: case NFT_FIB_RESULT_OIFNAME: diff --git a/net/netfilter/nft_fib_netdev.c b/net/netfilter/nft_fib_netdev.c index 3f3478abd845..5774a7544027 100644 --- a/net/netfilter/nft_fib_netdev.c +++ b/net/netfilter/nft_fib_netdev.c @@ -50,6 +50,33 @@ static void nft_fib_netdev_eval(const struct nft_expr *expr, regs->verdict.code = NFT_BREAK; } +static int nft_fib_netdev_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + const struct nft_fib *priv = nft_expr_priv(expr); + unsigned int hooks; + + switch (priv->result) { + case NFT_FIB_RESULT_OIF: + case NFT_FIB_RESULT_OIFNAME: + hooks = (1 << NF_NETDEV_INGRESS); + break; + case NFT_FIB_RESULT_ADDRTYPE: + if (priv->flags & NFTA_FIB_F_IIF) + hooks = (1 << NF_NETDEV_INGRESS); + else if (priv->flags & NFTA_FIB_F_OIF) + hooks = (1 << NF_NETDEV_EGRESS); + else + hooks = (1 << NF_NETDEV_INGRESS) | + (1 << NF_NETDEV_EGRESS); + break; + default: + return -EINVAL; + } + + return nft_chain_validate_hooks(ctx->chain, hooks); +} + static struct nft_expr_type nft_fib_netdev_type; static const struct nft_expr_ops nft_fib_netdev_ops = { .type = &nft_fib_netdev_type, @@ -57,7 +84,7 @@ static const struct nft_expr_ops nft_fib_netdev_ops = { .eval = nft_fib_netdev_eval, .init = nft_fib_init, .dump = nft_fib_dump, - .validate = nft_fib_validate, + .validate = nft_fib_netdev_validate, }; static struct nft_expr_type nft_fib_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c index 9b5821c64442..0a43e0787a68 100644 --- a/net/netfilter/nft_meta.c +++ b/net/netfilter/nft_meta.c @@ -635,8 +635,8 @@ static int nft_meta_get_validate_xfrm(const struct nft_ctx *ctx) #endif } -static int nft_meta_get_validate(const struct nft_ctx *ctx, - const struct nft_expr *expr) +int nft_meta_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) { const struct nft_meta *priv = nft_expr_priv(expr); @@ -652,6 +652,7 @@ static int nft_meta_get_validate(const struct nft_ctx *ctx, return 0; } +EXPORT_SYMBOL_GPL(nft_meta_get_validate); int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index ef2a80dfc68f..391539a1ceaa 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -224,11 +224,17 @@ static int nft_payload_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_payload *priv = nft_expr_priv(expr); + u32 offset; + int err; priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); - priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); + if (err < 0) + return err; + priv->offset = offset; + return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], &priv->dreg, NULL, NFT_DATA_VALUE, priv->len); @@ -621,7 +627,8 @@ static int nft_payload_inner_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_payload *priv = nft_expr_priv(expr); - u32 base; + u32 base, offset; + int err; if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] || !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG]) @@ -639,8 +646,11 @@ static int nft_payload_inner_init(const struct nft_ctx *ctx, } priv->base = base; - priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); + if (err < 0) + return err; + priv->offset = offset; return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], &priv->dreg, NULL, NFT_DATA_VALUE, @@ -824,6 +834,249 @@ nft_payload_set_vlan(const u32 *src, struct sk_buff *skb, u16 offset, u8 len, return true; } +/* Ingress is very early, before l3 protocol handlers. + * There should be no in-tree code that trusts l3/l4 headers + * between ingress and NF_INET_PRE_ROUTING hooks. + */ +static bool nft_in_ingress(const struct nf_hook_state *s) +{ + return s->pf == NFPROTO_NETDEV && s->hook == NF_NETDEV_INGRESS; +} + +static bool nft_nh_write_ok_ip4(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv, + const u32 *src) +{ + unsigned int offset = priv->offset + skb_network_offset(pkt->skb); + const u8 *new_octets = (const u8 *)src; + u8 old_octet; + + switch (priv->offset) { + case 0: /* csum fixups does expand dscp/tos store to 2 bytes. + * make sure ihl/version remain unchanged. + */ + if (skb_copy_bits(pkt->skb, offset, &old_octet, sizeof(old_octet))) + return false; + + return priv->len == 2 && + *new_octets == old_octet; + case offsetof(struct iphdr, tos): + return priv->len == 1; + case offsetof(struct iphdr, id): + return priv->len == 2; + case offsetof(struct iphdr, ttl): + if (priv->len == 1) + return true; + + if (priv->len != 2) + return false; + + /* same, csum fixup does expand ttl store to two bytes. + * check protocol is not altered. + */ + if (skb_copy_bits(pkt->skb, offset + 1, &old_octet, sizeof(old_octet))) + return false; + + return new_octets[1] == old_octet; + case offsetof(struct iphdr, check): + return priv->len <= 2 + 4 + 4; + case offsetof(struct iphdr, saddr): + return priv->len <= 4 + 4; + case offsetof(struct iphdr, daddr): + return priv->len <= 4; + } + + return false; +} + +static bool nft_nh_write_ok_ip6(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv, + const u32 *src) +{ + const struct ipv6hdr *ih = (const void *)src; + + switch (priv->offset) { + case 0: /* store to dscp must not alter ip6 version */ + return priv->len <= 4 && ih->version == 6; + case 2: + return priv->len <= 2; + case offsetof(struct ipv6hdr, hop_limit): + return priv->len == 1; + case offsetof(struct ipv6hdr, saddr): + return priv->len <= 16 + 16; + case offsetof(struct ipv6hdr, daddr): + return priv->len <= 16; + } + + return false; +} + +static bool nft_nh_write_ok_arp(const struct nft_payload_set *priv) +{ + /* Variable size for standard ethernet arp */ + const unsigned int eth_ip = 2 * (ETH_ALEN + 4); + unsigned int offset = priv->offset; + + switch (offset) { + case offsetof(struct arphdr, ar_op): + return priv->len == 2; + default: + break; + } + + /* permit writes post fixed arp header size. offset + len are + * checked vs skb size via skb_ensure_writable. + */ + return offset >= sizeof(struct arphdr) && priv->len <= eth_ip; +} + +static bool nft_nh_write_ok_netdev(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv, + const u32 *src) +{ +#ifdef CONFIG_NF_TABLES_NETDEV + switch (pkt->skb->protocol) { + case htons(ETH_P_ARP): + return nft_nh_write_ok_arp(priv); + case htons(ETH_P_IP): + return nft_nh_write_ok_ip4(pkt, priv, src); + case htons(ETH_P_IPV6): + return nft_nh_write_ok_ip6(pkt, priv, src); + } +#endif + /* default to false for now, relax later in case we have + * use-cases that need inner header manipulation for + * encapsulated traffic like vlan or PPPoE. + */ + return false; +} + +static bool nft_nh_write_ok_bridge(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv, + const u32 *src) +{ +#if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE) + switch (pkt->ethertype) { + case htons(ETH_P_ARP): + return nft_nh_write_ok_arp(priv); + case htons(ETH_P_IP): + return nft_nh_write_ok_ip4(pkt, priv, src); + case htons(ETH_P_IPV6): + return nft_nh_write_ok_ip6(pkt, priv, src); + } +#endif + /* see nft_nh_write_ok_netdev: default to false */ + return false; +} + +static bool nft_nh_write_ok(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv, + const u32 *src) +{ + switch (pkt->state->pf) { + case NFPROTO_ARP: + return nft_nh_write_ok_arp(priv); + case NFPROTO_BRIDGE: + return nft_nh_write_ok_bridge(pkt, priv, src); + case NFPROTO_IPV4: + return nft_nh_write_ok_ip4(pkt, priv, src); + case NFPROTO_IPV6: + return nft_nh_write_ok_ip6(pkt, priv, src); + case NFPROTO_NETDEV: + if (pkt->state->hook == NF_NETDEV_INGRESS) + return true; + return nft_nh_write_ok_netdev(pkt, priv, src); + } + + return false; +} + +/* check linklayer modifications don't spill into network header. */ +static bool nft_ll_write_ok(const struct nft_pktinfo *pkt, int offset) +{ + if (nft_in_ingress(pkt->state)) + return true; + + return offset <= skb_network_offset(pkt->skb); +} + +static bool nft_payload_validate_inet_csum_offset(const struct nft_ctx *ctx, + const struct nft_payload_set *priv) +{ + switch (priv->base) { + case NFT_PAYLOAD_LL_HEADER: + break; + case NFT_PAYLOAD_NETWORK_HEADER: + if (ctx->family == NFPROTO_IPV4) { + if (offsetof(struct iphdr, check) == priv->csum_offset) + return true; + + return false; + } + return true; /* run time validation required */ + case NFT_PAYLOAD_TRANSPORT_HEADER: + if (priv->csum_flags) /* makes no sense, asks for "re-update" of L4 checksum */ + return false; + + /* no further check here; offset can't be negative so bogus + * offsets can corrupt L4 or payload but not l3 headers. + * We already allow arbitrary l4/inner payload writes. + */ + return true; + case NFT_PAYLOAD_INNER_HEADER: + return true; + case NFT_PAYLOAD_TUN_HEADER: + break; + } + + return false; +} + +/* do not allow arbitrary network header mangling via bogus csum_off. + * We only support ipv4. Only NFPROTO_IPV4 can be checked from control + * plane. + */ +static bool nft_payload_csum_nh_write_ok(const struct nft_payload_set *priv, + const struct nft_pktinfo *pkt) +{ + switch (pkt->state->pf) { + case NFPROTO_IPV4: + /* Warning: NFPROTO_INET was not checked; we can't return true here. */ + return priv->csum_offset == offsetof(struct iphdr, check); + case NFPROTO_IPV6: + return false; + case NFPROTO_BRIDGE: + return pkt->ethertype == htons(ETH_P_IP) && + priv->csum_offset == offsetof(struct iphdr, check); + case NFPROTO_NETDEV: + return pkt->skb->protocol == htons(ETH_P_IP) && + priv->csum_offset == offsetof(struct iphdr, check); + } + + return false; +} + +static bool nft_payload_csum_write_ok(const struct nft_pktinfo *pkt, + const struct nft_payload_set *priv) +{ + switch (priv->base) { + case NFT_PAYLOAD_LL_HEADER: + break; + case NFT_PAYLOAD_NETWORK_HEADER: + return nft_payload_csum_nh_write_ok(priv, pkt); + case NFT_PAYLOAD_TRANSPORT_HEADER: + case NFT_PAYLOAD_INNER_HEADER: + /* neither offsets are validated, offsets cannot be + * negative so real l3 headers cannot be mangled. + */ + return true; + case NFT_PAYLOAD_TUN_HEADER: + break; + } + + return false; +} + static void nft_payload_set_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -851,8 +1104,12 @@ static void nft_payload_set_eval(const struct nft_expr *expr, } offset = skb_mac_header(skb) - skb->data - vlan_hlen; + if (!nft_ll_write_ok(pkt, priv->len + priv->offset + offset)) + goto err; break; case NFT_PAYLOAD_NETWORK_HEADER: + if (!nft_nh_write_ok(pkt, priv, src)) + goto err; offset = skb_network_offset(skb); break; case NFT_PAYLOAD_TRANSPORT_HEADER: @@ -884,6 +1141,7 @@ static void nft_payload_set_eval(const struct nft_expr *expr, tsum = csum_partial(src, priv->len, 0); if (priv->csum_type == NFT_PAYLOAD_CSUM_INET && + nft_payload_csum_write_ok(pkt, priv) && nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset)) goto err; @@ -950,7 +1208,26 @@ static int nft_payload_set_init(const struct nft_ctx *ctx, switch (csum_type) { case NFT_PAYLOAD_CSUM_NONE: + if (priv->csum_offset) /* nonsensical */ + return -EINVAL; + + if (priv->csum_flags == 0) + break; + + /* Userspace requests L4 checksum update, e.g.: + * - IPv6 stateless NAT (no l3 csum) + * - transport header mangling + * - inner data mangling + */ + if (priv->base == NFT_PAYLOAD_NETWORK_HEADER || + priv->base == NFT_PAYLOAD_TRANSPORT_HEADER || + priv->base == NFT_PAYLOAD_INNER_HEADER) + break; + + return -EINVAL; case NFT_PAYLOAD_CSUM_INET: + if (!nft_payload_validate_inet_csum_offset(ctx, priv)) + return -EINVAL; break; case NFT_PAYLOAD_CSUM_SCTP: if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER) @@ -958,6 +1235,9 @@ static int nft_payload_set_init(const struct nft_ctx *ctx, if (priv->csum_offset != offsetof(struct sctphdr, checksum)) return -EINVAL; + + if (priv->csum_flags) + return -EINVAL; break; default: return -EOPNOTSUPP; diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 706c78853f24..978bb0c01106 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -342,6 +342,8 @@ #include "nft_set_pipapo_avx2.h" #include "nft_set_pipapo.h" +static void nft_pipapo_abort(const struct nft_set *set); + /** * pipapo_refill() - For each set bit, set bits from selected mapping table item * @map: Bitmap to be scanned for set bits @@ -1296,7 +1298,7 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, const u8 *start_p, *end_p; int i, bsize_max, err = 0; - if (!m) + if (!m || m->state == NFT_PIPAPO_CLONE_ERR) return -ENOMEM; if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END)) @@ -1367,8 +1369,10 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, else ret = pipapo_expand(f, start, end, f->groups * f->bb); - if (ret < 0) - return ret; + if (ret < 0) { + err = ret; + goto abort; + } if (f->bsize > bsize_max) bsize_max = f->bsize; @@ -1384,7 +1388,7 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, err = pipapo_realloc_scratch(m, bsize_max); if (err) - return err; + goto abort; m->bsize_max = bsize_max; } else { @@ -1396,7 +1400,26 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set, pipapo_map(m, rulemap, e); + m->state = NFT_PIPAPO_CLONE_MOD; return 0; +abort: + DEBUG_NET_WARN_ON_ONCE(m->state == NFT_PIPAPO_CLONE_ERR); + + /* Two rollback cases: + * 1) no previous changes. nft_pipapo_abort is not + * guaranteed to be invoked (there might be no further + * add/delete requests coming after this). + * + * 2) we had previous changes: there are transaction + * records pointing to this set. Leave the rollback to + * the transaction handling. + */ + if (m->state == NFT_PIPAPO_CLONE_NEW) + nft_pipapo_abort(set); /* releases m */ + else + m->state = NFT_PIPAPO_CLONE_ERR; + + return err; } /** @@ -1473,6 +1496,7 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old) dst++; } + new->state = NFT_PIPAPO_CLONE_NEW; return new; out_mt: @@ -1896,7 +1920,7 @@ nft_pipapo_deactivate(const struct net *net, const struct nft_set *set, /* removal must occur on priv->clone, if we are low on memory * we have no choice and must fail the removal request. */ - if (!m) + if (!m || m->state == NFT_PIPAPO_CLONE_ERR) return NULL; e = pipapo_get(m, (const u8 *)elem->key.val.data, diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index b82abb03576e..a19e980d06ef 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -131,9 +131,16 @@ struct nft_pipapo_scratch { unsigned long __map[]; }; +enum nft_pipapo_clone_state { + NFT_PIPAPO_CLONE_NEW, + NFT_PIPAPO_CLONE_MOD, + NFT_PIPAPO_CLONE_ERR, +}; + /** * struct nft_pipapo_match - Data used for lookup and matching * @field_count: Amount of fields in set + * @state: add/delete state; used from control plane * @bsize_max: Maximum lookup table bucket size of all fields, in longs * @scratch: Preallocated per-CPU maps for partial matching results * @rcu: Matching data is swapped on commits @@ -141,6 +148,7 @@ struct nft_pipapo_scratch { */ struct nft_pipapo_match { u8 field_count; + enum nft_pipapo_clone_state state:8; unsigned int bsize_max; struct nft_pipapo_scratch * __percpu *scratch; struct rcu_head rcu; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 018bbb6df4ce..9894832281c4 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -184,10 +184,14 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, if (!interval || nft_set_elem_expired(interval->from)) return ERR_PTR(-ENOENT); - if (flags & NFT_SET_ELEM_INTERVAL_END) + if (flags & NFT_SET_ELEM_INTERVAL_END) { + if (!interval->to) + return NULL; + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); - else + } else { rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + } return &rbe->priv; } @@ -544,8 +548,7 @@ static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals) if (!intervals) return -ENOMEM; - if (array->intervals) - kvfree(array->intervals); + kvfree(array->intervals); array->intervals = intervals; array->max_intervals = max_intervals; diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 7641f249614c..9ed288c9d168 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -24,14 +24,13 @@ static const struct nla_policy nft_synproxy_policy[NFTA_SYNPROXY_MAX + 1] = { static void nft_synproxy_tcp_options(struct synproxy_options *opts, const struct tcphdr *tcp, struct synproxy_net *snet, - struct nf_synproxy_info *info, - const struct nft_synproxy *priv) + struct nf_synproxy_info *info) { this_cpu_inc(snet->stats->syn_received); if (tcp->ece && tcp->cwr) opts->options |= NF_SYNPROXY_OPT_ECN; - opts->options &= priv->info.options; + opts->options &= info->options; opts->mss_encode = opts->mss_option; opts->mss_option = info->mss; if (opts->options & NF_SYNPROXY_OPT_TIMESTAMP) @@ -56,7 +55,7 @@ static void nft_synproxy_eval_v4(const struct nft_synproxy *priv, if (tcp->syn) { /* Initial SYN from client */ - nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); + nft_synproxy_tcp_options(opts, tcp, snet, &info); synproxy_send_client_synack(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; @@ -87,7 +86,7 @@ static void nft_synproxy_eval_v6(const struct nft_synproxy *priv, if (tcp->syn) { /* Initial SYN from client */ - nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); + nft_synproxy_tcp_options(opts, tcp, snet, &info); synproxy_send_client_synack_ipv6(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 4e6708c23922..e64116bf2637 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1920,9 +1920,9 @@ static const struct seq_operations xt_target_seq_ops = { .show = xt_target_seq_show, }; -#define FORMAT_TABLES "_tables_names" -#define FORMAT_MATCHES "_tables_matches" -#define FORMAT_TARGETS "_tables_targets" +#define FORMAT_TABLES "%s_tables_names" +#define FORMAT_MATCHES "%s_tables_matches" +#define FORMAT_TARGETS "%s_tables_targets" #endif /* CONFIG_PROC_FS */ @@ -2033,8 +2033,7 @@ int xt_proto_init(struct net *net, u_int8_t af) root_uid = make_kuid(net->user_ns, 0); root_gid = make_kgid(net->user_ns, 0); - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_TABLES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]); proc = proc_create_net_data(buf, 0440, net->proc_net, &xt_table_seq_ops, sizeof(struct seq_net_private), (void *)(unsigned long)af); @@ -2043,8 +2042,7 @@ int xt_proto_init(struct net *net, u_int8_t af) if (uid_valid(root_uid) && gid_valid(root_gid)) proc_set_user(proc, root_uid, root_gid); - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_MATCHES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]); proc = proc_create_seq_private(buf, 0440, net->proc_net, &xt_match_seq_ops, sizeof(struct nf_mttg_trav), (void *)(unsigned long)af); @@ -2053,8 +2051,7 @@ int xt_proto_init(struct net *net, u_int8_t af) if (uid_valid(root_uid) && gid_valid(root_gid)) proc_set_user(proc, root_uid, root_gid); - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_TARGETS, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_TARGETS, xt_prefix[af]); proc = proc_create_seq_private(buf, 0440, net->proc_net, &xt_target_seq_ops, sizeof(struct nf_mttg_trav), (void *)(unsigned long)af); @@ -2068,13 +2065,11 @@ int xt_proto_init(struct net *net, u_int8_t af) #ifdef CONFIG_PROC_FS out_remove_matches: - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_MATCHES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]); remove_proc_entry(buf, net->proc_net); out_remove_tables: - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_TABLES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]); remove_proc_entry(buf, net->proc_net); out: return -1; @@ -2087,16 +2082,13 @@ void xt_proto_fini(struct net *net, u_int8_t af) #ifdef CONFIG_PROC_FS char buf[XT_FUNCTION_MAXNAMELEN]; - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_TABLES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]); remove_proc_entry(buf, net->proc_net); - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_TARGETS, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_TARGETS, xt_prefix[af]); remove_proc_entry(buf, net->proc_net); - strscpy(buf, xt_prefix[af], sizeof(buf)); - strlcat(buf, FORMAT_MATCHES, sizeof(buf)); + snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]); remove_proc_entry(buf, net->proc_net); #endif /*CONFIG_PROC_FS*/ } diff --git a/net/netfilter/xt_TCPOPTSTRIP.c b/net/netfilter/xt_TCPOPTSTRIP.c index 93f064306901..265d21697847 100644 --- a/net/netfilter/xt_TCPOPTSTRIP.c +++ b/net/netfilter/xt_TCPOPTSTRIP.c @@ -16,7 +16,7 @@ #include #include -static inline unsigned int optlen(const u_int8_t *opt, unsigned int offset) +static inline unsigned int optlen(const u8 *opt, unsigned int offset) { /* Beware zero-length options: make finite progress */ if (opt[offset] <= TCPOPT_NOP || opt[offset+1] == 0) @@ -33,8 +33,8 @@ tcpoptstrip_mangle_packet(struct sk_buff *skb, const struct xt_tcpoptstrip_target_info *info = par->targinfo; struct tcphdr *tcph, _th; unsigned int optl, i, j; - u_int16_t n, o; - u_int8_t *opt; + u16 n, o; + u8 *opt; int tcp_hdrlen; /* This is a fragment, no TCP header is available */ @@ -97,7 +97,7 @@ tcpoptstrip_tg6(struct sk_buff *skb, const struct xt_action_param *par) { struct ipv6hdr *ipv6h = ipv6_hdr(skb); int tcphoff; - u_int8_t nexthdr; + u8 nexthdr; __be16 frag_off; nexthdr = ipv6h->nexthdr; diff --git a/net/netfilter/xt_cluster.c b/net/netfilter/xt_cluster.c index 908fd5f2c3c8..eaf2511d63f0 100644 --- a/net/netfilter/xt_cluster.c +++ b/net/netfilter/xt_cluster.c @@ -107,7 +107,7 @@ xt_cluster_mt(const struct sk_buff *skb, struct xt_action_param *par) } ct = nf_ct_get(skb, &ctinfo); - if (ct == NULL) + if (!ct || nf_ct_is_template(ct)) return false; if (ct->master) diff --git a/net/netfilter/xt_connmark.c b/net/netfilter/xt_connmark.c index 4277084de2e7..2cf27f7d59b9 100644 --- a/net/netfilter/xt_connmark.c +++ b/net/netfilter/xt_connmark.c @@ -112,6 +112,16 @@ static int connmark_tg_check(const struct xt_tgchk_param *par) return ret; } +static int connmark_tg_check_v2(const struct xt_tgchk_param *par) +{ + const struct xt_connmark_tginfo2 *info = par->targinfo; + + if (info->shift_dir > D_SHIFT_RIGHT || info->shift_bits >= 32) + return -EINVAL; + + return connmark_tg_check(par); +} + static void connmark_tg_destroy(const struct xt_tgdtor_param *par) { nf_ct_netns_put(par->net, par->family); @@ -162,7 +172,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV4, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, @@ -183,7 +193,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV6, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, diff --git a/net/netfilter/xt_dscp.c b/net/netfilter/xt_dscp.c index fb0169a8f9bb..878f27016e99 100644 --- a/net/netfilter/xt_dscp.c +++ b/net/netfilter/xt_dscp.c @@ -49,6 +49,16 @@ static int dscp_mt_check(const struct xt_mtchk_param *par) return 0; } +static int tos_mt_check(const struct xt_mtchk_param *par) +{ + const struct xt_tos_match_info *info = par->matchinfo; + + if (info->invert > 1) + return -EINVAL; + + return 0; +} + static bool tos_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_tos_match_info *info = par->matchinfo; @@ -82,6 +92,7 @@ static struct xt_match dscp_mt_reg[] __read_mostly = { .name = "tos", .revision = 1, .family = NFPROTO_IPV4, + .checkentry = tos_mt_check, .match = tos_mt, .matchsize = sizeof(struct xt_tos_match_info), .me = THIS_MODULE, @@ -90,6 +101,7 @@ static struct xt_match dscp_mt_reg[] __read_mostly = { .name = "tos", .revision = 1, .family = NFPROTO_IPV6, + .checkentry = tos_mt_check, .match = tos_mt, .matchsize = sizeof(struct xt_tos_match_info), .me = THIS_MODULE, diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c index b1d736c15fcb..7c05b6342578 100644 --- a/net/netfilter/xt_rateest.c +++ b/net/netfilter/xt_rateest.c @@ -16,7 +16,7 @@ xt_rateest_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_rateest_match_info *info = par->matchinfo; struct gnet_stats_rate_est64 sample = {0}; - u_int32_t bps1, bps2, pps1, pps2; + u64 bps1, bps2, pps1, pps2; bool ret = true; gen_estimator_read(&info->est1->rate_est, &sample); diff --git a/net/netfilter/xt_recent.c b/net/netfilter/xt_recent.c index f72752fa4374..d34831ce3adf 100644 --- a/net/netfilter/xt_recent.c +++ b/net/netfilter/xt_recent.c @@ -400,7 +400,7 @@ static int recent_mt_check(const struct xt_mtchk_param *par, t->nstamps_max_mask = nstamp_mask; memcpy(&t->mask, &info->mask, sizeof(t->mask)); - strcpy(t->name, info->name); + strscpy(t->name, info->name); INIT_LIST_HEAD(&t->lru_list); for (i = 0; i < ip_list_hash_size; i++) INIT_LIST_HEAD(&t->iphash[i]); diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c index b9da8269161d..b08b077d7f0a 100644 --- a/net/netfilter/xt_tcpmss.c +++ b/net/netfilter/xt_tcpmss.c @@ -78,10 +78,23 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par) return false; } +static int tcpmss_mt_check(const struct xt_mtchk_param *par) +{ + const struct xt_tcpmss_match_info *info = par->matchinfo; + + if (info->mss_min > info->mss_max) + return -EINVAL; + if (info->invert > 1) + return -EINVAL; + + return 0; +} + static struct xt_match tcpmss_mt_reg[] __read_mostly = { { .name = "tcpmss", .family = NFPROTO_IPV4, + .checkentry = tcpmss_mt_check, .match = tcpmss_mt, .matchsize = sizeof(struct xt_tcpmss_match_info), .proto = IPPROTO_TCP, diff --git a/net/netfilter/xt_u32.c b/net/netfilter/xt_u32.c index 117d4615d668..ec1a21e3b6e2 100644 --- a/net/netfilter/xt_u32.c +++ b/net/netfilter/xt_u32.c @@ -100,7 +100,7 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) { const struct xt_u32 *data = par->matchinfo; const struct xt_u32_test *ct; - unsigned int i; + unsigned int i, j; if (data->ntests > ARRAY_SIZE(data->tests)) return -EINVAL; @@ -111,6 +111,16 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) if (ct->nnums > ARRAY_SIZE(ct->location) || ct->nvalues > ARRAY_SIZE(ct->value)) return -EINVAL; + + for (j = 1; j < ct->nnums; ++j) { + switch (ct->location[j].nextop) { + case XT_U32_LEFTSH: + case XT_U32_RIGHTSH: + if (ct->location[j].number >= 32) + return -EINVAL; + break; + } + } } return 0; diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index ae63c5eb06fa..ae6487c10a25 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -778,6 +778,8 @@ static void digital_in_recv_sensf_res(struct nfc_digital_dev *ddev, void *arg, sensf_res = (struct digital_sensf_res *)resp->data; + resp->len = min_t(unsigned int, resp->len, NFC_SENSF_RES_MAXSIZE); + memcpy(target.sensf_res, sensf_res, resp->len); target.sensf_res_len = resp->len; diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index 291f26facbf3..ca89fe967d6a 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -193,7 +193,8 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -201,9 +202,15 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, return -ENODEV; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { @@ -243,7 +250,8 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -251,9 +259,15 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, return -ENOTCONN; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index dc65c719f35f..aed5fe1afef0 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -1286,10 +1286,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, { struct nfc_llcp_sock *llcp_sock; u8 dsap, ssap, type, length, tid, sap; - const u8 *tlv; - u16 tlv_len, offset; + const u8 *tlv, *tlv_end; const char *service_name; - size_t service_name_len; + int service_name_len; struct nfc_llcp_sdp_tlv *sdp; HLIST_HEAD(llc_sdres_list); size_t sdres_tlvs_len; @@ -1305,22 +1304,34 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, return; } + /* + * Walk the SNL TLV list in the linear part of the skb only, + * bounded by skb_tail_pointer(). Each TLV needs a two-byte + * header (type, length) and its declared length must fit before + * the end; this also keeps the walk safe for very short frames. + */ tlv = &skb->data[LLCP_HEADER_SIZE]; - tlv_len = skb->len - LLCP_HEADER_SIZE; - offset = 0; + tlv_end = skb_tail_pointer(skb); sdres_tlvs_len = 0; - while (offset < tlv_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + switch (type) { case LLCP_TLV_SDREQ: + if (length < 1) + break; + tid = tlv[2]; service_name = (char *) &tlv[3]; service_name_len = length - 1; - pr_debug("Looking for %.16s\n", service_name); + pr_debug("Looking for %.*s\n", service_name_len, + service_name); if (service_name_len == strlen("urn:nfc:sn:sdp") && !strncmp(service_name, "urn:nfc:sn:sdp", @@ -1380,6 +1391,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; case LLCP_TLV_SDRES: + if (length != 2) + break; + mutex_lock(&local->sdreq_lock); pr_debug("LLCP_TLV_SDRES: searching tid %d\n", tlv[2]); @@ -1408,7 +1422,6 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; } - offset += length + 2; tlv += length + 2; } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index feab29fc62f4..5558d8a4d48b 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -319,14 +319,22 @@ static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname, if (get_user(len, optlen)) return -EFAULT; - local = llcp_sock->local; - if (!local) - return -ENODEV; + if (len < 0) + return -EINVAL; + + if (len < sizeof(u32)) + return -EINVAL; len = min_t(u32, len, sizeof(u32)); lock_sock(sk); + local = llcp_sock->local; + if (!local) { + release_sock(sk); + return -ENODEV; + } + switch (optname) { case NFC_LLCP_RW: rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw; diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 5f98c73db5af..4253edea5c8d 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -46,11 +46,11 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); - /* Mark the exchange as done before calling the callback. - * The callback (e.g. rawsock_data_exchange_complete) may - * want to immediately queue another data exchange. - */ - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + /* Claim completion atomically -- both close and rx_work may race here */ + if (!test_and_clear_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); + return; + } if (cb) { /* forward skb to nfc core */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index c96512bb8653..87f76a29f0ec 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -525,15 +525,19 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_nfca_poll_iso_dep *nfca_poll; struct activation_params_nfcb_poll_iso_dep *nfcb_poll; switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfca_poll = &ntf->activation_params.nfca_poll_iso_dep; nfca_poll->rats_res_len = min_t(__u8, *data++, NFC_ATS_MAXSIZE); + data_len--; + nfca_poll->rats_res_len = min_t(__u8, nfca_poll->rats_res_len, data_len); pr_debug("rats_res_len %d\n", nfca_poll->rats_res_len); if (nfca_poll->rats_res_len > 0) { memcpy(nfca_poll->rats_res, @@ -542,8 +546,12 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, break; case NCI_NFC_B_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfcb_poll = &ntf->activation_params.nfcb_poll_iso_dep; nfcb_poll->attrib_res_len = min_t(__u8, *data++, 50); + data_len--; + nfcb_poll->attrib_res_len = min_t(__u8, nfcb_poll->attrib_res_len, data_len); pr_debug("attrib_res_len %d\n", nfcb_poll->attrib_res_len); if (nfcb_poll->attrib_res_len > 0) { memcpy(nfcb_poll->attrib_res, @@ -562,7 +570,7 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_poll_nfc_dep *poll; struct activation_params_listen_nfc_dep *listen; @@ -570,9 +578,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: case NCI_NFC_F_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; poll = &ntf->activation_params.poll_nfc_dep; poll->atr_res_len = min_t(__u8, *data++, NFC_ATR_RES_MAXSIZE - 2); + data_len--; + poll->atr_res_len = min_t(__u8, poll->atr_res_len, data_len); pr_debug("atr_res_len %d\n", poll->atr_res_len); if (poll->atr_res_len > 0) memcpy(poll->atr_res, data, poll->atr_res_len); @@ -580,9 +592,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, case NCI_NFC_A_PASSIVE_LISTEN_MODE: case NCI_NFC_F_PASSIVE_LISTEN_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; listen = &ntf->activation_params.listen_nfc_dep; listen->atr_req_len = min_t(__u8, *data++, NFC_ATR_REQ_MAXSIZE - 2); + data_len--; + listen->atr_req_len = min_t(__u8, listen->atr_req_len, data_len); pr_debug("atr_req_len %d\n", listen->atr_req_len); if (listen->atr_req_len > 0) memcpy(listen->atr_req, data, listen->atr_req_len); @@ -603,6 +619,12 @@ static void nci_target_auto_activated(struct nci_dev *ndev, struct nfc_target *target; int rc; + /* This is a new target, check if we've enough room */ + if (ndev->n_targets == NCI_MAX_DISCOVERED_TARGETS) { + pr_debug("not enough room, ignoring new target...\n"); + return; + } + target = &ndev->targets[ndev->n_targets]; rc = nci_add_new_protocol(ndev, target, ntf->rf_protocol, @@ -806,12 +828,14 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: err = nci_extract_activation_params_iso_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_NFC_DEP: err = nci_extract_activation_params_nfc_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_FRAME: diff --git a/net/nfc/nci/rsp.c b/net/nfc/nci/rsp.c index 9eeb862825c5..dd6f16f49ceb 100644 --- a/net/nfc/nci/rsp.c +++ b/net/nfc/nci/rsp.c @@ -50,11 +50,26 @@ static u8 nci_core_init_rsp_packet_v1(struct nci_dev *ndev, const struct nci_core_init_rsp_1 *rsp_1 = (void *)skb->data; const struct nci_core_init_rsp_2 *rsp_2; + /* Ensure that the status field can be accessed. */ + if (skb->len < sizeof(*rsp_1)) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status 0x%x\n", rsp_1->status); if (rsp_1->status != NCI_STATUS_OK) return rsp_1->status; + /* + * supported_rf_interfaces[] and the trailing nci_core_init_rsp_2 are + * addressed using the on-wire (unclamped) interface count, so the + * response must be long enough for both before any of it is parsed or + * stored into @ndev - otherwise a truncated response would leave + * ndev->num_supported_rf_interfaces holding the unclamped count. + */ + if (skb->len < sizeof(*rsp_1) + + rsp_1->num_supported_rf_interfaces + sizeof(*rsp_2)) + return NCI_STATUS_SYNTAX_ERROR; + ndev->nfcc_features = __le32_to_cpu(rsp_1->nfcc_features); ndev->num_supported_rf_interfaces = rsp_1->num_supported_rf_interfaces; @@ -88,14 +103,24 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, { const struct nci_core_init_rsp_nci_ver2 *rsp = (void *)skb->data; const u8 *supported_rf_interface = rsp->supported_rf_interfaces; + const u8 *end = skb->data + skb->len; u8 rf_interface_idx = 0; u8 rf_extension_cnt = 0; + if (skb->len < sizeof(*rsp)) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status %x\n", rsp->status); if (rsp->status != NCI_STATUS_OK) return rsp->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp)) + return NCI_STATUS_SYNTAX_ERROR; + + supported_rf_interface = rsp->supported_rf_interfaces; + ndev->nfcc_features = __le32_to_cpu(rsp->nfcc_features); ndev->num_supported_rf_interfaces = rsp->num_supported_rf_interfaces; @@ -104,13 +129,21 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, NCI_MAX_SUPPORTED_RF_INTERFACES); while (rf_interface_idx < ndev->num_supported_rf_interfaces) { - ndev->supported_rf_interfaces[rf_interface_idx++] = *supported_rf_interface++; + /* one interface byte + one extension-count byte must be present */ + if (end - supported_rf_interface < 2) + return NCI_STATUS_SYNTAX_ERROR; + ndev->supported_rf_interfaces[rf_interface_idx++] = + *supported_rf_interface++; - /* skip rf extension parameters */ + /* skip rf extension parameters, bounded by the packet */ rf_extension_cnt = *supported_rf_interface++; + if (rf_extension_cnt > end - supported_rf_interface) + return NCI_STATUS_SYNTAX_ERROR; supported_rf_interface += rf_extension_cnt; } + ndev->num_supported_rf_interfaces = rf_interface_idx; + ndev->max_logical_connections = rsp->max_logical_connections; ndev->max_routing_table_size = __le16_to_cpu(rsp->max_routing_table_size); diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index c6fd9c424e8f..95697d4e16e6 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -883,7 +883,8 @@ static void ct_limit_set(const struct ovs_ct_limit_info *info, struct hlist_head *head; head = ct_limit_hash_bucket(info, new_ct_limit->zone); - hlist_for_each_entry_rcu(ct_limit, head, hlist_node) { + hlist_for_each_entry_rcu(ct_limit, head, hlist_node, + lockdep_ovsl_is_held()) { if (ct_limit->zone == new_ct_limit->zone) { hlist_replace_rcu(&ct_limit->hlist_node, &new_ct_limit->hlist_node); diff --git a/net/psample/psample.c b/net/psample/psample.c index 7763662036fb..c112e1f0ccac 100644 --- a/net/psample/psample.c +++ b/net/psample/psample.c @@ -476,15 +476,17 @@ void psample_sample_packet(struct psample_group *group, goto error; if (data_len) { - int nla_len = nla_total_size(data_len); + int nla_len = nla_attr_size(data_len); struct nlattr *nla; nla = skb_put(nl_skb, nla_len); nla->nla_type = PSAMPLE_ATTR_DATA; - nla->nla_len = nla_attr_size(data_len); + nla->nla_len = nla_len; if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) goto error; + + skb_put_zero(nl_skb, nla_padlen(data_len)); } #ifdef CONFIG_INET diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index 80e341d2f8a4..3990da1a65dc 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include diff --git a/net/rds/send.c b/net/rds/send.c index e5d58c29aabe..68be1bf0e0ad 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -967,6 +967,8 @@ static int rds_rm_size(struct msghdr *msg, int num_sgs, switch (cmsg->cmsg_type) { case RDS_CMSG_RDMA_ARGS: + if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_rdma_args))) + return -EINVAL; if (vct->indx >= vct->len) { vct->len += vct->incr; tmp_iov = diff --git a/net/rfkill/rfkill-gpio.c b/net/rfkill/rfkill-gpio.c index cf2dcec6ce5a..9c5b695dba7c 100644 --- a/net/rfkill/rfkill-gpio.c +++ b/net/rfkill/rfkill-gpio.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 5802f6f78723..ce946b0a03e2 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -669,7 +669,9 @@ enum rxrpc_call_event { enum rxrpc_call_state { RXRPC_CALL_UNINITIALISED, RXRPC_CALL_CLIENT_AWAIT_CONN, /* - client waiting for connection to become available */ + RXRPC_CALL_CLIENT_PRE_SEND, /* - client is connected, but hasn't sent anything yet */ RXRPC_CALL_CLIENT_SEND_REQUEST, /* - client sending request phase */ + RXRPC_CALL_CLIENT_AWAIT_ACK, /* - client awaiting ACKs of request */ RXRPC_CALL_CLIENT_AWAIT_REPLY, /* - client awaiting reply */ RXRPC_CALL_CLIENT_RECV_REPLY, /* - client receiving reply phase */ RXRPC_CALL_SERVER_PREALLOC, /* - service preallocation */ @@ -1374,9 +1376,9 @@ static inline struct rxrpc_net *rxrpc_net(struct net *net) } /* - * out_of_band.c + * oob.c */ -void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb); +bool rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb); void rxrpc_add_pending_oob(struct rxrpc_sock *rx, struct sk_buff *skb); int rxrpc_sendmsg_oob(struct rxrpc_sock *rx, struct msghdr *msg, size_t len); diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index fec59d9338b9..21be9c86d7a7 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -178,7 +178,7 @@ static void rxrpc_close_tx_phase(struct rxrpc_call *call) switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: - rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY); + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_ACK); break; case RXRPC_CALL_SERVER_SEND_REPLY: rxrpc_set_call_state(call, RXRPC_CALL_SERVER_AWAIT_ACK); @@ -244,6 +244,8 @@ static void rxrpc_transmit_fresh_data(struct rxrpc_call *call, unsigned int limi break; } while (req.n < limit && before(seq, send_top)); + if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_PRE_SEND) + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_SEND_REQUEST); if (txb->flags & RXRPC_LAST_PACKET) { rxrpc_close_tx_phase(call); tq = NULL; @@ -267,6 +269,7 @@ void rxrpc_transmit_some_data(struct rxrpc_call *call, unsigned int limit, fallthrough; case RXRPC_CALL_SERVER_SEND_REPLY: + case RXRPC_CALL_CLIENT_PRE_SEND: case RXRPC_CALL_CLIENT_SEND_REQUEST: if (!rxrpc_tx_window_space(call)) return; diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index fcb9d38bb521..817ed9acb91e 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -18,7 +18,9 @@ const char *const rxrpc_call_states[NR__RXRPC_CALL_STATES] = { [RXRPC_CALL_UNINITIALISED] = "Uninit ", [RXRPC_CALL_CLIENT_AWAIT_CONN] = "ClWtConn", + [RXRPC_CALL_CLIENT_PRE_SEND] = "ClPreSnd", [RXRPC_CALL_CLIENT_SEND_REQUEST] = "ClSndReq", + [RXRPC_CALL_CLIENT_AWAIT_ACK] = "ClAwtAck", [RXRPC_CALL_CLIENT_AWAIT_REPLY] = "ClAwtRpl", [RXRPC_CALL_CLIENT_RECV_REPLY] = "ClRcvRpl", [RXRPC_CALL_SERVER_PREALLOC] = "SvPrealc", diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c index 9b757798dedd..48519f0de185 100644 --- a/net/rxrpc/conn_client.c +++ b/net/rxrpc/conn_client.c @@ -449,7 +449,7 @@ static void rxrpc_activate_one_channel(struct rxrpc_connection *conn, trace_rxrpc_connect_call(call); call->tx_last_sent = ktime_get_real(); rxrpc_start_call_timer(call); - rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_SEND_REQUEST); + rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_PRE_SEND); wake_up(&call->waitq); } diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index c96ca615b787..611c790bc6d0 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -436,7 +436,7 @@ static bool rxrpc_post_challenge(struct rxrpc_connection *conn, struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_call *call = NULL; struct rxrpc_sock *rx; - bool respond = false; + bool respond = false, queued = false; sp->chall.conn = rxrpc_get_connection(conn, rxrpc_conn_get_challenge_input); @@ -472,8 +472,13 @@ static bool rxrpc_post_challenge(struct rxrpc_connection *conn, } if (call) - rxrpc_notify_socket_oob(call, skb); + queued = rxrpc_notify_socket_oob(call, skb); rcu_read_unlock(); + if (call && !queued) { + rxrpc_put_connection(conn, rxrpc_conn_put_challenge_input); + sp->chall.conn = NULL; + return false; + } if (!call) rxrpc_post_packet_to_conn(conn, skb); diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c index ce761466b02d..73cafe6bfa9f 100644 --- a/net/rxrpc/input.c +++ b/net/rxrpc/input.c @@ -181,7 +181,8 @@ void rxrpc_congestion_degrade(struct rxrpc_call *call) if (call->cong_ca_state != RXRPC_CA_SLOW_START && call->cong_ca_state != RXRPC_CA_CONGEST_AVOIDANCE) return; - if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY) + if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_ACK || + __rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY) return; rtt = ns_to_ktime(call->srtt_us * (NSEC_PER_USEC / 8)); @@ -236,6 +237,9 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, call->acks_lowest_nak = to; } + if (after(seq, to)) + return false; + /* We may have a left over fully-consumed buffer at the front that we * couldn't drop before (rotate_and_keep below). */ @@ -247,7 +251,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, tq = call->tx_queue; } - do { + while (before_eq(seq, to)) { unsigned int ix = seq - call->tx_qbase; _debug("tq=%x seq=%x i=%d f=%x", tq->qbase, seq, ix, tq->bufs[ix]->flags); @@ -317,8 +321,7 @@ static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to, break; } } - - } while (before_eq(seq, to)); + } if (trace) trace_rxrpc_rack_update(call, summary); @@ -356,6 +359,7 @@ static void rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun, switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_CLIENT_AWAIT_REPLY: if (reply_begun) { rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_RECV_REPLY); @@ -392,6 +396,14 @@ static bool rxrpc_receiving_reply(struct rxrpc_call *call) trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack); } + /* Deal with an apparent reply coming in before we've got the request + * queued or transmitted. + */ + if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) { + rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply); + return false; + } + if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) { if (!rxrpc_rotate_tx_window(call, top, &summary)) { rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply); @@ -694,6 +706,7 @@ static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb) switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_CLIENT_AWAIT_REPLY: /* Received data implicitly ACKs all of the request * packets we sent when we're acting as a client. @@ -1154,10 +1167,12 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb) if (hard_ack + 1 == 0) return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_zero); - /* Ignore ACKs unless we are or have just been transmitting. */ + /* Ignore ACKs unless we are transmitting or are waiting for + * acknowledgement of the packets we've just been transmitting. + */ switch (__rxrpc_call_state(call)) { case RXRPC_CALL_CLIENT_SEND_REQUEST: - case RXRPC_CALL_CLIENT_AWAIT_REPLY: + case RXRPC_CALL_CLIENT_AWAIT_ACK: case RXRPC_CALL_SERVER_SEND_REPLY: case RXRPC_CALL_SERVER_AWAIT_ACK: break; @@ -1215,7 +1230,17 @@ static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_ack_summary summary = { 0 }; - if (rxrpc_rotate_tx_window(call, call->tx_top, &summary)) + switch (__rxrpc_call_state(call)) { + case RXRPC_CALL_CLIENT_SEND_REQUEST: + case RXRPC_CALL_CLIENT_AWAIT_ACK: + case RXRPC_CALL_SERVER_SEND_REPLY: + case RXRPC_CALL_SERVER_AWAIT_ACK: + break; + default: + return; + } + + if (rxrpc_rotate_tx_window(call, call->tx_transmitted, &summary)) rxrpc_end_tx_phase(call, false, rxrpc_eproto_unexpected_ackall); } diff --git a/net/rxrpc/oob.c b/net/rxrpc/oob.c index 05ca9c1faa57..c80ee2487d09 100644 --- a/net/rxrpc/oob.c +++ b/net/rxrpc/oob.c @@ -32,11 +32,12 @@ struct rxrpc_oob_params { * Post an out-of-band message for attention by the socket or kernel service * associated with a reference call. */ -void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) +bool rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) { struct rxrpc_skb_priv *sp = rxrpc_skb(skb); struct rxrpc_sock *rx; struct sock *sk; + bool queued = false; rcu_read_lock(); @@ -49,6 +50,7 @@ void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) skb->skb_mstamp_ns = rx->oob_id_counter++; rxrpc_get_skb(skb, rxrpc_skb_get_post_oob); skb_queue_tail(&rx->recvmsg_oobq, skb); + queued = true; trace_rxrpc_notify_socket(call->debug_id, sp->hdr.serial); if (rx->app_ops) @@ -56,11 +58,12 @@ void rxrpc_notify_socket_oob(struct rxrpc_call *call, struct sk_buff *skb) } spin_unlock_irq(&rx->recvmsg_lock); - if (!rx->app_ops && !sock_flag(sk, SOCK_DEAD)) + if (queued && !rx->app_ops && !sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); } rcu_read_unlock(); + return queued; } /* @@ -210,6 +213,11 @@ static int rxrpc_respond_to_oob(struct rxrpc_sock *rx, break; } + switch (skb->mark) { + case RXRPC_OOB_CHALLENGE: + rxrpc_put_connection(sp->chall.conn, rxrpc_conn_put_oob); + break; + } rxrpc_free_skb(skb, rxrpc_skb_put_oob); return ret; } diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index 82614cbdb60f..efcba4b2e74f 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -27,8 +27,6 @@ void rxrpc_notify_socket(struct rxrpc_call *call) _enter("%d", call->debug_id); - if (!list_empty(&call->recvmsg_link)) - return; if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { rxrpc_see_call(call, rxrpc_call_see_notify_released); return; @@ -438,7 +436,8 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, return -EAGAIN; } - if (list_empty(&rx->recvmsg_q)) { + if (list_empty(&rx->recvmsg_q) && + skb_queue_empty_lockless(&rx->recvmsg_oobq)) { ret = -EWOULDBLOCK; if (timeo == 0) { call = NULL; @@ -471,7 +470,7 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, release_sock(&rx->sk); if (ret == -EAGAIN) goto try_again; - goto error_no_call; + goto error_trace; } /* Find the next call and dequeue it if we're not just peeking. If we @@ -530,8 +529,7 @@ int rxrpc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, if (test_bit(RXRPC_CALL_RELEASED, &call->flags)) { rxrpc_see_call(call, rxrpc_call_see_already_released); mutex_unlock(&call->user_mutex); - if (!(flags & MSG_PEEK)) - rxrpc_put_call(call, rxrpc_call_put_recvmsg); + rxrpc_put_call(call, rxrpc_call_put_recvmsg); goto try_again; } diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index c35de4fd75e3..ed2c9a51005a 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -366,7 +366,8 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, if (state >= RXRPC_CALL_COMPLETE) goto maybe_error; ret = -EPROTO; - if (state != RXRPC_CALL_CLIENT_SEND_REQUEST && + if (state != RXRPC_CALL_CLIENT_PRE_SEND && + state != RXRPC_CALL_CLIENT_SEND_REQUEST && state != RXRPC_CALL_SERVER_ACK_REQUEST && state != RXRPC_CALL_SERVER_SEND_REPLY) { /* Request phase complete for this client call */ diff --git a/net/sched/act_api.c b/net/sched/act_api.c index b68be143a067..f141634df214 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -148,10 +148,15 @@ static void offload_action_hw_count_dec(struct tc_action *act, static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) { - if (is_tcf_pedit(act)) - return tcf_pedit_nkeys(act); - else - return 1; + unsigned int count; + + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + count = tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + return count; + } + return 1; } static bool tc_act_skip_hw(u32 flags) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 58a074651176..09d46e195e33 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -44,7 +44,7 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, tcf_lastuse_update(&prog->tcf_tm); bstats_update(this_cpu_ptr(prog->common.cpu_bstats), skb); - filter = rcu_dereference(prog->filter); + filter = rcu_dereference_bh(prog->filter); if (at_ingress) { __skb_push(skb, skb->mac_len); filter_res = bpf_prog_run_data_pointers(filter, skb); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 6158e13c98d3..4ca7964e83c8 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -844,11 +844,11 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, u8 family, u16 zone, bool *defrag) { enum ip_conntrack_info ctinfo; + struct tc_skb_cb cb; struct nf_conn *ct; int err = 0; bool frag; u8 proto; - u16 mru; /* Previously seen (loopback)? Ignore. */ ct = nf_ct_get(skb, &ctinfo); @@ -862,12 +862,13 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, if (err || !frag) return err; - err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &mru); + cb = *tc_skb_cb(skb); + err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &cb.mru); if (err) return err; *defrag = true; - tc_skb_cb(skb)->mru = mru; + *tc_skb_cb(skb) = cb; return 0; } @@ -1295,7 +1296,8 @@ static int tcf_ct_fill_params(struct net *net, if (tb[TCA_CT_ZONE]) { if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) { NL_SET_ERR_MSG_MOD(extack, "Conntrack zones isn't enabled."); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto err; } tcf_ct_set_key_val(tb, @@ -1308,7 +1310,8 @@ static int tcf_ct_fill_params(struct net *net, tmpl = nf_ct_tmpl_alloc(net, &zone, GFP_KERNEL); if (!tmpl) { NL_SET_ERR_MSG_MOD(extack, "Failed to allocate conntrack template"); - return -ENOMEM; + err = -ENOMEM; + goto err; } p->tmpl = tmpl; if (tb[TCA_CT_HELPER_NAME]) { @@ -1524,8 +1527,8 @@ static int tcf_ct_dump_helper(struct sk_buff *skb, return 0; if (nla_put_string(skb, TCA_CT_HELPER_NAME, helper->name) || - nla_put_u8(skb, TCA_CT_HELPER_FAMILY, helper->tuple.src.l3num) || - nla_put_u8(skb, TCA_CT_HELPER_PROTO, helper->tuple.dst.protonum)) + nla_put_u8(skb, TCA_CT_HELPER_FAMILY, helper->nfproto) || + nla_put_u8(skb, TCA_CT_HELPER_PROTO, helper->l4proto)) return -1; return 0; diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 0d652dea4a69..d4d47a9921f4 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -567,9 +567,18 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, { if (bind) { struct flow_action_entry *entry = entry_data; + int nkeys = tcf_pedit_nkeys_locked(act); int k; - for (k = 0; k < tcf_pedit_nkeys(act); k++) { + /* If the required keys exceed the remaining capacity return + * -ENOSPC to abort the offload and fallback to software. + */ + if (nkeys > *index_inc) { + NL_SET_ERR_MSG_MOD(extack, "Not enough space to offload all pedit keys"); + return -ENOSPC; + } + + for (k = 0; k < nkeys; k++) { switch (tcf_pedit_cmd(act, k)) { case TCA_PEDIT_KEY_EX_CMD_SET: entry->id = FLOW_ACTION_MANGLE; @@ -606,7 +615,7 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return -EOPNOTSUPP; } - for (k = 1; k < tcf_pedit_nkeys(act); k++) { + for (k = 1; k < tcf_pedit_nkeys_locked(act); k++) { if (cmd != tcf_pedit_cmd(act, k)) { NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); return -EOPNOTSUPP; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 20f7f9ee0b35..ffeea6db8337 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3886,12 +3886,21 @@ int tc_setup_action(struct flow_action *flow_action, entry = &flow_action->entries[j]; spin_lock_bh(&act->tcfa_lock); + + /* Abort the offload if we have exhausted the allocated capacity */ + if (j >= flow_action->num_entries) { + NL_SET_ERR_MSG_MOD(extack, "Flow action buffer overflow"); + err = -ENOSPC; + goto err_out_locked; + } + err = tcf_act_get_user_cookie(entry, act); if (err) goto err_out_locked; - index = 0; - err = tc_setup_offload_act(act, entry, &index, extack); + index = flow_action->num_entries - j; + err = tc_setup_offload_act(act, entry, &index, + extack); if (err) goto err_out_locked; @@ -3945,10 +3954,13 @@ unsigned int tcf_exts_num_actions(struct tcf_exts *exts) int i; tcf_exts_for_each_action(i, act, exts) { - if (is_tcf_pedit(act)) - num_acts += tcf_pedit_nkeys(act); - else + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + num_acts += tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + } else { num_acts++; + } } return num_acts; } @@ -4049,6 +4061,9 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru skb_do_redirect(skb); *ret = __NET_XMIT_STOLEN; return NULL; + case TC_ACT_CONSUMED: + *ret = __NET_XMIT_STOLEN; + return NULL; } return skb; diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index d7c3254ef800..27088760eff4 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -346,6 +346,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, struct tcf_proto *fl; int result; + cb->classified = DUALPI2_C_CLASSIC; + dualpi2_read_ect(skb); if (cb->ect & q->ecn_mask) { cb->classified = DUALPI2_C_L4S; @@ -359,10 +361,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, } fl = rcu_dereference_bh(q->tcf_filters); - if (!fl) { - cb->classified = DUALPI2_C_CLASSIC; + if (!fl) return NET_XMIT_SUCCESS; - } result = tcf_classify(skb, NULL, fl, &res, false); if (result >= 0) { @@ -461,7 +461,7 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (IS_ERR_OR_NULL(nskb)) return qdisc_drop(skb, sch, to_free); - cnt = 1; + cnt = 0; byte_len = 0; orig_len = qdisc_pkt_len(skb); skb_list_walk_safe(nskb, nskb, next) { @@ -488,16 +488,15 @@ static int dualpi2_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, byte_len += nskb->len; } } - if (cnt > 1) { + if (cnt > 0) { /* The caller will add the original skb stats to its * backlog, compensate this if any nskb is enqueued. */ - --cnt; - byte_len -= orig_len; + qdisc_tree_reduce_backlog(sch, 1 - cnt, + orig_len - byte_len); } - qdisc_tree_reduce_backlog(sch, -cnt, -byte_len); consume_skb(skb); - return err; + return cnt > 0 ? NET_XMIT_SUCCESS : err; } return dualpi2_enqueue_skb(skb, sch, to_free); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 3f1c510df850..ef2b4bf51564 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -594,9 +594,8 @@ void netdev_watchdog_up(struct net_device *dev) return; if (dev->watchdog_timeo <= 0) dev->watchdog_timeo = 5*HZ; - spin_lock_bh(&dev->tx_global_lock); - spin_lock(&dev->watchdog_lock); + spin_lock_bh(&dev->watchdog_lock); if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) { if (!dev->watchdog_ref_held) { @@ -605,9 +604,7 @@ void netdev_watchdog_up(struct net_device *dev) dev->watchdog_ref_held = true; } } - spin_unlock(&dev->watchdog_lock); - - spin_unlock_bh(&dev->tx_global_lock); + spin_unlock_bh(&dev->watchdog_lock); } EXPORT_SYMBOL_GPL(netdev_watchdog_up); diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 1e25b75daae2..d85cb0263b67 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -462,12 +462,39 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch) return skb; } +static void hhf_reset_classifier(struct hhf_sched_data *q) +{ + int i; + + if (!q->hh_flows) + return; + + for (i = 0; i < HH_FLOWS_CNT; i++) { + struct hh_flow_state *flow, *next; + struct list_head *head = &q->hh_flows[i]; + + list_for_each_entry_safe(flow, next, head, flowchain) { + list_del(&flow->flowchain); + kfree(flow); + } + } + WRITE_ONCE(q->hh_flows_current_cnt, 0); + + for (i = 0; i < HHF_ARRAYS_CNT; i++) { + if (q->hhf_valid_bits[i]) + bitmap_zero(q->hhf_valid_bits[i], HHF_ARRAYS_LEN); + } + q->hhf_arrays_reset_timestamp = hhf_time_stamp(); +} + static void hhf_reset(struct Qdisc *sch) { + struct hhf_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; while ((skb = hhf_dequeue(sch)) != NULL) rtnl_kfree_skbs(skb, skb); + hhf_reset_classifier(q); } static void hhf_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index 4e465d11e3d7..a467dd122369 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -103,7 +103,7 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch) if (!netif_xmit_stopped( netdev_get_tx_queue(qdisc_dev(sch), q->curband))) { qdisc = q->queues[q->curband]; - skb = qdisc->dequeue(qdisc); + skb = qdisc_dequeue_peeked(qdisc); if (skb) { qdisc_bstats_update(sch, skb); qdisc_qlen_dec(sch); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 558987d9b977..299234a5f0fe 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -749,7 +749,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq, return NULL; skip_peek_checks: - skb = child->ops->dequeue(child); + skb = qdisc_dequeue_peeked(child); if (unlikely(!skb)) return NULL; diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index e7bbc9e5174d..5c42a29a981c 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -52,7 +52,8 @@ struct teql_master { struct Qdisc_ops qops; struct net_device *dev; - struct Qdisc *slaves; + struct Qdisc __rcu *slaves; + spinlock_t slaves_lock; /* serializes writes to ->slaves */ struct list_head master_list; unsigned long tx_bytes; unsigned long tx_packets; @@ -61,7 +62,7 @@ struct teql_master { }; struct teql_sched_data { - struct Qdisc *next; + struct Qdisc __rcu *next; struct teql_master *m; struct sk_buff_head q; }; @@ -101,7 +102,9 @@ teql_dequeue(struct Qdisc *sch) if (skb == NULL) { struct net_device *m = qdisc_dev(q); if (m) { - dat->m->slaves = sch; + spin_lock_bh(&dat->m->slaves_lock); + rcu_assign_pointer(dat->m->slaves, sch); + spin_unlock_bh(&dat->m->slaves_lock); netif_wake_queue(m); } } else { @@ -132,34 +135,49 @@ teql_destroy(struct Qdisc *sch) struct Qdisc *q, *prev; struct teql_sched_data *dat = qdisc_priv(sch); struct teql_master *master = dat->m; + struct netdev_queue *txq = NULL; + bool reset_master_queue = false; if (!master) return; - prev = master->slaves; + spin_lock_bh(&master->slaves_lock); + prev = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); if (prev) { do { - q = NEXT_SLAVE(prev); - if (q == sch) { - NEXT_SLAVE(prev) = NEXT_SLAVE(q); - if (q == master->slaves) { - master->slaves = NEXT_SLAVE(q); - if (q == master->slaves) { - struct netdev_queue *txq; - - txq = netdev_get_tx_queue(master->dev, 0); - master->slaves = NULL; - - dev_reset_queue(master->dev, - txq, NULL); - } - } - skb_queue_purge(&dat->q); - break; + struct Qdisc *head, *next; + + q = rcu_dereference_protected(NEXT_SLAVE(prev), + lockdep_is_held(&master->slaves_lock)); + if (q != sch) { + prev = q; + continue; } - } while ((prev = q) != master->slaves); + next = rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock)); + rcu_assign_pointer(NEXT_SLAVE(prev), next); + + head = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); + if (q == head) { + rcu_assign_pointer(master->slaves, next); + if (q == next) { + txq = netdev_get_tx_queue(master->dev, 0); + rcu_assign_pointer(master->slaves, NULL); + reset_master_queue = true; + } + } + skb_queue_purge(&dat->q); + break; + } while (prev != rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock))); } + spin_unlock_bh(&master->slaves_lock); + + if (reset_master_queue) + dev_reset_queue(master->dev, txq, NULL); } static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, @@ -168,6 +186,7 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, struct net_device *dev = qdisc_dev(sch); struct teql_master *m = (struct teql_master *)sch->ops; struct teql_sched_data *q = qdisc_priv(sch); + struct Qdisc *first; if (dev->hard_header_len > m->dev->hard_header_len) return -EINVAL; @@ -184,7 +203,9 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, skb_queue_head_init(&q->q); - if (m->slaves) { + spin_lock_bh(&m->slaves_lock); + first = rcu_dereference_protected(m->slaves, lockdep_is_held(&m->slaves_lock)); + if (first) { if (m->dev->flags & IFF_UP) { if ((m->dev->flags & IFF_POINTOPOINT && !(dev->flags & IFF_POINTOPOINT)) || @@ -192,8 +213,10 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, !(dev->flags & IFF_BROADCAST)) || (m->dev->flags & IFF_MULTICAST && !(dev->flags & IFF_MULTICAST)) || - dev->mtu < m->dev->mtu) + dev->mtu < m->dev->mtu) { + spin_unlock_bh(&m->slaves_lock); return -EINVAL; + } } else { if (!(dev->flags&IFF_POINTOPOINT)) m->dev->flags &= ~IFF_POINTOPOINT; @@ -204,14 +227,17 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, if (dev->mtu < m->dev->mtu) m->dev->mtu = dev->mtu; } - q->next = NEXT_SLAVE(m->slaves); - NEXT_SLAVE(m->slaves) = sch; + rcu_assign_pointer(q->next, + rcu_dereference_protected(NEXT_SLAVE(first), + lockdep_is_held(&m->slaves_lock))); + rcu_assign_pointer(NEXT_SLAVE(first), sch); } else { - q->next = sch; - m->slaves = sch; + rcu_assign_pointer(q->next, sch); + rcu_assign_pointer(m->slaves, sch); m->dev->mtu = dev->mtu; m->dev->flags = (m->dev->flags&~FMASK)|(dev->flags&FMASK); } + spin_unlock_bh(&m->slaves_lock); return 0; } @@ -285,12 +311,14 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) int subq = skb_get_queue_mapping(skb); struct sk_buff *skb_res = NULL; - start = master->slaves; - restart: nores = 0; busy = 0; + rcu_read_lock(); + + start = rcu_dereference(master->slaves); + q = start; if (!q) goto drop; @@ -317,10 +345,17 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { __netif_tx_unlock(slave_txq); - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); netif_wake_queue(dev); master->tx_packets++; master->tx_bytes += length; + rcu_read_unlock(); return NETDEV_TX_OK; } __netif_tx_unlock(slave_txq); @@ -329,45 +364,56 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) busy = 1; break; case 1: - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); + rcu_read_unlock(); return NETDEV_TX_OK; default: nores = 1; break; } __skb_pull(skb, skb_network_offset(skb)); - } while ((q = NEXT_SLAVE(q)) != start); + } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); if (nores && skb_res == NULL) { skb_res = skb; + rcu_read_unlock(); goto restart; } if (busy) { netif_stop_queue(dev); + rcu_read_unlock(); return NETDEV_TX_BUSY; } master->tx_errors++; drop: master->tx_dropped++; + rcu_read_unlock(); dev_kfree_skb(skb); return NETDEV_TX_OK; } static int teql_master_open(struct net_device *dev) { - struct Qdisc *q; + struct Qdisc *q, *first; struct teql_master *m = netdev_priv(dev); int mtu = 0xFFFE; unsigned int flags = IFF_NOARP | IFF_MULTICAST; - if (m->slaves == NULL) + first = rtnl_dereference(m->slaves); + if (!first) return -EUNATCH; flags = FMASK; - q = m->slaves; + q = first; do { struct net_device *slave = qdisc_dev(q); @@ -389,7 +435,7 @@ static int teql_master_open(struct net_device *dev) flags &= ~IFF_BROADCAST; if (!(slave->flags&IFF_MULTICAST)) flags &= ~IFF_MULTICAST; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); m->dev->mtu = mtu; m->dev->flags = (m->dev->flags&~FMASK) | flags; @@ -417,14 +463,15 @@ static void teql_master_stats64(struct net_device *dev, static int teql_master_mtu(struct net_device *dev, int new_mtu) { struct teql_master *m = netdev_priv(dev); - struct Qdisc *q; + struct Qdisc *q, *first; - q = m->slaves; + first = rtnl_dereference(m->slaves); + q = first; if (q) { do { if (new_mtu > qdisc_dev(q)->mtu) return -EINVAL; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); } WRITE_ONCE(dev->mtu, new_mtu); @@ -444,6 +491,7 @@ static __init void teql_master_setup(struct net_device *dev) struct teql_master *master = netdev_priv(dev); struct Qdisc_ops *ops = &master->qops; + spin_lock_init(&master->slaves_lock); master->dev = dev; ops->priv_size = sizeof(struct teql_sched_data); diff --git a/net/sctp/diag.c b/net/sctp/diag.c index d758f5c3e06e..c2a0de2adf6f 100644 --- a/net/sctp/diag.c +++ b/net/sctp/diag.c @@ -92,6 +92,7 @@ static int inet_diag_msg_sctpladdrs_fill(struct sk_buff *skb, if (!--addrcnt) break; } + WARN_ON_ONCE(addrcnt); rcu_read_unlock(); return 0; @@ -373,42 +374,39 @@ static int sctp_ep_dump(struct sctp_endpoint *ep, void *p) struct sk_buff *skb = commp->skb; struct netlink_callback *cb = commp->cb; const struct inet_diag_req_v2 *r = commp->r; - struct net *net = sock_net(skb->sk); struct inet_sock *inet = inet_sk(sk); int err = 0; - if (!net_eq(sock_net(sk), net)) + lock_sock(sk); + if (ep->base.dead) goto out; - if (cb->args[4] < cb->args[1]) - goto next; - - if (!(r->idiag_states & TCPF_LISTEN) && !list_empty(&ep->asocs)) - goto next; + /* Skip eps with assocs if non-LISTEN states were requested, since + * they'll be dumped by sctp_sock_dump() during assoc traversal. + */ + if ((r->idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE)) && + !list_empty(&ep->asocs)) + goto out; if (r->sdiag_family != AF_UNSPEC && sk->sk_family != r->sdiag_family) - goto next; + goto out; if (r->id.idiag_sport != inet->inet_sport && r->id.idiag_sport) - goto next; + goto out; if (r->id.idiag_dport != inet->inet_dport && r->id.idiag_dport) - goto next; - - if (inet_sctp_diag_fill(sk, NULL, skb, r, - sk_user_ns(NETLINK_CB(cb->skb).sk), - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, NLM_F_MULTI, - cb->nlh, commp->net_admin) < 0) { - err = 2; goto out; - } -next: - cb->args[4]++; + + err = inet_sctp_diag_fill(sk, NULL, skb, r, + sk_user_ns(NETLINK_CB(cb->skb).sk), + NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NLM_F_MULTI, + cb->nlh, commp->net_admin); out: + release_sock(sk); return err; } @@ -479,41 +477,40 @@ static void sctp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, .r = r, .net_admin = netlink_net_capable(cb->skb, CAP_NET_ADMIN), }; - int pos = cb->args[2]; + int pos; /* eps hashtable dumps * args: * 0 : if it will traversal listen sock * 1 : to record the sock pos of this time's traversal - * 4 : to work as a temporary variable to traversal list */ if (cb->args[0] == 0) { - if (!(idiag_states & TCPF_LISTEN)) - goto skip; - if (sctp_for_each_endpoint(sctp_ep_dump, &commp)) - goto done; -skip: + if (idiag_states & TCPF_LISTEN) { + pos = cb->args[1]; + if (sctp_for_each_endpoint(sctp_ep_dump, net, &pos, + &commp)) { + cb->args[1] = pos; + return; + } + } cb->args[0] = 1; cb->args[1] = 0; - cb->args[4] = 0; } + if (!(idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE))) + return; + /* asocs by transport hashtable dump * args: * 1 : to record the assoc pos of this time's traversal * 2 : to record the transport pos of this time's traversal * 3 : to mark if we have dumped the ep info of the current asoc - * 4 : to work as a temporary variable to traversal list - * 5 : to save the sk we get from travelsing the tsp list. + * 4 : to track position within ep->asocs list in sctp_sock_dump() */ - if (!(idiag_states & ~(TCPF_LISTEN | TCPF_CLOSE))) - goto done; - + pos = cb->args[2]; sctp_transport_traverse_process(sctp_sock_filter, sctp_sock_dump, net, &pos, &commp); cb->args[2] = pos; - -done: cb->args[1] = cb->args[4]; cb->args[4] = 0; } diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index 587b0017a67d..cf335494bffe 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -663,8 +663,9 @@ static void sctp_free_addr_wq(struct net *net) struct sctp_sockaddr_entry *addrw; struct sctp_sockaddr_entry *temp; + timer_shutdown_sync(&net->sctp.addr_wq_timer); + spin_lock_bh(&net->sctp.addr_wq_lock); - timer_delete(&net->sctp.addr_wq_timer); list_for_each_entry_safe(addrw, temp, &net->sctp.addr_waitq, list) { list_del(&addrw->list); kfree(addrw); diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 41958b8e59fd..8adac9e0cd66 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -1761,6 +1761,8 @@ struct sctp_association *sctp_unpack_cookie( bear_cookie = &cookie->c; ch = (struct sctp_chunkhdr *)(bear_cookie + 1); + if (ch->type != SCTP_CID_INIT) + goto malformed; chlen = ntohs(ch->length); if (chlen < sizeof(struct sctp_init_chunk)) goto malformed; @@ -2298,7 +2300,8 @@ int sctp_verify_init(struct net *net, const struct sctp_endpoint *ep, * VIOLATION error. We build the ERROR chunk here and let the normal * error handling code build and send the packet. */ - if (param.v != (void *)chunk->chunk_end) + if (param.v != (void *)peer_init + + SCTP_PAD4(ntohs(peer_init->chunk_hdr.length))) return sctp_process_inv_paramlength(asoc, param.p, chunk, errp); /* The only missing mandatory param possible today is diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 9b23c11cbb9e..d23d935e128e 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -415,6 +415,8 @@ enum sctp_disposition sctp_sf_do_5_1B_init(struct net *net, /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -705,11 +707,12 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, struct sctp_cmd_seq *commands) { struct sctp_ulpevent *ev, *ai_ev = NULL, *auth_ev = NULL; + struct sctp_chunk *err_chk_p = NULL; struct sctp_association *new_asoc; struct sctp_init_chunk *peer_init; struct sctp_chunk *chunk = arg; - struct sctp_chunk *err_chk_p; struct sctp_chunk *repl; + enum sctp_cid cid; struct sock *sk; int error = 0; @@ -783,6 +786,19 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, } } + peer_init = (struct sctp_init_chunk *)(chunk->subh.cookie_hdr + 1); + cid = peer_init->chunk_hdr.type; + if (!sctp_sk(sk)->cookie_auth_enable && + !sctp_verify_init(net, ep, asoc, cid, peer_init, chunk, + &err_chk_p)) { + sctp_association_free(new_asoc); + if (err_chk_p) + sctp_chunk_free(err_chk_p); + return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); + } + if (err_chk_p) + sctp_chunk_free(err_chk_p); + if (security_sctp_assoc_request(new_asoc, chunk->head_skb ?: chunk->skb)) { sctp_association_free(new_asoc); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); @@ -796,7 +812,6 @@ enum sctp_disposition sctp_sf_do_5_1D_ce(struct net *net, /* This is a brand-new association, so these are not yet side * effects--it is safe to run them here. */ - peer_init = (struct sctp_init_chunk *)(chunk->subh.cookie_hdr + 1); if (!sctp_process_init(new_asoc, chunk, &chunk->subh.cookie_hdr->c.peer_addr, peer_init, GFP_ATOMIC)) @@ -1606,6 +1621,8 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -1671,6 +1688,7 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( * parameter type. */ sctp_addto_chunk(repl, len, unk_param); + sctp_chunk_free(err_chunk); } sctp_add_cmd_sf(commands, SCTP_CMD_NEW_ASOC, SCTP_ASOC(new_asoc)); @@ -2210,10 +2228,12 @@ enum sctp_disposition sctp_sf_do_5_2_4_dupcook( void *arg, struct sctp_cmd_seq *commands) { + struct sctp_chunk *err_chk_p = NULL; struct sctp_association *new_asoc; + struct sctp_init_chunk *peer_init; struct sctp_chunk *chunk = arg; enum sctp_disposition retval; - struct sctp_chunk *err_chk_p; + enum sctp_cid cid; int error = 0; char action; @@ -2282,6 +2302,21 @@ enum sctp_disposition sctp_sf_do_5_2_4_dupcook( switch (action) { case 'A': /* Association restart. */ case 'B': /* Collision case B. */ + peer_init = (struct sctp_init_chunk *) + (chunk->subh.cookie_hdr + 1); + cid = peer_init->chunk_hdr.type; + if (!sctp_sk(ep->base.sk)->cookie_auth_enable && + !sctp_verify_init(net, ep, asoc, cid, peer_init, chunk, + &err_chk_p)) { + sctp_association_free(new_asoc); + if (err_chk_p) + sctp_chunk_free(err_chk_p); + return sctp_sf_pdiscard(net, ep, asoc, type, arg, + commands); + } + if (err_chk_p) + sctp_chunk_free(err_chk_p); + fallthrough; case 'D': /* Collision case D. */ /* Update socket peer label if first association. */ if (security_sctp_assoc_request((struct sctp_association *)asoc, diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 66e12fb0c646..c7b9e325ec1c 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4111,8 +4111,9 @@ static int sctp_setsockopt_reset_streams(struct sock *sk, if (optlen < sizeof(*params)) return -EINVAL; /* srs_number_streams is u16, so optlen can't be bigger than this. */ - optlen = min_t(unsigned int, optlen, USHRT_MAX + - sizeof(__u16) * sizeof(*params)); + optlen = min_t(unsigned int, optlen, + struct_size_t(struct sctp_reset_streams, srs_stream_list, + USHRT_MAX)); if (params->srs_number_streams * sizeof(__u16) > optlen - sizeof(*params)) @@ -4598,8 +4599,8 @@ static int sctp_setsockopt(struct sock *sk, int level, int optname, if (optlen > 0) { /* Trim it to the biggest size sctp sockopt may need if necessary */ optlen = min_t(unsigned int, optlen, - PAGE_ALIGN(USHRT_MAX + - sizeof(__u16) * sizeof(struct sctp_reset_streams))); + PAGE_ALIGN(struct_size_t(struct sctp_reset_streams, + srs_stream_list, USHRT_MAX))); kopt = memdup_sockptr(optval, optlen); if (IS_ERR(kopt)) return PTR_ERR(kopt); @@ -5369,24 +5370,39 @@ struct sctp_transport *sctp_transport_get_idx(struct net *net, } int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), - void *p) { - int err = 0; - int hash = 0; - struct sctp_endpoint *ep; + struct net *net, int *pos, void *p) { + int err, hash = 0, idx = 0, start; struct sctp_hashbucket *head; + struct sctp_endpoint *ep; for (head = sctp_ep_hashtable; hash < sctp_ep_hashsize; hash++, head++) { + start = idx; +again: read_lock_bh(&head->lock); sctp_for_each_hentry(ep, &head->chain) { - err = cb(ep, p); - if (err) + if (sock_net(ep->base.sk) != net) + continue; + if (idx++ >= *pos) { + sctp_endpoint_hold(ep); break; + } } read_unlock_bh(&head->lock); + + if (ep) { + err = cb(ep, p); + sctp_endpoint_put(ep); + if (err) + return err; + (*pos)++; + + idx = start; + goto again; + } } - return err; + return 0; } EXPORT_SYMBOL_GPL(sctp_for_each_endpoint); diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index 619b3bab3824..32d6d03df321 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -470,9 +470,9 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) { struct smc_link *link = (struct smc_link *)wc->qp->qp_context; struct smc_cdc_msg *cdc = buf; + struct smc_sock *smc = NULL; struct smc_connection *conn; struct smc_link_group *lgr; - struct smc_sock *smc; if (wc->byte_len < offsetof(struct smc_cdc_msg, reserved)) return; /* short message */ @@ -483,21 +483,26 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) lgr = smc_get_lgr(link); read_lock_bh(&lgr->conns_lock); conn = smc_lgr_find_conn(ntohl(cdc->token), lgr); - read_unlock_bh(&lgr->conns_lock); - if (!conn || conn->out_of_sync) + if (!conn || conn->out_of_sync) { + read_unlock_bh(&lgr->conns_lock); return; + } smc = container_of(conn, struct smc_sock, conn); + sock_hold(&smc->sk); + read_unlock_bh(&lgr->conns_lock); if (cdc->prod_flags.failover_validation) { smc_cdc_msg_validate(smc, cdc, link); - return; + goto out; } if (smc_cdc_before(ntohs(cdc->seqno), conn->local_rx_ctrl.seqno)) /* received seqno is old */ - return; + goto out; smc_cdc_msg_recv(smc, cdc); +out: + sock_put(&smc->sk); } static struct smc_wr_rx_handler smc_cdc_rx_handlers[] = { diff --git a/net/socket.c b/net/socket.c index 63c69a0fa74e..068cd8a7705b 100644 --- a/net/socket.c +++ b/net/socket.c @@ -465,6 +465,31 @@ static const struct xattr_handler sockfs_user_xattr_handler = { .set = sockfs_user_xattr_set, }; +/** + * sock_read_xattr - read a user.* xattr from a socket's sockfs inode + * @sock: socket whose inode holds the xattr + * @name: full xattr name, e.g. "user.bpf_test" + * @value: output buffer + * @size: size of @value in bytes + * + * SOCK_INODE() is valid only for sockfs sockets; sock_from_file() rejects + * anything else (e.g. tun, tap). + * Lockless: simple_xattr_get() looks up the value under RCU, no inode lock. + * + * Return: length of the value on success, a negative errno on error. + */ +int sock_read_xattr(struct socket *sock, const char *name, void *value, size_t size) +{ + struct file *file = sock->file; + struct sockfs_inode *si; + + if (!file || sock_from_file(file) != sock) + return -EOPNOTSUPP; + + si = SOCKFS_I(SOCK_INODE(sock)); + return simple_xattr_get(&sockfs_xa_cache, &si->xattrs, name, value, size); +} + static const struct xattr_handler * const sockfs_xattr_handlers[] = { &sockfs_xattr_handler, &sockfs_security_xattr_handler, @@ -698,7 +723,11 @@ struct socket *sock_alloc(void) } EXPORT_SYMBOL(sock_alloc); -static void __sock_release(struct socket *sock, struct inode *inode) +static +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +noinline +#endif +void __sock_release(struct socket *sock, struct inode *inode) { const struct proto_ops *ops = READ_ONCE(sock->ops); @@ -770,7 +799,13 @@ static noinline void call_trace_sock_send_length(struct sock *sk, int ret, trace_sock_send_length(sk, ret, 0); } -static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg) +static +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +noinline +#else +inline +#endif +int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg) { int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->sendmsg, inet6_sendmsg, inet_sendmsg, sock, msg, @@ -1120,8 +1155,13 @@ static noinline void call_trace_sock_recv_length(struct sock *sk, int ret, int f trace_sock_recv_length(sk, ret, flags); } -static inline int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg, - int flags) +static +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +noinline +#else +inline +#endif +int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg, int flags) { int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->recvmsg, inet6_recvmsg, @@ -2624,9 +2664,12 @@ static int copy_msghdr_from_user(struct msghdr *kmsg, return err < 0 ? err : 0; } -static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys, - unsigned int flags, struct used_address *used_address, - unsigned int allowed_msghdr_flags) +static +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +noinline +#endif +int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys, unsigned int flags, + struct used_address *used_address, unsigned int allowed_msghdr_flags) { unsigned char ctl[sizeof(struct cmsghdr) + 20] __aligned(sizeof(__kernel_size_t)); diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 9d3fb6848f40..8ddc65e894da 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -2072,7 +2072,11 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, goto unwrap_failed; opaque_len = be32_to_cpup(p++); offset = (u8 *)(p) - (u8 *)head->iov_base; - if (offset + opaque_len > rcv_buf->len) + if (offset > rcv_buf->len) + goto unwrap_failed; + if (opaque_len > rcv_buf->len - offset) + goto unwrap_failed; + if (opaque_len <= GSS_KRB5_TOK_HDR_LEN) goto unwrap_failed; maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset, diff --git a/net/sunrpc/auth_gss/gss_krb5_unseal.c b/net/sunrpc/auth_gss/gss_krb5_unseal.c index b5fb70419faa..4d12d49434c2 100644 --- a/net/sunrpc/auth_gss/gss_krb5_unseal.c +++ b/net/sunrpc/auth_gss/gss_krb5_unseal.c @@ -89,6 +89,9 @@ gss_krb5_verify_mic_v2(struct krb5_ctx *ctx, struct xdr_buf *message_buffer, dprintk("RPC: %s\n", __func__); + if (read_token->len < GSS_KRB5_TOK_HDR_LEN + cksum_len) + return GSS_S_DEFECTIVE_TOKEN; + memcpy(&be16_ptr, (char *) ptr, 2); if (be16_to_cpu(be16_ptr) != KG2_TOK_MIC) return GSS_S_DEFECTIVE_TOKEN; diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index ac4b32df42b9..d3f61c4b5a13 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -73,6 +73,8 @@ static void _rotate_left(struct xdr_buf *buf, unsigned int shift) int shifted = 0; int this_shift; + if (!buf->len) + return; shift %= buf->len; while (shifted < shift) { this_shift = min(shift - shifted, LOCAL_BUF_LEN); @@ -85,6 +87,8 @@ static void rotate_left(u32 base, struct xdr_buf *buf, unsigned int shift) { struct xdr_buf subbuf; + if (buf->len <= base) + return; xdr_buf_subsegment(buf, &subbuf, base, buf->len - base); _rotate_left(&subbuf, shift); } @@ -154,6 +158,9 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, dprintk("RPC: %s\n", __func__); + if (len - offset <= GSS_KRB5_TOK_HDR_LEN) + return GSS_S_DEFECTIVE_TOKEN; + ptr = buf->head[0].iov_base + offset; if (be16_to_cpu(*((__be16 *)ptr)) != KG2_TOK_WRAP) @@ -220,14 +227,16 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, * head buffer space rather than that actually occupied. */ movelen = min_t(unsigned int, buf->head[0].iov_len, len); + if (movelen < offset + GSS_KRB5_TOK_HDR_LEN + headskip) + return GSS_S_DEFECTIVE_TOKEN; movelen -= offset + GSS_KRB5_TOK_HDR_LEN + headskip; - BUG_ON(offset + GSS_KRB5_TOK_HDR_LEN + headskip + movelen > - buf->head[0].iov_len); memmove(ptr, ptr + GSS_KRB5_TOK_HDR_LEN + headskip, movelen); buf->head[0].iov_len -= GSS_KRB5_TOK_HDR_LEN + headskip; buf->len = len - (GSS_KRB5_TOK_HDR_LEN + headskip); /* Trim off the trailing "extra count" and checksum blob */ + if (ec + GSS_KRB5_TOK_HDR_LEN + tailskip > buf->len - offset) + return GSS_S_DEFECTIVE_TOKEN; xdr_buf_trim(buf, ec + GSS_KRB5_TOK_HDR_LEN + tailskip); *align = XDR_QUADLEN(GSS_KRB5_TOK_HDR_LEN + headskip); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c index 0fa4778620d9..b7f70b1adb18 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.c +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c @@ -121,12 +121,6 @@ static int gssp_rpc_create(struct net *net, struct rpc_clnt **_clnt) return result; } -void init_gssp_clnt(struct sunrpc_net *sn) -{ - mutex_init(&sn->gssp_lock); - sn->gssp_clnt = NULL; -} - int set_gssp_clnt(struct net *net) { struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.h b/net/sunrpc/auth_gss/gss_rpc_upcall.h index 31e96344167e..b3c2b2b90798 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.h +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.h @@ -29,7 +29,6 @@ int gssp_accept_sec_context_upcall(struct net *net, struct gssp_upcall_data *data); void gssp_free_upcall_data(struct gssp_upcall_data *data); -void init_gssp_clnt(struct sunrpc_net *); int set_gssp_clnt(struct net *); void clear_gssp_clnt(struct sunrpc_net *); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index fceee648d545..0549edae1ebe 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -222,7 +222,8 @@ static int gssx_dec_linux_creds(struct xdr_stream *xdr, return 0; out_free_groups: - groups_free(creds->cr_group_info); + put_group_info(creds->cr_group_info); + creds->cr_group_info = NULL; return err; } @@ -230,6 +231,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, struct gssx_option_array *oa) { struct svc_cred *creds; + bool creds_decoded = false; u32 count, i; __be32 *p; int err; @@ -242,12 +244,12 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; /* we recognize only 1 currently: CREDS_VALUE */ - oa->count = 1; - oa->data = kmalloc_obj(struct gssx_option); if (!oa->data) return -ENOMEM; + oa->count = 1; + creds = kzalloc_obj(struct svc_cred); if (!creds) { err = -ENOMEM; @@ -280,9 +282,14 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, if (length == sizeof(CREDS_VALUE) && memcmp(p, CREDS_VALUE, sizeof(CREDS_VALUE)) == 0) { /* We have creds here. parse them */ + if (creds_decoded) { + err = -EINVAL; + goto free_creds; + } err = gssx_dec_linux_creds(xdr, creds); if (err) goto free_creds; + creds_decoded = true; oa->data[0].value.len = 1; /* presence */ } else { /* consume uninteresting buffer */ @@ -294,8 +301,10 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; free_creds: + free_svc_cred(creds); kfree(creds); free_oa: + oa->count = 0; kfree(oa->data); oa->data = NULL; return err; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index d14209031e18..967e9d53080d 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -949,6 +949,8 @@ svcauth_gss_unwrap_priv(struct svc_rqst *rqstp, u32 seq, struct gss_ctx *ctx) } if (len > xdr_stream_remaining(xdr)) goto unwrap_failed; + if (len <= GSS_KRB5_TOK_HDR_LEN) + goto unwrap_failed; offset = xdr_stream_pos(xdr); saved_len = buf->len; @@ -1466,7 +1468,6 @@ static int create_use_gss_proxy_proc_entry(struct net *net) &use_gss_proxy_proc_ops, net); if (!*p) return -ENOMEM; - init_gssp_clnt(sn); return 0; } @@ -1573,6 +1574,9 @@ svcauth_gss_decode_credbody(struct xdr_stream *xdr, u32 body_len; __be32 *p; + /* Early-return paths leave deterministic state, not stale residue. */ + memset(gc, 0, sizeof(*gc)); + p = xdr_inline_decode(xdr, XDR_UNIT); if (!p) return false; @@ -1942,6 +1946,8 @@ svcauth_gss_release(struct svc_rqst *rqstp) if (!gsd) goto out; + if (rqstp->rq_auth_stat != rpc_auth_ok) + goto out; gc = &gsd->clcred; if (gc->gc_proc != RPC_GSS_PROC_DATA) goto out; diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c index 0ffa4d01a938..1482b06e0f38 100644 --- a/net/sunrpc/backchannel_rqst.c +++ b/net/sunrpc/backchannel_rqst.c @@ -25,20 +25,39 @@ unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt) } /* - * Helper function to nullify backchannel server pointer in transport. - * We need to synchronize setting the pointer to NULL (done so after - * the backchannel server is shutdown) with the usage of that pointer - * by the backchannel request processing routines - * xprt_complete_bc_request() and rpcrdma_bc_receive_call(). + * Close the backchannel producer side, drain any requests still + * queued on sv_cb_list, then destroy the callback service. */ void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { - spin_lock(&xprt->bc_pa_lock); + struct svc_serv *bc_serv = *serv; + struct rpc_rqst *req; + + xprt_svc_shutdown_bc(xprt); + while ((req = lwq_dequeue(&bc_serv->sv_cb_list, struct rpc_rqst, + rq_bc_list)) != NULL) { + atomic_dec(&req->rq_xprt->bc_slot_count); + xprt_free_bc_request(req); + } svc_destroy(serv); +} +EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); + +/* + * Clear the backchannel server pointer in the transport. The NULL + * store is serialized under bc_pa_lock against readers of + * xprt->bc_serv in xprt_complete_bc_request() and + * rpcrdma_bc_receive_call(). Clearing it before the callback service + * is stopped prevents a producer from enqueueing onto a service that + * is being torn down. + */ +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ + spin_lock(&xprt->bc_pa_lock); xprt->bc_serv = NULL; spin_unlock(&xprt->bc_pa_lock); } -EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc); +EXPORT_SYMBOL_GPL(xprt_svc_shutdown_bc); /* * Helper routines that track the number of preallocation elements @@ -393,7 +412,12 @@ void xprt_enqueue_bc_request(struct rpc_rqst *req) if (bc_serv) { lwq_enqueue(&req->rq_bc_list, &bc_serv->sv_cb_list); svc_pool_wake_idle_thread(&bc_serv->sv_pools[0]); + spin_unlock(&xprt->bc_pa_lock); + return; } spin_unlock(&xprt->bc_pa_lock); + + atomic_dec(&xprt->bc_slot_count); + xprt_free_bc_request(req); } EXPORT_SYMBOL_GPL(xprt_enqueue_bc_request); diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 391037f15292..1bc04109d213 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -430,10 +430,9 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd) list_del_init(&cd->others); spin_unlock(&cd->hash_lock); spin_unlock(&cache_list_lock); - if (list_empty(&cache_list)) { - /* module must be being unloaded so its safe to kill the worker */ - cancel_delayed_work_sync(&cache_cleaner); - } + cancel_delayed_work_sync(&cache_cleaner); + if (!list_empty(&cache_list)) + queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0); } EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail); diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index ab88ce46afb5..1a3884a0376a 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -57,6 +57,7 @@ static __net_init int sunrpc_init_net(struct net *net) INIT_LIST_HEAD(&sn->all_clients); spin_lock_init(&sn->rpc_client_lock); spin_lock_init(&sn->rpcb_clnt_lock); + mutex_init(&sn->gssp_lock); return 0; err_pipefs: diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index ae9ec4bf34f7..dd80a2eaaa74 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -476,6 +476,35 @@ __svc_init_bc(struct svc_serv *serv) } #endif +static int svc_pool_init_counters(struct svc_pool *pool) +{ + int err; + + err = percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); + if (err) + return err; + err = percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); + if (err) + goto err_sockets; + err = percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (err) + goto err_threads; + return 0; + +err_threads: + percpu_counter_destroy(&pool->sp_sockets_queued); +err_sockets: + percpu_counter_destroy(&pool->sp_messages_arrived); + return err; +} + +static void svc_pool_destroy_counters(struct svc_pool *pool) +{ + percpu_counter_destroy(&pool->sp_messages_arrived); + percpu_counter_destroy(&pool->sp_sockets_queued); + percpu_counter_destroy(&pool->sp_threads_woken); +} + /* * Create an RPC service */ @@ -540,12 +569,18 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, INIT_LIST_HEAD(&pool->sp_all_threads); init_llist_head(&pool->sp_idle_threads); - percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (svc_pool_init_counters(pool)) + goto out_err; } return serv; + +out_err: + while (i--) + svc_pool_destroy_counters(&serv->sv_pools[i]); + kfree(serv->sv_pools); + kfree(serv); + return NULL; } /** @@ -624,9 +659,7 @@ svc_destroy(struct svc_serv **servp) for (i = 0; i < serv->sv_nrpools; i++) { struct svc_pool *pool = &serv->sv_pools[i]; - percpu_counter_destroy(&pool->sp_messages_arrived); - percpu_counter_destroy(&pool->sp_sockets_queued); - percpu_counter_destroy(&pool->sp_threads_woken); + svc_pool_destroy_counters(pool); } kfree(serv->sv_pools); kfree(serv); @@ -683,6 +716,15 @@ svc_release_buffer(struct svc_rqst *rqstp) } } +static void svc_rqst_free_rcu(struct rcu_head *head) +{ + struct svc_rqst *rqstp = container_of(head, struct svc_rqst, rq_rcu_head); + + kfree(rqstp->rq_resp); + kfree(rqstp->rq_argp); + kfree(rqstp); +} + static void svc_rqst_free(struct svc_rqst *rqstp) { @@ -691,10 +733,8 @@ svc_rqst_free(struct svc_rqst *rqstp) svc_release_buffer(rqstp); if (rqstp->rq_scratch_folio) folio_put(rqstp->rq_scratch_folio); - kfree(rqstp->rq_resp); - kfree(rqstp->rq_argp); kfree(rqstp->rq_auth_data); - kfree_rcu(rqstp, rq_rcu_head); + call_rcu(&rqstp->rq_rcu_head, svc_rqst_free_rcu); } static struct svc_rqst * @@ -954,6 +994,29 @@ svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, } EXPORT_SYMBOL_GPL(svc_set_num_threads); +/** + * svc_serv_maxthreads - report a service's configured thread ceiling + * @serv: RPC service to query + * + * A pooled service sizes its threads dynamically, so the number of + * threads running at any moment tracks recent load rather than the + * service's capacity. The per-pool maximum is the stable figure a + * consumer should size against. + * + * The caller must keep @serv valid for the duration of the call. + * + * Return: the sum of every pool's maximum thread count. + */ +unsigned int svc_serv_maxthreads(const struct svc_serv *serv) +{ + unsigned int i, max = 0; + + for (i = 0; i < serv->sv_nrpools; i++) + max += data_race(serv->sv_pools[i].sp_nrthrmax); + return max; +} +EXPORT_SYMBOL_GPL(svc_serv_maxthreads); + /** * svc_rqst_replace_page - Replace one page in rq_respages[] * @rqstp: svc_rqst with pages to replace diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 64a2658faddb..aebd97e7f66c 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -133,7 +133,7 @@ static void ip_map_init(struct cache_head *cnew, struct cache_head *citem) struct ip_map *new = container_of(cnew, struct ip_map, h); struct ip_map *item = container_of(citem, struct ip_map, h); - strcpy(new->m_class, item->m_class); + strscpy(new->m_class, item->m_class); new->m_addr = item->m_addr; } static void update(struct cache_head *cnew, struct cache_head *citem) @@ -296,7 +296,7 @@ static struct ip_map *__ip_map_lookup(struct cache_detail *cd, char *class, struct ip_map ip; struct cache_head *ch; - strcpy(ip.m_class, class); + strscpy(ip.m_class, class); ip.m_addr = *addr; ch = sunrpc_cache_lookup_rcu(cd, &ip.h, hash_str(class, IP_HASHBITS) ^ diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index a90480f80154..e638b92b7ad1 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -327,7 +327,7 @@ static ssize_t rpc_sysfs_xprt_switch_add_xprt_store(struct kobject *kobj, { struct rpc_xprt_switch *xprt_switch = rpc_sysfs_xprt_switch_kobj_get_xprt(kobj); - struct xprt_create xprt_create_args; + struct xprt_create xprt_create_args = {}; struct rpc_xprt *xprt, *new; if (!xprt_switch) @@ -348,7 +348,7 @@ static ssize_t rpc_sysfs_xprt_switch_add_xprt_store(struct kobject *kobj, xprt_create_args.reconnect_timeout = xprt->max_reconnect_timeout; new = xprt_create_transport(&xprt_create_args); - if (IS_ERR_OR_NULL(new)) { + if (IS_ERR(new)) { count = PTR_ERR(new); goto out_put_xprt; } diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index fa6a30b5f046..cb2ef428651f 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -2049,7 +2049,7 @@ void xdr_buf_trim(struct xdr_buf *buf, unsigned int len) trim -= cur; } fix_len: - buf->len -= (len - trim); + buf->len -= min_t(unsigned int, buf->len, len - trim); } EXPORT_SYMBOL_GPL(xdr_buf_trim); diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index 2f0f9618dd05..e5b3463da25f 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -159,9 +159,7 @@ void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst) rpcrdma_rep_put(&r_xprt->rx_buf, rep); req->rl_reply = NULL; - spin_lock(&xprt->bc_pa_lock); - list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list); - spin_unlock(&xprt->bc_pa_lock); + rpcrdma_req_put(req); xprt_put(xprt); } @@ -203,6 +201,7 @@ static struct rpc_rqst *rpcrdma_bc_rqst_get(struct rpcrdma_xprt *r_xprt) rqst->rq_xprt = xprt; __set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state); xdr_buf_init(&rqst->rq_snd_buf, rdmab_data(req->rl_sendbuf), size); + kref_init(&req->rl_kref); return rqst; } diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index 7f79a0a2601e..e5c71cf705a3 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -474,7 +474,7 @@ int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) ++num_wrs; } - if ((kref_read(&req->rl_kref) > 1) || num_wrs > ep->re_send_count) { + if (req->rl_sendctx->sc_unmap_count || num_wrs > ep->re_send_count) { send_wr->send_flags |= IB_SEND_SIGNALED; ep->re_send_count = min_t(unsigned int, ep->re_send_batch, num_wrs - ep->re_send_count); diff --git a/net/sunrpc/xprtrdma/ib_client.c b/net/sunrpc/xprtrdma/ib_client.c index de49ad02053d..188f7a13397f 100644 --- a/net/sunrpc/xprtrdma/ib_client.c +++ b/net/sunrpc/xprtrdma/ib_client.c @@ -51,7 +51,11 @@ static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device) * to be invoked when the device is removed, unless this notification * is unregistered first. * - * On failure, a negative errno is returned. + * On failure, a negative errno is returned. rn->rn_done is left + * NULL on every failure path (it is armed before xa_alloc but + * cleared again if xa_alloc fails), so the @rn may safely be + * passed to rpcrdma_rn_unregister() without a separate + * registered/unregistered flag in the caller. */ int rpcrdma_rn_register(struct ib_device *device, struct rpcrdma_notification *rn, @@ -62,10 +66,21 @@ int rpcrdma_rn_register(struct ib_device *device, if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags)) return -ENETUNREACH; - if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) + /* + * Arm rn_done before xa_alloc() publishes @rn: once @rn is + * visible in rd_xa, a concurrent rpcrdma_remove_one() can + * call rn->rn_done(), so the pointer must already be set. + * + * Restore NULL if xa_alloc() fails. rn_done doubles as the + * registration sentinel for rpcrdma_rn_unregister(); a stale + * value would unregister an @rn that was never inserted. + */ + rn->rn_done = done; + if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) { + rn->rn_done = NULL; return -ENOMEM; + } kref_get(&rd->rd_kref); - rn->rn_done = done; trace_rpcrdma_client_register(device, rn); return 0; } @@ -83,6 +98,10 @@ static void rpcrdma_rn_release(struct kref *kref) * rpcrdma_rn_unregister - stop device removal notifications * @device: monitored device * @rn: notification object that no longer wishes to be notified + * + * It is safe to call this on an @rn whose registration never + * completed or failed; rn_done == NULL is treated as + * never-registered and the call is a no-op. */ void rpcrdma_rn_unregister(struct ib_device *device, struct rpcrdma_notification *rn) @@ -92,6 +111,21 @@ void rpcrdma_rn_unregister(struct ib_device *device, if (!rd) return; + /* + * rn_done is the registration sentinel: rpcrdma_rn_register + * leaves it NULL on every failure path, clearing it again if + * xa_alloc fails, so a non-NULL rn_done marks a completed + * registration. A NULL rn_done means this notification was + * never registered (or its registration failed) or has + * already been unregistered, and the call is a no-op. + * Without this guard, rn_index == 0 from a kzalloc'd + * parent would erase another caller's slot 0 and underflow + * rd_kref. + */ + if (!rn->rn_done) + return; + rn->rn_done = NULL; + trace_rpcrdma_client_unregister(device, rn); xa_erase(&rd->rd_xa, rn->rn_index); kref_put(&rd->rd_kref, rpcrdma_rn_release); diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 0e0f21974710..1285f04cdac1 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -467,29 +467,11 @@ static int rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, return 0; } -static void rpcrdma_sendctx_done(struct kref *kref) -{ - struct rpcrdma_req *req = - container_of(kref, struct rpcrdma_req, rl_kref); - struct rpcrdma_rep *rep = req->rl_reply; - - rpcrdma_complete_rqst(rep); - rep->rr_rxprt->rx_stats.reply_waits_for_send++; -} - -/** - * rpcrdma_sendctx_unmap - DMA-unmap Send buffer - * @sc: sendctx containing SGEs to unmap - * - */ -void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) +static void rpcrdma_sendctx_dma_unmap(struct rpcrdma_sendctx *sc) { struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf; struct ib_sge *sge; - if (!sc->sc_unmap_count) - return; - /* The first two SGEs contain the transport header and * the inline buffer. These are always left mapped so * they can be cheaply re-used. @@ -498,8 +480,33 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) ++sge, --sc->sc_unmap_count) ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length, DMA_TO_DEVICE); +} + +/** + * rpcrdma_sendctx_unmap - DMA-unmap Send buffer and release Send owner + * @sc: sendctx containing SGEs to unmap + * + */ +void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) +{ + struct rpcrdma_req *req = sc->sc_req; - kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done); + rpcrdma_sendctx_dma_unmap(sc); + sc->sc_req = NULL; + req->rl_sendctx = NULL; + rpcrdma_req_put(req); +} + +/* No Send was posted. Release DMA mappings prepared for this + * sendctx, but leave the request reference count alone. + */ +static void rpcrdma_sendctx_cancel(struct rpcrdma_sendctx *sc) +{ + struct rpcrdma_req *req = sc->sc_req; + + rpcrdma_sendctx_dma_unmap(sc); + sc->sc_req = NULL; + req->rl_sendctx = NULL; } /* Prepare an SGE for the RPC-over-RDMA transport header. @@ -691,8 +698,6 @@ static bool rpcrdma_prepare_noch_mapped(struct rpcrdma_xprt *r_xprt, tail->iov_len)) return false; - if (req->rl_sendctx->sc_unmap_count) - kref_get(&req->rl_kref); return true; } @@ -722,7 +727,6 @@ static bool rpcrdma_prepare_readch(struct rpcrdma_xprt *r_xprt, len -= len & 3; if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len)) return false; - kref_get(&req->rl_kref); } return true; @@ -743,6 +747,7 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdr, enum rpcrdma_chunktype rtype) { + struct rpcrdma_sendctx *sc; int ret; ret = -EAGAIN; @@ -751,7 +756,6 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, goto out_nosc; req->rl_sendctx->sc_unmap_count = 0; req->rl_sendctx->sc_req = req; - kref_init(&req->rl_kref); req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe; req->rl_wr.sg_list = req->rl_sendctx->sc_sges; req->rl_wr.num_sge = 0; @@ -779,10 +783,16 @@ inline int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt, goto out_unmap; } + /* The Send-side owner releases this reference when the + * Send has completed. + */ + kref_get(&req->rl_kref); return 0; out_unmap: - rpcrdma_sendctx_unmap(req->rl_sendctx); + sc = req->rl_sendctx; + rpcrdma_sendctx_cancel(sc); + rpcrdma_sendctx_unget_locked(r_xprt, sc); out_nosc: trace_xprtrdma_prepsend_failed(&req->rl_slot, ret); return ret; @@ -1081,6 +1091,8 @@ rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) /* Peek at stream contents without advancing. */ p = xdr_inline_decode(xdr, 0); + if ((char *)xdr->end - (char *)p < 5 * XDR_UNIT) + return false; /* Chunk lists */ if (xdr_item_is_present(p++)) @@ -1105,7 +1117,7 @@ rpcrdma_is_bcall(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep) */ p = xdr_inline_decode(xdr, 3 * sizeof(*p)); if (unlikely(!p)) - return true; + return false; rpcrdma_bc_receive_call(r_xprt, rep); return true; @@ -1329,6 +1341,11 @@ void rpcrdma_complete_rqst(struct rpcrdma_rep *rep) struct rpc_rqst *rqst = rep->rr_rqst; int status; + /* I3: rl_registered has been drained by frwr_unmap before + * complete_rqst runs. + */ + WARN_ON_ONCE(!list_empty(&rpcr_to_rdmar(rqst)->rl_registered)); + switch (rep->rr_proc) { case rdma_msg: status = rpcrdma_decode_msg(r_xprt, rep, rqst); @@ -1360,13 +1377,69 @@ void rpcrdma_complete_rqst(struct rpcrdma_rep *rep) goto out; } -static void rpcrdma_reply_done(struct kref *kref) -{ - struct rpcrdma_req *req = - container_of(kref, struct rpcrdma_req, rl_kref); - - rpcrdma_complete_rqst(req->rl_reply); -} +/* Reply-side ownership invariants + * + * I1 (Receive WR ownership). A struct rpcrdma_rep is owned by the + * HCA between ib_post_recv() and the matching Receive completion. + * After ib_dma_sync_single_for_cpu() in rpcrdma_wc_receive() it is + * owned by the CPU until rpcrdma_rep_put() returns it to + * rb_free_reps; a rep on rb_free_reps is not re-posted until + * rpcrdma_post_recvs() pulls it off. Asserted: rpcrdma_post_recvs() + * WARNs that a pulled rep has rr_rqst == NULL. + * + * I2 (rep attachment). While req->rl_reply == rep, the rep cannot be + * re-posted. rpcrdma_reply_put() NULLs req->rl_reply before handing + * the rep to rpcrdma_rep_put(). Asserted: rpcrdma_reply_put() WARNs + * that rl_reply is NULL after the put. + * + * I3 (Registered-MR fence). On entry to rpcrdma_complete_rqst() every + * MR that was on req->rl_registered has had its rkey invalidated + * (remotely via IB_WC_WITH_INVALIDATE or locally via IB_WR_LOCAL_INV) + * and its pages ib_dma_unmap_sg()'d. The LocalInv chain is posted + * on a single QP; strong send-queue ordering makes the last + * completion (frwr_wc_localinv_done) observe the + * ib_dma_unmap_sg() that ran from each earlier completion's + * frwr_mr_put() before complete_rqst is called. The inline + * frwr_reminv() path unmaps its one MR synchronously before + * rpcrdma_reply_handler() reaches complete_rqst. Asserted: + * rpcrdma_complete_rqst() WARNs that rl_registered is empty. + * + * I4 (Send-buffer release). req->rl_kref carries two unconditional + * owners while a Send is outstanding: the RPC-layer reference (set + * at xprt_rdma_alloc_slot / xprt_rdma_bc_rqst_get / rpcrdma_req_release + * pool-entry) and the Send-side reference (kref_get() in + * rpcrdma_prepare_send_sges()). rpcrdma_req_release() runs only + * after both have dropped, so the req does not return to its free + * pool until rpcrdma_sendctx_unmap() has fired -- the HCA has + * released the send buffer before the req can be reused. Asserted: + * rpcrdma_req_release() WARNs that rl_sendctx is NULL. + * + * I5 (req lifecycle). A req is owned by the RPC layer between slot + * acquisition and the matching xprt_rdma_free_slot() (or, for the + * backchannel, xprt_rdma_bc_free_rqst()). While owned, rl_kref >= 1. + * The pools (rb_send_bufs, bc_pa_list, backlog wake target) never + * contain a req with outstanding Send-side or Reply-side work. + * + * Non-hazards. The following claims have been raised by adversarial + * review and are each closed by the invariants above: + * + * * "Reply completes the RPC while the HCA still holds the send + * buffer" -- excluded by I4. The Send-side kref reference is held + * until rpcrdma_sendctx_unmap() runs from Send completion. + * + * * "Signal-driven release races the in-flight Send" -- same + * resolution. xprt_rdma_free() does not touch rl_kref; the + * Send-side reference keeps the req out of its pool until Send + * completion fires. + * + * * "Receive completion races rep reuse" -- excluded by I1. A rep + * is on rb_free_reps only after rpcrdma_rep_put() has been called + * and rpcrdma_post_recvs() owns the next transition back to the HCA. + * + * * "Pages still DMA-mapped when call_decode reads them" -- excluded + * by I3. The matching ib_dma_unmap_sg() for every MR has run on + * the same CPU thread that calls rpcrdma_complete_rqst(). + */ /** * rpcrdma_reply_handler - Process received RPC/RDMA messages @@ -1402,6 +1475,14 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) credits = be32_to_cpu(*p++); rep->rr_proc = *p++; + /* The credit grant from the wire is not trustworthy; + * sanitize it before any code path consumes it. + */ + if (credits == 0) + credits = 1; /* don't deadlock */ + else if (credits > r_xprt->rx_ep->re_max_requests) + credits = r_xprt->rx_ep->re_max_requests; + if (rep->rr_vers != rpcrdma_version) goto out_badversion; @@ -1418,10 +1499,6 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) xprt_pin_rqst(rqst); spin_unlock(&xprt->queue_lock); - if (credits == 0) - credits = 1; /* don't deadlock */ - else if (credits > r_xprt->rx_ep->re_max_requests) - credits = r_xprt->rx_ep->re_max_requests; if (buf->rb_credits != credits) rpcrdma_update_cwnd(r_xprt, credits); @@ -1439,7 +1516,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) frwr_unmap_async(r_xprt, req); /* LocalInv completion will complete the RPC */ else - kref_put(&req->rl_kref, rpcrdma_reply_done); + rpcrdma_complete_rqst(rep); out_post: rpcrdma_post_recvs(r_xprt, @@ -1454,11 +1531,13 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep) out_badversion: trace_xprtrdma_reply_vers_err(rep); - goto out; + rpcrdma_rep_put(buf, rep); + credits = buf->rb_credits; + goto out_post; out_shortreply: trace_xprtrdma_reply_short_err(rep); - -out: rpcrdma_rep_put(buf, rep); + credits = buf->rb_credits; + goto out_post; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_pcl.c b/net/sunrpc/xprtrdma/svc_rdma_pcl.c index 1f8f7dad8b6f..8623722790f2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_pcl.c +++ b/net/sunrpc/xprtrdma/svc_rdma_pcl.c @@ -149,9 +149,6 @@ bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) * cl_count is updated to be the number of chunks (ie. * unique position values) in the Read list. * %false: Memory allocation failed. - * - * TODO: - * - Check for chunk range overlaps */ bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { @@ -213,7 +210,6 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk = pcl_alloc_chunk(segcount, 0); if (!chunk) return false; - list_add_tail(&chunk->ch_list, &pcl->cl_chunks); for (j = 0; j < segcount; j++) { segment = &chunk->ch_segments[j]; @@ -225,7 +221,66 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk->ch_length += segment->rs_length; chunk->ch_segcount++; } + list_add_tail(&chunk->ch_list, &pcl->cl_chunks); + } + return true; +} + +/** + * pcl_check_read_chunk_positions - Validate Read chunk positions + * @rctxt: Ingress receive context with populated chunk lists + * @inline_len: Length of the inline RPC body after the transport header + * + * Read chunk positions are offsets in the unreduced XDR stream + * (RFC 8166 Section 3.4.4), so each position includes the + * cumulative length of preceding Read chunks. This function + * subtracts those lengths to recover the inline-body offset + * before comparing against @inline_len or the Call chunk length. + * + * Rejects frames where a Read chunk's inline-body offset exceeds + * the bound, where adjacent Read chunks overlap, or where any + * single chunk length exceeds the page budget. + * + * Return values: + * %true: Read chunk positions and lengths are valid + * %false: Malformed chunk list detected + */ +bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt, + unsigned int inline_len) +{ + unsigned int max_len, bound, total_read; + struct svc_rdma_chunk *chunk, *next; + + max_len = rctxt->rc_maxpages << PAGE_SHIFT; + + if (!pcl_is_empty(&rctxt->rc_call_pcl)) { + chunk = pcl_first_chunk(&rctxt->rc_call_pcl); + if (chunk->ch_length > max_len) + return false; + bound = chunk->ch_length; + } else { + bound = inline_len; + } + + if (pcl_is_empty(&rctxt->rc_read_pcl)) + return true; + + total_read = 0; + pcl_for_each_chunk(chunk, &rctxt->rc_read_pcl) { + if (chunk->ch_position - total_read > bound) + return false; + if (chunk->ch_length > max_len) + return false; + + next = pcl_next_chunk(&rctxt->rc_read_pcl, chunk); + if (!next) + break; + + if (chunk->ch_position + chunk->ch_length > next->ch_position) + return false; + total_read += chunk->ch_length; } + return true; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index fe9bf0371b6e..fdfed1be97da 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -440,11 +440,14 @@ static void svc_rdma_build_arg_xdr(struct svc_rqst *rqstp, * to the first byte past the Read list. rc_read_pcl and * rc_call_pcl cl_count fields are set to the number of * Read segments in the list. - * %false: Read list is corrupt. @rctxt's xdr_stream is left in an - * unknown state. + * %false: Read list is corrupt or exceeds the page budget. @rctxt's + * xdr_stream is left in an unknown state. */ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { + unsigned int maxlen = rctxt->rc_maxpages << PAGE_SHIFT; + unsigned int total_len = 0; + rctxt->rc_call_pcl.cl_count = 0; rctxt->rc_read_pcl.cl_count = 0; while (xdr_item_is_present(p)) { @@ -458,6 +461,11 @@ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) xdr_decode_read_segment(p, &position, &handle, &length, &offset); + if (length > maxlen) + return false; + total_len += length; + if (PAGE_ALIGN(total_len) > maxlen) + return false; if (position) { if (position & 3) return false; @@ -508,10 +516,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt) return false; /* Before trusting the segcount value enough to use it in - * a computation, perform a simple range check. This is an - * arbitrary but sensible limit (ie, not architectural). + * a computation, perform a simple range check. A zero + * segcount describes no remote buffer and is rejected so + * downstream consumers never see a degenerate ch_segcount==0 + * chunk. The upper bound is an arbitrary but sensible limit + * (ie, not architectural). */ - if (unlikely(segcount > rctxt->rc_maxpages)) + if (segcount == 0 || unlikely(segcount > rctxt->rc_maxpages)) return false; p = xdr_inline_decode(&rctxt->rc_stream, @@ -719,6 +730,9 @@ static int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg, rq_arg->head[0].iov_base = rctxt->rc_stream.p; hdr_len = xdr_stream_pos(&rctxt->rc_stream); + if (!pcl_check_read_chunk_positions(rctxt, + rq_arg->head[0].iov_len - hdr_len)) + goto out_inval; rq_arg->head[0].iov_len -= hdr_len; rq_arg->len -= hdr_len; trace_svcrdma_decode_rqst(rctxt, rdma_argp, hdr_len); diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index f7fd22cc4a59..9aaaade99e6e 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -795,7 +795,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, len -= seg_len; if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages)) - goto out_overrun; + goto out_put; } ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset, @@ -809,7 +809,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, cc->cc_sqecount += ret; return 0; -out_overrun: +out_put: + svc_rdma_put_rw_ctxt(rdma, ctxt); trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage); return -EINVAL; } @@ -847,7 +848,7 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, * svc_rdma_copy_inline_range - Copy part of the inline content into pages * @rqstp: RPC transaction context * @head: context for ongoing I/O - * @offset: offset into the Receive buffer of region to copy + * @offset: offset into the inline content of region to copy * @remaining: length of region to copy * * Take a page at a time from rqstp->rq_pages and copy the inline @@ -864,9 +865,13 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, unsigned int offset, unsigned int remaining) { - unsigned char *dst, *src = head->rc_recv_buf; + unsigned char *dst, *src = head->rc_saved_arg.head[0].iov_base; + unsigned int inline_len = head->rc_saved_arg.head[0].iov_len; unsigned int page_no, numpages; + if (offset > inline_len || remaining > inline_len - offset) + return -EINVAL; + numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT; for (page_no = 0; page_no < numpages; page_no++) { unsigned int page_len; @@ -917,9 +922,10 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, { const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int inline_len, start, length; int ret; + inline_len = head->rc_saved_arg.head[0].iov_len; start = 0; chunk = pcl_first_chunk(pcl); length = chunk->ch_position; @@ -937,6 +943,8 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, break; start += length; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_copy_inline_range(rqstp, head, start, length); if (ret < 0) @@ -944,7 +952,9 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, } start += length; - length = head->rc_byte_len - start; + if (start > inline_len) + return -EINVAL; + length = inline_len - start; return svc_rdma_copy_inline_range(rqstp, head, start, length); } @@ -969,8 +979,12 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, static int svc_rdma_read_data_item(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { - return svc_rdma_build_read_chunk(rqstp, head, - pcl_first_chunk(&head->rc_read_pcl)); + struct svc_rdma_chunk *chunk = pcl_first_chunk(&head->rc_read_pcl); + + if (chunk->ch_position > head->rc_saved_arg.head[0].iov_len) + return -EINVAL; + + return svc_rdma_build_read_chunk(rqstp, head, chunk); } /** @@ -996,17 +1010,20 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, const struct svc_rdma_segment *segment; int ret; + if (!length) + return 0; + ret = -EINVAL; pcl_for_each_segment(segment, chunk) { struct svc_rdma_segment dummy; - if (offset > segment->rs_length) { + if (offset >= segment->rs_length) { offset -= segment->rs_length; continue; } dummy.rs_handle = segment->rs_handle; - dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; + dummy.rs_length = min_t(u32, length, segment->rs_length - offset); dummy.rs_offset = segment->rs_offset + offset; ret = svc_rdma_build_read_segment(rqstp, head, &dummy); @@ -1015,6 +1032,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, head->rc_readbytes += dummy.rs_length; length -= dummy.rs_length; + if (!length) + break; offset = 0; } return ret; @@ -1039,14 +1058,17 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, pcl_first_chunk(&head->rc_call_pcl); const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int call_len, start, length; int ret; if (pcl_is_empty(pcl)) return svc_rdma_build_read_chunk(rqstp, head, call_chunk); + call_len = call_chunk->ch_length; start = 0; chunk = pcl_first_chunk(pcl); + if (chunk->ch_position > call_len) + return -EINVAL; length = chunk->ch_position; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1063,6 +1085,10 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, break; start += length; + if (next->ch_position > call_len) + return -EINVAL; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1071,7 +1097,9 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, } start += length; - length = call_chunk->ch_length - start; + if (start > call_len) + return -EINVAL; + length = call_len - start; return svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 7ca71741106b..093371f9d245 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -43,6 +43,7 @@ */ #include +#include #include #include #include @@ -245,12 +246,16 @@ svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt, * structure for the listening endpoint. * * This function creates a new xprt for the new connection and enqueues it on - * the accept queue for the listent xprt. When the listen thread is kicked, it + * the accept queue for the listen xprt. When the listen thread is kicked, it * will call the recvfrom method on the listen xprt which will accept the new * connection. + * + * Return values: + * %0: Do not destroy @new_cma_id + * %1: Destroy @new_cma_id (allocation failure) */ -static void handle_connect_req(struct rdma_cm_id *new_cma_id, - struct rdma_conn_param *param) +static int handle_connect_req(struct rdma_cm_id *new_cma_id, + struct rdma_conn_param *param) { struct svcxprt_rdma *listen_xprt = new_cma_id->context; struct svcxprt_rdma *newxprt; @@ -260,7 +265,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, listen_xprt->sc_xprt.xpt_net, ibdev_to_node(new_cma_id->device)); if (!newxprt) - return; + return 1; newxprt->sc_cm_id = new_cma_id; new_cma_id->context = newxprt; svc_rdma_parse_connect_private(newxprt, param); @@ -294,6 +299,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags); svc_xprt_enqueue(&listen_xprt->sc_xprt); + return 0; } /** @@ -317,14 +323,14 @@ static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id, switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: - handle_connect_req(cma_id, &event->param.conn); - break; + return handle_connect_req(cma_id, &event->param.conn); case RDMA_CM_EVENT_ADDR_CHANGE: listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net, sap, cma_xprt); if (IS_ERR(listen_id)) { pr_err("Listener dead, address change failed for device %s\n", cma_id->device->name); + cma_xprt->sc_cm_id = NULL; } else cma_xprt->sc_cm_id = listen_id; return 1; @@ -387,7 +393,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt); if (IS_ERR(listen_id)) { - kfree(cma_xprt); + /* _svc_xprt_create() acquired one module reference and + * puts it on xpo_create failure. svc_xprt_free() puts + * a second one when the kref drops to zero. Take a + * compensating reference so both puts are balanced. + */ + __module_get(cma_xprt->sc_xprt.xpt_class->xcl_owner); + svc_xprt_put(&cma_xprt->sc_xprt); return ERR_CAST(listen_id); } cma_xprt->sc_cm_id = listen_id; @@ -598,13 +610,26 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) return &newxprt->sc_xprt; errout: - /* Take a reference in case the DTO handler runs */ - svc_xprt_get(&newxprt->sc_xprt); - if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp)) - ib_destroy_qp(newxprt->sc_qp); - rdma_destroy_id(newxprt->sc_cm_id); - rpcrdma_rn_unregister(dev, &newxprt->sc_rn); - /* This call to put will destroy the transport */ + /* + * Drop the kref_init birth reference. svc_xprt_free will + * dispatch xpo_free = svc_rdma_free, which tears down sc_qp, + * sc_sq_cq, sc_rq_cq, and sc_pd under existing IS_ERR/NULL + * guards, and sc_rn under the rn_done sentinel guard inside + * rpcrdma_rn_unregister. + * + * sc_cm_id is destroyed unconditionally by svc_rdma_free; that + * is safe here because sc_cm_id is non-NULL by caller invariant + * on every path that reaches this errout: handle_connect_req + * installs newxprt->sc_cm_id before queueing the new xprt for + * accept, and svc_rdma_accept has already dereferenced it above + * the first goto errout. + * + * svc_handle_xprt() drops its pre-acquired module reference when + * ->xpo_accept() returns NULL. Take a replacement reference before + * freeing @newxprt, because svc_xprt_free() drops the module + * reference associated with @newxprt. + */ + __module_get(newxprt->sc_xprt.xpt_class->xcl_owner); svc_xprt_put(&newxprt->sc_xprt); return NULL; } @@ -614,7 +639,8 @@ static void svc_rdma_detach(struct svc_xprt *xprt) struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - rdma_disconnect(rdma->sc_cm_id); + if (rdma->sc_cm_id) + rdma_disconnect(rdma->sc_cm_id); /* * Most close paths go through svc_rdma_xprt_deferred_close(), @@ -634,10 +660,15 @@ static void svc_rdma_free(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - struct ib_device *device = rdma->sc_cm_id->device; + struct ib_device *device; might_sleep(); + if (!rdma->sc_cm_id) + goto out_free; + + device = rdma->sc_cm_id->device; + /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) ib_drain_qp(rdma->sc_qp); @@ -662,11 +693,13 @@ static void svc_rdma_free(struct svc_xprt *xprt) if (rdma->sc_pd && !IS_ERR(rdma->sc_pd)) ib_dealloc_pd(rdma->sc_pd); + if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) + rpcrdma_rn_unregister(device, &rdma->sc_rn); + /* Destroy the CM ID */ rdma_destroy_id(rdma->sc_cm_id); - if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) - rpcrdma_rn_unregister(device, &rdma->sc_rn); +out_free: kfree(rdma); } diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 61706df5e485..d4e6746d8ecd 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -279,6 +279,13 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) cancel_delayed_work_sync(&r_xprt->rx_connect_worker); rpcrdma_xprt_disconnect(r_xprt); + + /* The disconnect's sendctx drain can return bc_prealloc reqs + * to bc_pa_list after xprt_destroy_backchannel() emptied it. + */ +#if defined(CONFIG_SUNRPC_BACKCHANNEL) + xprt_rdma_bc_destroy(xprt, 0); +#endif rpcrdma_buffer_destroy(&r_xprt->rx_buf); xprt_rdma_free_addresses(xprt); @@ -484,7 +491,52 @@ xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO); } trace_xprtrdma_op_connect(r_xprt, delay); - queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay); + queue_delayed_work(system_dfl_long_wq, &r_xprt->rx_connect_worker, + delay); +} + +/* rl_kref has two owners while a Send is outstanding: the rpc_rqst + * owner and the sendctx. Replies complete the RPC but do not drop + * either reference. The req returns to its free pool only after + * xprt_rdma_free_slot() or xprt_rdma_bc_free_rqst() has dropped the + * RPC-layer reference and rpcrdma_sendctx_unmap() has dropped the + * Send-side reference. + */ +static void rpcrdma_req_release(struct kref *kref) +{ + struct rpcrdma_req *req = + container_of(kref, struct rpcrdma_req, rl_kref); + struct rpc_rqst *rqst = &req->rl_slot; + struct rpc_xprt *xprt = rqst->rq_xprt; + struct rpcrdma_xprt *r_xprt; + + /* I4: both the RPC-layer and Send-side owners have dropped, + * so rpcrdma_sendctx_unmap() has cleared rl_sendctx. + */ + WARN_ON_ONCE(req->rl_sendctx); + + kref_init(&req->rl_kref); + +#if defined(CONFIG_SUNRPC_BACKCHANNEL) + if (bc_prealloc(rqst)) { + spin_lock(&xprt->bc_pa_lock); + list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list); + spin_unlock(&xprt->bc_pa_lock); + return; + } +#endif + + if (xprt_wake_up_backlog(xprt, rqst)) + return; + + r_xprt = rpcx_to_rdmax(xprt); + memset(rqst, 0, sizeof(*rqst)); + rpcrdma_buffer_put(&r_xprt->rx_buf, req); +} + +void rpcrdma_req_put(struct rpcrdma_req *req) +{ + kref_put(&req->rl_kref, rpcrdma_req_release); } /** @@ -505,6 +557,7 @@ xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) req = rpcrdma_buffer_get(&r_xprt->rx_buf); if (!req) goto out_sleep; + kref_init(&req->rl_kref); task->tk_rqstp = &req->rl_slot; task->tk_status = 0; return; @@ -520,6 +573,7 @@ xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) if (req) { struct rpc_rqst *rqst = &req->rl_slot; + kref_init(&req->rl_kref); if (!xprt_wake_up_backlog(xprt, rqst)) { memset(rqst, 0, sizeof(*rqst)); rpcrdma_buffer_put(&r_xprt->rx_buf, req); @@ -540,10 +594,7 @@ xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst) container_of(xprt, struct rpcrdma_xprt, rx_xprt); rpcrdma_reply_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); - if (!xprt_wake_up_backlog(xprt, rqst)) { - memset(rqst, 0, sizeof(*rqst)); - rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); - } + rpcrdma_req_put(rpcr_to_rdmar(rqst)); } static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt, @@ -607,10 +658,10 @@ xprt_rdma_free(struct rpc_task *task) frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req); } - /* XXX: If the RPC is completing because of a signal and - * not because a reply was received, we ought to ensure - * that the Send completion has fired, so that memory - * involved with the Send is not still visible to the NIC. + /* The Send-side rl_kref owner keeps req out of its free pool + * until rpcrdma_sendctx_unmap() has fired -- see I4 above + * rpcrdma_reply_handler() -- so signal-driven release here + * does not let the HCA touch a recycled send buffer. */ } @@ -716,7 +767,7 @@ void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) r_xprt->rx_stats.mrs_allocated, r_xprt->rx_stats.local_inv_needed, r_xprt->rx_stats.empty_sendctx_q, - r_xprt->rx_stats.reply_waits_for_send); + 0LU); /* was reply_waits_for_send; column preserved */ } static int diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index aecf9c0a153f..04b286223b24 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -65,6 +65,8 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt); static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt); +static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, + unsigned long item); static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_sendctx *sc); static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt); @@ -79,6 +81,8 @@ rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction, int node); static struct rpcrdma_regbuf * rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); +static bool rpcrdma_regbuf_realloc_node(struct rpcrdma_regbuf *rb, + size_t size, gfp_t flags, int node); static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); @@ -243,8 +247,17 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) complete(&ep->re_done); return 0; case RDMA_CM_EVENT_ADDR_CHANGE: - ep->re_connect_status = -ENODEV; - goto disconnected; + switch (xchg(&ep->re_connect_status, -ENODEV)) { + case 0: + goto wake_connect_worker; + case 1: + /* The later DISCONNECTED event balances the + * ESTABLISHED get; do not put here. + */ + rpcrdma_force_disconnect(ep); + return 0; + } + return 0; case RDMA_CM_EVENT_ESTABLISHED: rpcrdma_ep_get(ep); ep->re_connect_status = 1; @@ -267,7 +280,6 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) return 0; case RDMA_CM_EVENT_DISCONNECTED: ep->re_connect_status = -ECONNABORTED; -disconnected: rpcrdma_force_disconnect(ep); return rpcrdma_ep_put(ep); default: @@ -324,6 +336,7 @@ static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, if (rc) goto out; + ep->re_id = id; rc = rpcrdma_rn_register(id->device, &ep->re_rn, rpcrdma_ep_removal_done); if (rc) goto out; @@ -396,7 +409,6 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) } __module_get(THIS_MODULE); device = id->device; - ep->re_id = id; reinit_completion(&ep->re_done); ep->re_max_requests = r_xprt->rx_xprt.max_reqs; @@ -539,7 +551,17 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) goto out; } rpcrdma_mrs_create(r_xprt); - frwr_wp_create(r_xprt); + + /* + * rpcrdma_encode_write_list() dereferences the write-pad + * MR with no NULL check, so fail the connect rather than + * publish a transport whose write-pad MR is NULL. + */ + rc = frwr_wp_create(r_xprt); + if (rc) { + rc = -ENOTCONN; + goto out; + } out: trace_xprtrdma_connect(r_xprt, rc); @@ -571,9 +593,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) rpcrdma_xprt_drain(r_xprt); rpcrdma_reps_unmap(r_xprt); + rpcrdma_sendctxs_destroy(r_xprt); rpcrdma_reqs_reset(r_xprt); rpcrdma_mrs_destroy(r_xprt); - rpcrdma_sendctxs_destroy(r_xprt); if (rpcrdma_ep_put(ep)) rdma_destroy_id(id); @@ -605,6 +627,25 @@ static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt) if (!buf->rb_sc_ctxs) return; + + /* The QP is drained, but the final unsignaled Sends might not + * have been walked by a signaled Send completion. Release those + * Send owners before request buffers are reset. + * + * Unlike the completion sweep, this walk can visit slots with + * no Send posted: after a partial rpcrdma_sendctxs_create() + * failure on reconnect, rb_sc_head and rb_sc_tail are stale, + * and slots between them can be NULL or have sc_req clear. + */ + for (i = rpcrdma_sendctx_next(buf, buf->rb_sc_tail); + i != rpcrdma_sendctx_next(buf, buf->rb_sc_head); + i = rpcrdma_sendctx_next(buf, i)) { + struct rpcrdma_sendctx *sc = buf->rb_sc_ctxs[i]; + + if (sc && sc->sc_req) + rpcrdma_sendctx_unmap(sc); + } + for (i = 0; i <= buf->rb_sc_last; i++) kfree(buf->rb_sc_ctxs[i]); kfree(buf->rb_sc_ctxs); @@ -667,6 +708,12 @@ static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, return likely(item < buf->rb_sc_last) ? item + 1 : 0; } +static unsigned long rpcrdma_sendctx_prev(struct rpcrdma_buffer *buf, + unsigned long item) +{ + return item > 0 ? item - 1 : buf->rb_sc_last; +} + /** * rpcrdma_sendctx_get_locked - Acquire a send context * @r_xprt: controlling transport instance @@ -723,6 +770,29 @@ struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt) return NULL; } +/** + * rpcrdma_sendctx_unget_locked - Release an unposted send context + * @r_xprt: controlling transport instance + * @sc: send context to release + * + * Usage: Called when no Send is posted for the sendctx most + * recently returned by rpcrdma_sendctx_get_locked(). + * + * The caller serializes calls to this function and to + * rpcrdma_sendctx_get_locked() (per transport). + */ +void rpcrdma_sendctx_unget_locked(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_sendctx *sc) +{ + struct rpcrdma_buffer *buf = &r_xprt->rx_buf; + + if (WARN_ON_ONCE(buf->rb_sc_ctxs[buf->rb_sc_head] != sc)) + return; + + buf->rb_sc_head = rpcrdma_sendctx_prev(buf, buf->rb_sc_head); + xprt_write_space(&r_xprt->rx_xprt); +} + /** * rpcrdma_sendctx_put_locked - Release a send context * @r_xprt: controlling transport instance @@ -739,15 +809,18 @@ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, struct rpcrdma_buffer *buf = &r_xprt->rx_buf; unsigned long next_tail; - /* Unmap SGEs of previously completed but unsignaled - * Sends by walking up the queue until @sc is found. + /* Release previously completed but unsignaled Sends by walking + * up the queue until @sc is found. */ next_tail = buf->rb_sc_tail; do { + struct rpcrdma_sendctx *cur; + next_tail = rpcrdma_sendctx_next(buf, next_tail); /* ORDER: item must be accessed _before_ tail is updated */ - rpcrdma_sendctx_unmap(buf->rb_sc_ctxs[next_tail]); + cur = buf->rb_sc_ctxs[next_tail]; + rpcrdma_sendctx_unmap(cur); } while (buf->rb_sc_ctxs[next_tail] != sc); @@ -1022,9 +1095,15 @@ static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf) * @buf: buffer pool * @rep: rep to release * + * The rep's transient association with an rpc_rqst, established + * by rpcrdma_reply_handler() and torn down here, must not survive + * onto rb_free_reps: rpcrdma_post_recvs() pulls reps from the free + * list to re-post them, and a non-NULL rr_rqst on a free-listed rep + * would imply the rep is still referenced by a req. */ void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep) { + rep->rr_rqst = NULL; llist_add(&rep->rr_node, &buf->rb_free_reps); } @@ -1059,6 +1138,22 @@ static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) spin_unlock(&buf->rb_lock); } +static unsigned int rpcrdma_req_pool_slack(unsigned int max_reqs) +{ + /* The sendctx ring can hold up to one Send-signaling batch + * (re_send_batch, set by frwr_open() to re_max_requests >> 3) + * of unfinished Sends. Each pins its req until a signaled Send + * completion releases the sendctx. Size the pool above max_reqs + * by that batch so the recycle delay does not stall a slot + * allocation that the RPC/RDMA credit window would admit. + * + * Round up: re_max_requests >> 3 is zero when max_reqs < 8, but + * a single unsignaled Send is still enough to pin one req. One + * slack slot covers that case. + */ + return DIV_ROUND_UP(max_reqs, 8); +} + /** * rpcrdma_buffer_create - Create initial set of req/rep objects * @r_xprt: transport instance to (re)initialize @@ -1068,6 +1163,7 @@ static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; + unsigned int max_reqs; int i, rc; buf->rb_bc_srv_max_requests = 0; @@ -1076,19 +1172,21 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) INIT_LIST_HEAD(&buf->rb_all_mrs); INIT_WORK(&buf->rb_refresh_worker, rpcrdma_mr_refresh_worker); - INIT_LIST_HEAD(&buf->rb_send_bufs); + init_llist_head(&buf->rb_send_bufs); INIT_LIST_HEAD(&buf->rb_allreqs); INIT_LIST_HEAD(&buf->rb_all_reps); rc = -ENOMEM; - for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) { + max_reqs = r_xprt->rx_xprt.max_reqs; + max_reqs += rpcrdma_req_pool_slack(max_reqs); + for (i = 0; i < max_reqs; i++) { struct rpcrdma_req *req; req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2); if (!req) goto out; - list_add(&req->rl_list, &buf->rb_send_bufs); + llist_add(&req->rl_node, &buf->rb_send_bufs); } init_llist_head(&buf->rb_free_reps); @@ -1168,16 +1266,14 @@ static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt) void rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) { - rpcrdma_reps_destroy(buf); + struct rpcrdma_req *req, *next; + struct llist_node *node; - while (!list_empty(&buf->rb_send_bufs)) { - struct rpcrdma_req *req; + rpcrdma_reps_destroy(buf); - req = list_first_entry(&buf->rb_send_bufs, - struct rpcrdma_req, rl_list); - list_del(&req->rl_list); + node = llist_del_all(&buf->rb_send_bufs); + llist_for_each_entry_safe(req, next, node, rl_node) rpcrdma_req_destroy(req); - } } /** @@ -1207,9 +1303,11 @@ rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) */ void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) { - if (req->rl_reply) { - rpcrdma_rep_put(buffers, req->rl_reply); + struct rpcrdma_rep *rep = req->rl_reply; + + if (rep) { req->rl_reply = NULL; + rpcrdma_rep_put(buffers, rep); } } @@ -1222,15 +1320,15 @@ void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) struct rpcrdma_req * rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) { - struct rpcrdma_req *req; + struct llist_node *node; + /* Calls to llist_del_first are required to be serialized */ spin_lock(&buffers->rb_lock); - req = list_first_entry_or_null(&buffers->rb_send_bufs, - struct rpcrdma_req, rl_list); - if (req) - list_del_init(&req->rl_list); + node = llist_del_first(&buffers->rb_send_bufs); spin_unlock(&buffers->rb_lock); - return req; + if (!node) + return NULL; + return llist_entry(node, struct rpcrdma_req, rl_node); } /** @@ -1243,9 +1341,7 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) { rpcrdma_reply_put(buffers, req); - spin_lock(&buffers->rb_lock); - list_add(&req->rl_list, &buffers->rb_send_bufs); - spin_unlock(&buffers->rb_lock); + llist_add(&req->rl_node, &buffers->rb_send_bufs); } /* Returns a pointer to a rpcrdma_regbuf object, or NULL. @@ -1291,10 +1387,16 @@ rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) * returned, @rb is left untouched. */ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) +{ + return rpcrdma_regbuf_realloc_node(rb, size, flags, NUMA_NO_NODE); +} + +static bool rpcrdma_regbuf_realloc_node(struct rpcrdma_regbuf *rb, + size_t size, gfp_t flags, int node) { void *buf; - buf = kmalloc(size, flags); + buf = kmalloc_node(size, flags, node); if (!buf) return false; @@ -1306,6 +1408,23 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) return true; } +static bool rpcrdma_rep_resize(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_rep *rep) +{ + struct rpcrdma_regbuf *rb = rep->rr_rdmabuf; + struct rpcrdma_ep *ep = r_xprt->rx_ep; + size_t size = ep->re_inline_recv; + + if (likely(rdmab_length(rb) >= size)) + return true; + if (!rpcrdma_regbuf_realloc_node(rb, size, XPRTRDMA_GFP_FLAGS, + ibdev_to_node(ep->re_id->device))) + return false; + + xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rb), rdmab_length(rb)); + return true; +} + /** * __rpcrdma_regbuf_dma_map - DMA-map a regbuf * @r_xprt: controlling transport instance @@ -1387,6 +1506,12 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed) rep = rpcrdma_rep_create(r_xprt); if (!rep) break; + /* I1: a rep on rb_free_reps must carry no rqst pointer. */ + WARN_ON_ONCE(rep->rr_rqst); + if (!rpcrdma_rep_resize(r_xprt, rep)) { + rpcrdma_rep_put(buf, rep); + break; + } if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) { rpcrdma_rep_put(buf, rep); break; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index f53a77472724..4cbc941e4a3e 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -332,7 +332,7 @@ enum { struct rpcrdma_buffer; struct rpcrdma_req { - struct list_head rl_list; + struct llist_node rl_node; struct rpc_rqst rl_slot; struct rpcrdma_rep *rl_reply; struct xdr_stream rl_stream; @@ -374,14 +374,14 @@ rpcrdma_mr_pop(struct list_head *list) } /* - * struct rpcrdma_buffer -- holds list/queue of pre-registered memory for - * inline requests/replies, and client/server credits. + * struct rpcrdma_buffer -- holds pre-registered memory for inline + * requests/replies, and client/server credits. * * One of these is associated with a transport instance */ struct rpcrdma_buffer { spinlock_t rb_lock; - struct list_head rb_send_bufs; + struct llist_head rb_send_bufs; struct list_head rb_mrs; unsigned long rb_sc_head; @@ -427,7 +427,6 @@ struct rpcrdma_stats { /* accessed when receiving a reply */ unsigned long long total_rdma_reply; unsigned long long fixup_copy_count; - unsigned long reply_waits_for_send; unsigned long local_inv_needed; unsigned long nomsg_call_count; unsigned long bcall_count; @@ -496,6 +495,8 @@ void rpcrdma_req_destroy(struct rpcrdma_req *req); int rpcrdma_buffer_create(struct rpcrdma_xprt *); void rpcrdma_buffer_destroy(struct rpcrdma_buffer *); struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt); +void rpcrdma_sendctx_unget_locked(struct rpcrdma_xprt *r_xprt, + struct rpcrdma_sendctx *sc); struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt); void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt); @@ -505,6 +506,7 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req); void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep); void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req); +void rpcrdma_req_put(struct rpcrdma_req *req); bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags); diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c index 76a1585d3f6b..10d1ec593084 100644 --- a/net/tipc/bcast.c +++ b/net/tipc/bcast.c @@ -497,12 +497,13 @@ void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, */ int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr, - struct sk_buff_head *retrq) + struct sk_buff_head *retrq, bool *valid) { struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq; struct tipc_gap_ack_blks *ga; struct sk_buff_head xmitq; int rc = 0; + u16 glen; __skb_queue_head_init(&xmitq); @@ -510,13 +511,18 @@ int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, if (msg_type(hdr) != STATE_MSG) { tipc_link_bc_init_rcv(l, hdr); } else if (!msg_bc_ack_invalid(hdr)) { - tipc_get_gap_ack_blks(&ga, l, hdr, false); - if (!sysctl_tipc_bc_retruni) - retrq = &xmitq; - rc = tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr), - msg_bc_gap(hdr), ga, &xmitq, - retrq); - rc |= tipc_link_bc_sync_rcv(l, hdr, &xmitq); + glen = tipc_get_gap_ack_blks(&ga, l, hdr, false); + if (glen > msg_data_sz(hdr)) { + /* Malformed Gap ACK blocks; caller drops the msg */ + *valid = false; + } else { + if (!sysctl_tipc_bc_retruni) + retrq = &xmitq; + rc = tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr), + msg_bc_gap(hdr), ga, &xmitq, + retrq); + rc |= tipc_link_bc_sync_rcv(l, hdr, &xmitq); + } } tipc_bcast_unlock(net); diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h index 2d9352dc7b0e..55d17b5413e1 100644 --- a/net/tipc/bcast.h +++ b/net/tipc/bcast.h @@ -97,7 +97,7 @@ void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr); int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr, - struct sk_buff_head *retrq); + struct sk_buff_head *retrq, bool *valid); int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg, struct tipc_link *bcl); int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]); diff --git a/net/tipc/core.c b/net/tipc/core.c index 434e70eabe08..315975c3be81 100644 --- a/net/tipc/core.c +++ b/net/tipc/core.c @@ -45,6 +45,7 @@ #include "crypto.h" #include +#include /* configurable TIPC parameters */ unsigned int tipc_net_id __read_mostly; @@ -118,8 +119,7 @@ static void __net_exit tipc_exit_net(struct net *net) #ifdef CONFIG_TIPC_CRYPTO tipc_crypto_stop(&tipc_net(net)->crypto_tx); #endif - while (atomic_read(&tn->wq_count)) - cond_resched(); + wait_var_event(&tn->wq_count, atomic_read(&tn->wq_count) == 0); } static void __net_exit tipc_pernet_pre_exit(struct net *net) @@ -218,6 +218,11 @@ static void __exit tipc_exit(void) unregister_pernet_device(&tipc_net_ops); tipc_unregister_sysctl(); + /* TODO: Wait for all timers that called call_rcu() to finish before + * calling rcu_barrier(). + */ + rcu_barrier(); + pr_info("Deactivated\n"); } diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 6d3b6b89b1d1..16f1ed1f6b1b 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -941,12 +941,20 @@ static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead, goto exit; } + /* Get net to avoid freed tipc_crypto when delete namespace */ + if (!maybe_get_net(net)) { + tipc_bearer_put(b); + rc = -ENODEV; + goto exit; + } + /* Now, do decrypt */ rc = crypto_aead_decrypt(req); if (rc == -EINPROGRESS || rc == -EBUSY) return rc; tipc_bearer_put(b); + put_net(net); exit: kfree(ctx); @@ -984,6 +992,7 @@ static void tipc_aead_decrypt_done(void *data, int err) } tipc_bearer_put(b); + put_net(net); } static inline int tipc_ehdr_size(struct tipc_ehdr *ehdr) diff --git a/net/tipc/discover.c b/net/tipc/discover.c index 3e54d2df5683..b9d06595b067 100644 --- a/net/tipc/discover.c +++ b/net/tipc/discover.c @@ -58,6 +58,7 @@ * @skb: request message to be (repeatedly) sent * @timer: timer governing period between requests * @timer_intv: current interval between requests (in ms) + * @rcu: RCU head for deferred freeing */ struct tipc_discoverer { u32 bearer_id; @@ -69,6 +70,7 @@ struct tipc_discoverer { struct sk_buff *skb; struct timer_list timer; unsigned long timer_intv; + struct rcu_head rcu; }; /** @@ -382,6 +384,15 @@ int tipc_disc_create(struct net *net, struct tipc_bearer *b, return 0; } +static void tipc_disc_free_rcu(struct rcu_head *rp) +{ + struct tipc_discoverer *d = container_of(rp, struct tipc_discoverer, + rcu); + + kfree_skb(d->skb); + kfree(d); +} + /** * tipc_disc_delete - destroy object sending periodic link setup requests * @d: ptr to link dest structure @@ -389,8 +400,7 @@ int tipc_disc_create(struct net *net, struct tipc_bearer *b, void tipc_disc_delete(struct tipc_discoverer *d) { timer_shutdown_sync(&d->timer); - kfree_skb(d->skb); - kfree(d); + call_rcu(&d->rcu, tipc_disc_free_rcu); } /** diff --git a/net/tipc/node.c b/net/tipc/node.c index 97aa970a0d83..8e4ef2630ae4 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1831,12 +1831,15 @@ static void tipc_node_mcast_rcv(struct tipc_node *n) } static void tipc_node_bc_sync_rcv(struct tipc_node *n, struct tipc_msg *hdr, - int bearer_id, struct sk_buff_head *xmitq) + int bearer_id, struct sk_buff_head *xmitq, + bool *valid) { struct tipc_link *ucl; int rc; - rc = tipc_bcast_sync_rcv(n->net, n->bc_entry.link, hdr, xmitq); + rc = tipc_bcast_sync_rcv(n->net, n->bc_entry.link, hdr, xmitq, valid); + if (!*valid) + return; if (rc & TIPC_LINK_DOWN_EVT) { tipc_node_reset_links(n); @@ -2140,12 +2143,18 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b) /* Ensure broadcast reception is in synch with peer's send state */ if (unlikely(usr == LINK_PROTOCOL)) { + bool valid = true; + if (unlikely(skb_linearize(skb))) { tipc_node_put(n); goto discard; } hdr = buf_msg(skb); - tipc_node_bc_sync_rcv(n, hdr, bearer_id, &xmitq); + tipc_node_bc_sync_rcv(n, hdr, bearer_id, &xmitq, &valid); + if (!valid) { + tipc_node_put(n); + goto discard; + } } else if (unlikely(tipc_link_acked(n->bc_entry.link) != bc_ack)) { tipc_bcast_ack_rcv(net, n->bc_entry.link, hdr); } diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c index 988b8a7f953a..62ae7f5b5840 100644 --- a/net/tipc/udp_media.c +++ b/net/tipc/udp_media.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -803,6 +804,14 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b, return err; } +static void rcast_free_rcu(struct rcu_head *rcu) +{ + struct udp_replicast *rcast = container_of(rcu, struct udp_replicast, rcu); + + dst_cache_destroy(&rcast->dst_cache); + kfree(rcast); +} + /* cleanup_bearer - break the socket/bearer association */ static void cleanup_bearer(struct work_struct *work) { @@ -811,19 +820,19 @@ static void cleanup_bearer(struct work_struct *work) struct tipc_net *tn; list_for_each_entry_safe(rcast, tmp, &ub->rcast.list, list) { - dst_cache_destroy(&rcast->dst_cache); list_del_rcu(&rcast->list); - kfree_rcu(rcast, rcu); + call_rcu_hurry(&rcast->rcu, rcast_free_rcu); } tn = tipc_net(sock_net(ub->sk)); - dst_cache_destroy(&ub->rcast.dst_cache); udp_tunnel_sock_release(ub->sk); - /* Note: could use a call_rcu() to avoid another synchronize_net() */ synchronize_net(); - atomic_dec(&tn->wq_count); + + dst_cache_destroy(&ub->rcast.dst_cache); + if (atomic_dec_and_test(&tn->wq_count)) + wake_up_var(&tn->wq_count); kfree(ub); } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index f7a9d55eee8a..92a768cc9ecf 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1196,17 +1196,12 @@ static struct sock *unix_find_bsd(struct sockaddr_un *sunaddr, int addr_len, unix_mkname_bsd(sunaddr, addr_len); if (flags & SOCK_COREDUMP) { - struct path root; - - task_lock(&init_task); - get_fs_root(init_task.fs, &root); - task_unlock(&init_task); - - scoped_with_kernel_creds() - err = vfs_path_lookup(root.dentry, root.mnt, sunaddr->sun_path, - LOOKUP_BENEATH | LOOKUP_NO_SYMLINKS | - LOOKUP_NO_MAGICLINKS, &path); - path_put(&root); + scoped_with_init_fs() { + scoped_with_kernel_creds() + err = kern_path(sunaddr->sun_path, + LOOKUP_NO_SYMLINKS | + LOOKUP_NO_MAGICLINKS, &path); + } if (err) goto fail; } else { diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 09475007165b..41c2a0b82a8e 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -328,38 +328,35 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) return pkt_len; - if (info->msg) { - /* If zerocopy is not enabled by 'setsockopt()', we behave as - * there is no MSG_ZEROCOPY flag set. + if (info->msg && (info->msg->msg_flags & MSG_ZEROCOPY)) { + /* If 'info->msg' is not NULL, this is only VIRTIO_VSOCK_OP_RW. + * 'MSG_ZEROCOPY' flag handling here is based on the same flag + * handling from 'tcp_sendmsg_locked()'. */ - if (!sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY)) - info->msg->msg_flags &= ~MSG_ZEROCOPY; + if (info->msg->msg_ubuf) { + uarg = info->msg->msg_ubuf; + can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len); + } else if (sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY)) { + uarg = msg_zerocopy_realloc(sk_vsock(vsk), pkt_len, + NULL, false); + if (!uarg) { + virtio_transport_put_credit(vvs, pkt_len); + return -ENOMEM; + } - if (info->msg->msg_flags & MSG_ZEROCOPY) can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len); + if (!can_zcopy) + uarg_to_msgzc(uarg)->zerocopy = 0; + have_uref = true; + } + + /* 'can_zcopy' means that this transmission will be + * in zerocopy way (e.g. using 'frags' array). + */ if (can_zcopy) max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE, (MAX_SKB_FRAGS * PAGE_SIZE)); - - if (info->msg->msg_flags & MSG_ZEROCOPY && - info->op == VIRTIO_VSOCK_OP_RW) { - uarg = info->msg->msg_ubuf; - - if (!uarg) { - uarg = msg_zerocopy_realloc(sk_vsock(vsk), - pkt_len, NULL, false); - if (!uarg) { - virtio_transport_put_credit(vvs, pkt_len); - return -ENOMEM; - } - - if (!can_zcopy) - uarg_to_msgzc(uarg)->zerocopy = 0; - - have_uref = true; - } - } } rest_len = pkt_len; diff --git a/net/wireless/core.c b/net/wireless/core.c index 3dcf63b04c41..082f0ee12f1b 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -1335,6 +1335,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->sched_scan_res_wk); cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); @@ -1613,7 +1614,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev) INIT_LIST_HEAD(&wdev->mgmt_registrations); INIT_LIST_HEAD(&wdev->pmsr_list); spin_lock_init(&wdev->pmsr_lock); - INIT_WORK(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); + wiphy_work_init(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); #ifdef CONFIG_CFG80211_WEXT wdev->wext.default_key = -1; @@ -1747,7 +1748,6 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, cfg80211_remove_links(wdev); /* since we just did cfg80211_leave() nothing to do there */ cancel_work_sync(&wdev->disconnect_wk); - cancel_work_sync(&wdev->pmsr_free_wk); break; case NETDEV_DOWN: wiphy_lock(&rdev->wiphy); diff --git a/net/wireless/core.h b/net/wireless/core.h index df47ed6208a5..f60c66b88677 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -586,7 +586,7 @@ cfg80211_get_6ghz_power_type(const u8 *elems, size_t elems_len, void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid); void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev); -void cfg80211_pmsr_free_wk(struct work_struct *work); +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work); void cfg80211_remove_link(struct wireless_dev *wdev, unsigned int link_id); void cfg80211_remove_links(struct wireless_dev *wdev); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 53b4b3f76697..d354eb901c52 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -461,7 +461,9 @@ nl80211_ftm_responder_policy[NL80211_FTM_RESP_ATTR_MAX + 1] = { static const struct nla_policy nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = { [NL80211_PMSR_FTM_REQ_ATTR_ASAP] = { .type = NLA_FLAG }, - [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = { .type = NLA_U32 }, + [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = + NLA_POLICY_RANGE(NLA_U32, NL80211_PREAMBLE_LEGACY, + NL80211_PREAMBLE_HE), [NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP] = NLA_POLICY_MAX(NLA_U8, 15), [NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD] = { .type = NLA_U16 }, @@ -630,7 +632,7 @@ nl80211_mbssid_config_policy[NL80211_MBSSID_CONFIG_ATTR_MAX + 1] = { [NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX] = { .type = NLA_U32 }, [NL80211_MBSSID_CONFIG_ATTR_EMA] = { .type = NLA_FLAG }, [NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID] = - NLA_POLICY_MAX(NLA_U8, IEEE80211_MLD_MAX_NUM_LINKS), + NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1), }; static const struct nla_policy @@ -1093,6 +1095,7 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_NPCA_PRIMARY_FREQ] = { .type = NLA_U32 }, [NL80211_ATTR_NPCA_PUNCT_BITMAP] = NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range), + [NL80211_ATTR_STA_DUMP_LINK_STATS] = { .type = NLA_FLAG }, }; /* policy for the key attributes */ @@ -2444,7 +2447,7 @@ static int nl80211_add_commands_unsplit(struct cfg80211_registered_device *rdev, } if (rdev->wiphy.max_sched_scan_reqs) CMD(sched_scan_start, START_SCHED_SCAN); - CMD(probe_client, PROBE_CLIENT); + CMD(probe_peer, PROBE_PEER); CMD(set_noack_map, SET_NOACK_MAP); if (rdev->wiphy.flags & WIPHY_FLAG_REPORTS_OBSS) { i++; @@ -6510,7 +6513,8 @@ static int nl80211_parse_mbssid_config(struct wiphy *wiphy, } static struct cfg80211_mbssid_elems * -nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) +nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs, + struct netlink_ext_ack *extack) { struct nlattr *nl_elems; struct cfg80211_mbssid_elems *elems; @@ -6521,6 +6525,12 @@ nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) return ERR_PTR(-EINVAL); nla_for_each_nested(nl_elems, attrs, rem_elems) { + int ret; + + ret = validate_ie_attr(nl_elems, extack); + if (ret) + return ERR_PTR(ret); + if (num_elems >= 255) return ERR_PTR(-EINVAL); num_elems++; @@ -6787,7 +6797,8 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (attrs[NL80211_ATTR_MBSSID_ELEMS]) { struct cfg80211_mbssid_elems *mbssid = nl80211_parse_mbssid_elems(&rdev->wiphy, - attrs[NL80211_ATTR_MBSSID_ELEMS]); + attrs[NL80211_ATTR_MBSSID_ELEMS], + extack); if (IS_ERR(mbssid)) return PTR_ERR(mbssid); @@ -6803,8 +6814,10 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (IS_ERR(rnr)) return PTR_ERR(rnr); - if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) + if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) { + kfree(rnr); return -EINVAL; + } bcn->rnr_ies = rnr; } @@ -7870,7 +7883,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg, goto nla_put_failure; \ } while (0) - link_sinfoattr = nla_nest_start_noflag(msg, NL80211_ATTR_STA_INFO); + link_sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO); if (!link_sinfoattr) goto nla_put_failure; @@ -7936,8 +7949,8 @@ static int nl80211_fill_link_station(struct sk_buff *msg, PUT_LINK_SINFO(BEACON_LOSS, beacon_loss_count, u32); if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) { - bss_param = nla_nest_start_noflag(msg, - NL80211_STA_INFO_BSS_PARAM); + bss_param = nla_nest_start(msg, + NL80211_STA_INFO_BSS_PARAM); if (!bss_param) goto nla_put_failure; @@ -7979,8 +7992,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg, struct nlattr *tidsattr; int tid; - tidsattr = nla_nest_start_noflag(msg, - NL80211_STA_INFO_TID_STATS); + tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS); if (!tidsattr) goto nla_put_failure; @@ -7993,7 +8005,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg, if (!tidstats->filled) continue; - tidattr = nla_nest_start_noflag(msg, tid + 1); + tidattr = nla_nest_start(msg, tid + 1); if (!tidattr) goto nla_put_failure; @@ -8029,36 +8041,15 @@ static int nl80211_fill_link_station(struct sk_buff *msg, return -EMSGSIZE; } -static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, - u32 seq, int flags, - struct cfg80211_registered_device *rdev, - struct wireless_dev *wdev, - const u8 *mac_addr, struct station_info *sinfo, - bool link_stats) +static int nl80211_put_sta_info_common(struct sk_buff *msg, + struct cfg80211_registered_device *rdev, + struct station_info *sinfo) { - void *hdr; struct nlattr *sinfoattr, *bss_param; - struct link_station_info *link_sinfo; - struct nlattr *links, *link; - int link_id; - - hdr = nl80211hdr_put(msg, portid, seq, flags, cmd); - if (!hdr) { - cfg80211_sinfo_release_content(sinfo); - return -1; - } - - if ((wdev->netdev && - nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) || - nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev), - NL80211_ATTR_PAD) || - nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr) || - nla_put_u32(msg, NL80211_ATTR_GENERATION, sinfo->generation)) - goto nla_put_failure; - sinfoattr = nla_nest_start_noflag(msg, NL80211_ATTR_STA_INFO); + sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO); if (!sinfoattr) - goto nla_put_failure; + return -EMSGSIZE; #define PUT_SINFO(attr, memb, type) do { \ BUILD_BUG_ON(sizeof(type) == sizeof(u64)); \ @@ -8149,8 +8140,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, PUT_SINFO_U64(T_OFFSET, t_offset); if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) { - bss_param = nla_nest_start_noflag(msg, - NL80211_STA_INFO_BSS_PARAM); + bss_param = nla_nest_start(msg, NL80211_STA_INFO_BSS_PARAM); if (!bss_param) goto nla_put_failure; @@ -8192,8 +8182,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, struct nlattr *tidsattr; int tid; - tidsattr = nla_nest_start_noflag(msg, - NL80211_STA_INFO_TID_STATS); + tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS); if (!tidsattr) goto nla_put_failure; @@ -8206,7 +8195,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, if (!tidstats->filled) continue; - tidattr = nla_nest_start_noflag(msg, tid + 1); + tidattr = nla_nest_start(msg, tid + 1); if (!tidattr) goto nla_put_failure; @@ -8236,6 +8225,37 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, } nla_nest_end(msg, sinfoattr); + return 0; + +nla_put_failure: + nla_nest_cancel(msg, sinfoattr); + return -EMSGSIZE; +} + +static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, + u32 seq, int flags, + struct cfg80211_registered_device *rdev, + struct wireless_dev *wdev, + const u8 *mac_addr, struct station_info *sinfo) +{ + void *hdr; + + hdr = nl80211hdr_put(msg, portid, seq, flags, cmd); + if (!hdr) { + cfg80211_sinfo_release_content(sinfo); + return -1; + } + + if ((wdev->netdev && + nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) || + nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev), + NL80211_ATTR_PAD) || + nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr) || + nla_put_u32(msg, NL80211_ATTR_GENERATION, sinfo->generation)) + goto nla_put_failure; + + if (nl80211_put_sta_info_common(msg, rdev, sinfo)) + goto nla_put_failure; if (sinfo->assoc_req_ies_len && nla_put(msg, NL80211_ATTR_IE, sinfo->assoc_req_ies_len, @@ -8258,45 +8278,11 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid, goto nla_put_failure; } - if (link_stats && sinfo->valid_links) { - links = nla_nest_start(msg, NL80211_ATTR_MLO_LINKS); - if (!links) - goto nla_put_failure; - - for_each_valid_link(sinfo, link_id) { - link_sinfo = sinfo->links[link_id]; - - if (WARN_ON_ONCE(!link_sinfo)) - continue; - - if (!is_valid_ether_addr(link_sinfo->addr)) - continue; - - link = nla_nest_start(msg, link_id + 1); - if (!link) - goto nla_put_failure; - - if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, - link_id)) - goto nla_put_failure; - - if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, - link_sinfo->addr)) - goto nla_put_failure; - - if (nl80211_fill_link_station(msg, rdev, link_sinfo)) - goto nla_put_failure; - - nla_nest_end(msg, link); - } - nla_nest_end(msg, links); - } - cfg80211_sinfo_release_content(sinfo); genlmsg_end(msg, hdr); return 0; - nla_put_failure: +nla_put_failure: cfg80211_sinfo_release_content(sinfo); genlmsg_cancel(msg, hdr); return -EMSGSIZE; @@ -8490,82 +8476,261 @@ static void cfg80211_sta_set_mld_sinfo(struct station_info *sinfo) sinfo->filled &= ~BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG); } +enum nl80211_dump_station_phase { + NL80211_DUMP_STA_PHASE_AGGREGATED = 0, + NL80211_DUMP_STA_PHASE_PER_LINK = 1, +}; + +struct nl80211_dump_station_ctx { + int sta_idx; + int link_idx; + enum nl80211_dump_station_phase phase; + bool dump_link_stats; + bool filter_mac; + u8 filter_mac_addr[ETH_ALEN]; + u8 mac_addr[ETH_ALEN]; + struct station_info sinfo; +}; + +static int nl80211_put_link_station_payload(struct sk_buff *msg, + struct cfg80211_registered_device *rdev, + struct station_info *sinfo, + int link_idx) +{ + struct link_station_info *link_sinfo = sinfo->links[link_idx]; + struct nlattr *links, *link; + + if (WARN_ON_ONCE(!link_sinfo)) + return -ENOENT; + + if (!is_valid_ether_addr(link_sinfo->addr)) + return -EADDRNOTAVAIL; + + links = nla_nest_start(msg, NL80211_ATTR_MLO_LINKS); + if (!links) + return -EMSGSIZE; + + link = nla_nest_start(msg, link_idx + 1); + if (!link) + goto nla_put_failure; + + if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_idx) || + nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, link_sinfo->addr)) + goto nla_put_failure; + + if (nl80211_fill_link_station(msg, rdev, link_sinfo)) + goto nla_put_failure; + + nla_nest_end(msg, link); + nla_nest_end(msg, links); + return 0; + +nla_put_failure: + nla_nest_cancel(msg, links); + return -EMSGSIZE; +} + static int nl80211_dump_station(struct sk_buff *skb, struct netlink_callback *cb) { - struct station_info sinfo; struct cfg80211_registered_device *rdev; struct wireless_dev *wdev; - u8 mac_addr[ETH_ALEN]; - int sta_idx = cb->args[2]; - bool sinfo_alloc = false; - int err, i; + struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2]; + struct nlattr **attrbuf __free(kfree) = NULL; + int err; - err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL); + if (!ctx) { + attrbuf = kzalloc_objs(*attrbuf, NUM_NL80211_ATTR); + if (!attrbuf) + return -ENOMEM; + } + + err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, attrbuf); if (err) return err; /* nl80211_prepare_wdev_dump acquired it in the successful case */ __acquire(&rdev->wiphy.mtx); + if (!ctx) { + ctx = kzalloc_obj(*ctx); + if (!ctx) { + err = -ENOMEM; + goto out_err; + } + cb->args[2] = (long)ctx; + ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED; + ctx->dump_link_stats = + !!attrbuf[NL80211_ATTR_STA_DUMP_LINK_STATS]; + if (attrbuf[NL80211_ATTR_MAC]) { + const u8 *mac = nla_data(attrbuf[NL80211_ATTR_MAC]); + + if (!is_valid_ether_addr(mac)) { + kfree(ctx); + cb->args[2] = 0; + err = -EINVAL; + goto out_err; + } + ctx->filter_mac = true; + memcpy(ctx->filter_mac_addr, mac, ETH_ALEN); + } + } + if (!wdev->netdev && wdev->iftype != NL80211_IFTYPE_NAN) { err = -EINVAL; goto out_err; } - if (!rdev->ops->dump_station) { + if (ctx->filter_mac) { + if (!rdev->ops->get_station) { + err = -EOPNOTSUPP; + goto out_err; + } + } else if (!rdev->ops->dump_station) { err = -EOPNOTSUPP; goto out_err; } - while (1) { - memset(&sinfo, 0, sizeof(sinfo)); + while (true) { + void *hdr; + int ret; - for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) { - sinfo.links[i] = - kzalloc_obj(*sinfo.links[0]); - if (!sinfo.links[i]) { - err = -ENOMEM; - goto out_err; + /* AGGREGATED phase: fetch sinfo from driver once per station */ + if (ctx->phase == NL80211_DUMP_STA_PHASE_AGGREGATED) { + memset(&ctx->sinfo, 0, sizeof(ctx->sinfo)); + for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) { + ctx->sinfo.links[i] = + kzalloc_obj(*ctx->sinfo.links[0]); + if (!ctx->sinfo.links[i]) { + err = -ENOMEM; + goto out_err_release; + } } - sinfo_alloc = true; - } - err = rdev_dump_station(rdev, wdev, sta_idx, - mac_addr, &sinfo); - if (err == -ENOENT) - break; - if (err) - goto out_err; + if (ctx->filter_mac) { + if (ctx->sta_idx > 0) { + err = skb->len; + goto out_err_release; + } + err = rdev_get_station(rdev, wdev, + ctx->filter_mac_addr, + &ctx->sinfo); + if (!err) + memcpy(ctx->mac_addr, + ctx->filter_mac_addr, ETH_ALEN); + } else { + err = rdev_dump_station(rdev, wdev, ctx->sta_idx, + ctx->mac_addr, + &ctx->sinfo); + } + if (err == -ENOENT) { + err = skb->len; + goto out_err_release; + } + if (err) + goto out_err_release; - if (sinfo.valid_links) - cfg80211_sta_set_mld_sinfo(&sinfo); + if (ctx->sinfo.valid_links) + cfg80211_sta_set_mld_sinfo(&ctx->sinfo); + } else { + /* PER_LINK phase: advance to next valid link */ + while (ctx->link_idx < IEEE80211_MLD_MAX_NUM_LINKS && + !(ctx->sinfo.valid_links & BIT(ctx->link_idx))) + ctx->link_idx++; + + if (ctx->link_idx >= IEEE80211_MLD_MAX_NUM_LINKS) { + cfg80211_sinfo_release_content(&ctx->sinfo); + ctx->sta_idx++; + ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED; + continue; + } + } - /* reset the sinfo_alloc flag as nl80211_send_station() - * always releases sinfo - */ - sinfo_alloc = false; + /* Build common header for both phases */ + hdr = nl80211hdr_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NLM_F_MULTI, + NL80211_CMD_NEW_STATION); + if (!hdr) { + err = skb->len; + if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK) + goto out_err; + goto out_err_release; + } - if (nl80211_send_station(skb, NL80211_CMD_NEW_STATION, - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, NLM_F_MULTI, - rdev, wdev, mac_addr, - &sinfo, false) < 0) - goto out; + if ((wdev->netdev && + nla_put_u32(skb, NL80211_ATTR_IFINDEX, + wdev->netdev->ifindex)) || + nla_put_u64_64bit(skb, NL80211_ATTR_WDEV, + wdev_id(wdev), NL80211_ATTR_PAD) || + nla_put(skb, NL80211_ATTR_MAC, ETH_ALEN, ctx->mac_addr) || + nla_put_u32(skb, NL80211_ATTR_GENERATION, + ctx->sinfo.generation)) { + genlmsg_cancel(skb, hdr); + err = skb->len; + if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK) + goto out_err; + goto out_err_release; + } + + switch (ctx->phase) { + case NL80211_DUMP_STA_PHASE_AGGREGATED: + ret = nl80211_put_sta_info_common(skb, rdev, &ctx->sinfo); + if (ret) { + genlmsg_cancel(skb, hdr); + err = ret; + goto out_err_release; + } + genlmsg_end(skb, hdr); + + if (ctx->dump_link_stats && ctx->sinfo.valid_links) { + ctx->phase = NL80211_DUMP_STA_PHASE_PER_LINK; + ctx->link_idx = 0; + } else { + cfg80211_sinfo_release_content(&ctx->sinfo); + ctx->sta_idx++; + } + break; - sta_idx++; + case NL80211_DUMP_STA_PHASE_PER_LINK: + ret = nl80211_put_link_station_payload(skb, rdev, + &ctx->sinfo, + ctx->link_idx); + if (ret == -EMSGSIZE) { + genlmsg_cancel(skb, hdr); + err = skb->len; + goto out_err; + } + if (ret) { + /* skip invalid link, do not abort the dump */ + genlmsg_cancel(skb, hdr); + ctx->link_idx++; + continue; + } + genlmsg_end(skb, hdr); + ctx->link_idx++; + break; + } } - out: - cb->args[2] = sta_idx; - err = skb->len; - out_err: - if (sinfo_alloc) - cfg80211_sinfo_release_content(&sinfo); +out_err_release: + cfg80211_sinfo_release_content(&ctx->sinfo); + memset(&ctx->sinfo, 0, sizeof(ctx->sinfo)); +out_err: wiphy_unlock(&rdev->wiphy); return err; } +static int nl80211_dump_station_done(struct netlink_callback *cb) +{ + struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2]; + + if (ctx) { + cfg80211_sinfo_release_content(&ctx->sinfo); + kfree(ctx); + } + return 0; +} + static int nl80211_get_station(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; @@ -8613,7 +8778,7 @@ static int nl80211_get_station(struct sk_buff *skb, struct genl_info *info) if (nl80211_send_station(msg, NL80211_CMD_NEW_STATION, info->snd_portid, info->snd_seq, 0, - rdev, wdev, mac_addr, &sinfo, false) < 0) { + rdev, wdev, mac_addr, &sinfo) < 0) { nlmsg_free(msg); return -ENOBUFS; } @@ -16150,26 +16315,41 @@ static int nl80211_register_unexpected_frame(struct sk_buff *skb, return 0; } -static int nl80211_probe_client(struct sk_buff *skb, - struct genl_info *info) +static int nl80211_probe_peer(struct sk_buff *skb, struct genl_info *info) { struct cfg80211_registered_device *rdev = info->user_ptr[0]; struct net_device *dev = info->user_ptr[1]; struct wireless_dev *wdev = dev->ieee80211_ptr; struct sk_buff *msg; void *hdr; - const u8 *addr; + const u8 *addr = NULL; u64 cookie; int err; - if (wdev->iftype != NL80211_IFTYPE_AP && - wdev->iftype != NL80211_IFTYPE_P2P_GO) + /* Allow in AP, STA, and their P2P counterparts */ + switch (wdev->iftype) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_P2P_GO: + if (!info->attrs[NL80211_ATTR_MAC]) + return -EINVAL; + addr = nla_data(info->attrs[NL80211_ATTR_MAC]); + break; + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_PROBE_AP)) + return -EOPNOTSUPP; + if (!wdev->connected) + return -ENOLINK; + /* STA/P2P-client probes the currently associated AP/GO. */ + if (info->attrs[NL80211_ATTR_MAC]) + return -EINVAL; + break; + default: return -EOPNOTSUPP; + } - if (!info->attrs[NL80211_ATTR_MAC]) - return -EINVAL; - - if (!rdev->ops->probe_client) + if (!rdev->ops->probe_peer) return -EOPNOTSUPP; msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); @@ -16177,15 +16357,13 @@ static int nl80211_probe_client(struct sk_buff *skb, return -ENOMEM; hdr = nl80211hdr_put(msg, info->snd_portid, info->snd_seq, 0, - NL80211_CMD_PROBE_CLIENT); + NL80211_CMD_PROBE_PEER); if (!hdr) { err = -ENOBUFS; goto free_msg; } - addr = nla_data(info->attrs[NL80211_ATTR_MAC]); - - err = rdev_probe_client(rdev, dev, addr, &cookie); + err = rdev_probe_peer(rdev, dev, addr, &cookie); if (err) goto free_msg; @@ -19530,6 +19708,14 @@ static const struct genl_ops nl80211_ops[] = { /* can be retrieved by unprivileged users */ .internal_flags = IFLAGS(NL80211_FLAG_NEED_WIPHY), }, + { + .cmd = NL80211_CMD_GET_STATION, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = nl80211_get_station, + .dumpit = nl80211_dump_station, + .done = nl80211_dump_station_done, + .internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV), + }, }; static const struct genl_small_ops nl80211_small_ops[] = { @@ -19629,13 +19815,6 @@ static const struct genl_small_ops nl80211_small_ops[] = { .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP | NL80211_FLAG_MLO_VALID_LINK_ID), }, - { - .cmd = NL80211_CMD_GET_STATION, - .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .doit = nl80211_get_station, - .dumpit = nl80211_dump_station, - .internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV), - }, { .cmd = NL80211_CMD_SET_STATION, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, @@ -20042,9 +20221,9 @@ static const struct genl_small_ops nl80211_small_ops[] = { .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV), }, { - .cmd = NL80211_CMD_PROBE_CLIENT, + .cmd = NL80211_CMD_PROBE_PEER, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .doit = nl80211_probe_client, + .doit = nl80211_probe_peer, .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), }, @@ -20890,6 +21069,9 @@ void cfg80211_rx_unprot_mlme_mgmt(struct net_device *dev, const u8 *buf, } else if (ieee80211_is_disassoc(mgmt->frame_control)) { event.cmd = NL80211_CMD_UNPROT_DISASSOCIATE; } else if (ieee80211_is_beacon(mgmt->frame_control)) { + if (wdev->iftype == NL80211_IFTYPE_AP || + wdev->iftype == NL80211_IFTYPE_P2P_GO) + return; if (wdev->unprot_beacon_reported && elapsed_jiffies_msecs(wdev->unprot_beacon_reported) < 10000) return; @@ -21635,7 +21817,7 @@ void cfg80211_new_sta(struct wireless_dev *wdev, const u8 *mac_addr, return; if (nl80211_send_station(msg, NL80211_CMD_NEW_STATION, 0, 0, 0, - rdev, wdev, mac_addr, sinfo, false) < 0) { + rdev, wdev, mac_addr, sinfo) < 0) { nlmsg_free(msg); return; } @@ -21665,7 +21847,7 @@ void cfg80211_del_sta_sinfo(struct wireless_dev *wdev, const u8 *mac_addr, } if (nl80211_send_station(msg, NL80211_CMD_DEL_STATION, 0, 0, 0, - rdev, wdev, mac_addr, sinfo, false) < 0) { + rdev, wdev, mac_addr, sinfo) < 0) { nlmsg_free(msg); return; } @@ -22598,8 +22780,8 @@ void cfg80211_sta_opmode_change_notify(struct net_device *dev, const u8 *mac, } EXPORT_SYMBOL(cfg80211_sta_opmode_change_notify); -void cfg80211_probe_status(struct net_device *dev, const u8 *addr, - u64 cookie, bool acked, s32 ack_signal, +void cfg80211_probe_status(struct net_device *dev, const u8 *peer, u64 cookie, + int link_id, bool acked, s32 ack_signal, bool is_valid_ack_signal, gfp_t gfp) { struct wireless_dev *wdev = dev->ieee80211_ptr; @@ -22607,14 +22789,14 @@ void cfg80211_probe_status(struct net_device *dev, const u8 *addr, struct sk_buff *msg; void *hdr; - trace_cfg80211_probe_status(dev, addr, cookie, acked); + trace_cfg80211_probe_status(dev, peer, cookie, acked); msg = nlmsg_new(NLMSG_DEFAULT_SIZE, gfp); if (!msg) return; - hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_PROBE_CLIENT); + hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_PROBE_PEER); if (!hdr) { nlmsg_free(msg); return; @@ -22622,12 +22804,18 @@ void cfg80211_probe_status(struct net_device *dev, const u8 *addr, if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) || nla_put_u32(msg, NL80211_ATTR_IFINDEX, dev->ifindex) || - nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, addr) || + (peer && nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, peer)) || nla_put_u64_64bit(msg, NL80211_ATTR_COOKIE, cookie, - NL80211_ATTR_PAD) || - (acked && nla_put_flag(msg, NL80211_ATTR_ACK)) || - (is_valid_ack_signal && nla_put_s32(msg, NL80211_ATTR_ACK_SIGNAL, - ack_signal))) + NL80211_ATTR_PAD)) + goto nla_put_failure; + + if (link_id >= 0 && + nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id)) + goto nla_put_failure; + + if ((acked && nla_put_flag(msg, NL80211_ATTR_ACK)) || + (is_valid_ack_signal && + nla_put_s32(msg, NL80211_ATTR_ACK_SIGNAL, ack_signal))) goto nla_put_failure; genlmsg_end(msg, hdr); diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index c8447448f3a5..34c3625f7fd5 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -125,6 +125,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI], "FTM: LCI request not supported"); + return -EOPNOTSUPP; } out->ftm.request_civicloc = @@ -133,6 +134,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC], "FTM: civic location request not supported"); + return -EOPNOTSUPP; } out->ftm.trigger_based = @@ -310,6 +312,7 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, { struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1]; struct nlattr *req[NL80211_PMSR_REQ_ATTR_MAX + 1]; + bool have_measurement_type = false; struct nlattr *treq; int err, rem; @@ -376,6 +379,14 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, } nla_for_each_nested(treq, req[NL80211_PMSR_REQ_ATTR_DATA], rem) { + if (have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, treq, + "multiple measurement types in request data"); + return -EINVAL; + } + + have_measurement_type = true; + switch (nla_type(treq)) { case NL80211_PMSR_TYPE_FTM: err = pmsr_parse_ftm(rdev, treq, out, info); @@ -385,10 +396,16 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, "unsupported measurement type"); err = -EINVAL; } + if (err) + return err; } - if (err) - return err; + if (!have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, + req[NL80211_PMSR_REQ_ATTR_DATA], + "missing measurement type in request data"); + return -EINVAL; + } return 0; } @@ -427,6 +444,11 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info) } } + if (!count) { + NL_SET_ERR_MSG_ATTR(info->extack, peers, "No peers specified"); + return -EINVAL; + } + req = kzalloc_flex(*req, peers, count); if (!req) return -ENOMEM; @@ -807,13 +829,11 @@ static void cfg80211_pmsr_process_abort(struct wireless_dev *wdev) } } -void cfg80211_pmsr_free_wk(struct work_struct *work) +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work) { struct wireless_dev *wdev = container_of(work, struct wireless_dev, pmsr_free_wk); - guard(wiphy)(wdev->wiphy); - cfg80211_pmsr_process_abort(wdev); } @@ -829,7 +849,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev) } spin_unlock_bh(&wdev->pmsr_lock); - cancel_work_sync(&wdev->pmsr_free_wk); + wiphy_work_cancel(wdev->wiphy, &wdev->pmsr_free_wk); if (found) cfg80211_pmsr_process_abort(wdev); @@ -844,7 +864,7 @@ void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid) list_for_each_entry(req, &wdev->pmsr_list, list) { if (req->nl_portid == portid) { req->nl_portid = 0; - schedule_work(&wdev->pmsr_free_wk); + wiphy_work_queue(wdev->wiphy, &wdev->pmsr_free_wk); } } spin_unlock_bh(&wdev->pmsr_lock); diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index 63c26e8b1139..6c3bad8b2d6f 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -948,13 +948,13 @@ static inline int rdev_tdls_oper(struct cfg80211_registered_device *rdev, return ret; } -static inline int rdev_probe_client(struct cfg80211_registered_device *rdev, - struct net_device *dev, const u8 *peer, - u64 *cookie) +static inline int rdev_probe_peer(struct cfg80211_registered_device *rdev, + struct net_device *dev, const u8 *peer, + u64 *cookie) { int ret; - trace_rdev_probe_client(&rdev->wiphy, dev, peer); - ret = rdev->ops->probe_client(&rdev->wiphy, dev, peer, cookie); + trace_rdev_probe_peer(&rdev->wiphy, dev, peer); + ret = rdev->ops->probe_peer(&rdev->wiphy, dev, peer, cookie); trace_rdev_return_int_cookie(&rdev->wiphy, ret, *cookie); return ret; } diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 05b7dc6b766c..5c97b5bd5d69 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -326,8 +326,11 @@ cfg80211_gen_new_ie(const u8 *ie, size_t ielen, /* For ML probe response, match the MLE in the frame body with * MLD id being 'bssid_index' */ - if (parent->id == WLAN_EID_EXTENSION && parent->datalen > 1 && + if (parent->id == WLAN_EID_EXTENSION && parent->data[0] == WLAN_EID_EXT_EHT_MULTI_LINK && + ieee80211_mle_type_ok(parent->data + 1, + IEEE80211_ML_CONTROL_TYPE_BASIC, + parent->datalen - 1) && bssid_index == ieee80211_mle_get_mld_id(parent->data + 1)) { if (!cfg80211_copy_elem_with_frags(parent, ie, ielen, @@ -3311,14 +3314,15 @@ cfg80211_inform_bss_frame_data(struct wiphy *wiphy, bssid = ext->u.s1g_beacon.sa; capability = le16_to_cpu(compat->compat_info); beacon_interval = le16_to_cpu(compat->beacon_int); + tsf = le32_to_cpu(ext->u.s1g_beacon.timestamp); + tsf |= (u64)le32_to_cpu(compat->tsf_completion) << 32; } else { bssid = mgmt->bssid; beacon_interval = le16_to_cpu(mgmt->u.probe_resp.beacon_int); capability = le16_to_cpu(mgmt->u.probe_resp.capab_info); + tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); } - tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); - if (ieee80211_is_probe_resp(mgmt->frame_control)) ftype = CFG80211_BSS_FTYPE_PRESP; else if (ext) @@ -3612,8 +3616,10 @@ int cfg80211_wext_siwscan(struct net_device *dev, /* translate "Scan for SSID" request */ if (wreq) { if (wrqu->data.flags & IW_SCAN_THIS_ESSID) { - if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) - return -EINVAL; + if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) { + err = -EINVAL; + goto out; + } memcpy(creq->req.ssids[0].ssid, wreq->essid, wreq->essid_len); creq->req.ssids[0].ssid_len = wreq->essid_len; diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 94944f2a39a4..8c2a91b85c39 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -2132,7 +2132,7 @@ DECLARE_EVENT_CLASS(rdev_pmksa, WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->bssid) ); -TRACE_EVENT(rdev_probe_client, +TRACE_EVENT(rdev_probe_peer, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, const u8 *peer), TP_ARGS(wiphy, netdev, peer), diff --git a/net/wireless/wext-core.c b/net/wireless/wext-core.c index c19dece2bc6e..db77912b3994 100644 --- a/net/wireless/wext-core.c +++ b/net/wireless/wext-core.c @@ -660,8 +660,7 @@ struct iw_statistics *get_wireless_stats(struct net_device *dev) dev->ieee80211_ptr->wiphy->wext && dev->ieee80211_ptr->wiphy->wext->get_wireless_stats) { wireless_warn_cfg80211_wext(); - if (dev->ieee80211_ptr->wiphy->flags & (WIPHY_FLAG_SUPPORTS_MLO | - WIPHY_FLAG_DISABLE_WEXT)) + if (dev->ieee80211_ptr->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO) return NULL; return dev->ieee80211_ptr->wiphy->wext->get_wireless_stats(dev); } @@ -703,8 +702,7 @@ static iw_handler get_handler(struct net_device *dev, unsigned int cmd) #ifdef CONFIG_CFG80211_WEXT if (dev->ieee80211_ptr && dev->ieee80211_ptr->wiphy) { wireless_warn_cfg80211_wext(); - if (dev->ieee80211_ptr->wiphy->flags & (WIPHY_FLAG_SUPPORTS_MLO | - WIPHY_FLAG_DISABLE_WEXT)) + if (dev->ieee80211_ptr->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO) return NULL; handlers = dev->ieee80211_ptr->wiphy->wext; } diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index d9035546375e..374e1b964438 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -212,43 +212,23 @@ static int espintcp_sendskmsg_locked(struct sock *sk, struct sk_msg *skmsg = &emsg->skmsg; bool more = flags & MSG_MORE; struct scatterlist *sg; - int done = 0; int ret; - sg = &skmsg->sg.data[skmsg->sg.start]; do { struct bio_vec bvec; - size_t size = sg->length - emsg->offset; - int offset = sg->offset + emsg->offset; - struct page *p; - - emsg->offset = 0; + sg = &skmsg->sg.data[skmsg->sg.start]; if (sg_is_last(sg) && !more) msghdr.msg_flags &= ~MSG_MORE; - p = sg_page(sg); -retry: - bvec_set_page(&bvec, p, size, offset); - iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size); - ret = tcp_sendmsg_locked(sk, &msghdr, size); - if (ret < 0) { - emsg->offset = offset - sg->offset; - skmsg->sg.start += done; + bvec_set_page(&bvec, sg_page(sg), sg->length, sg->offset); + iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, sg->length); + ret = tcp_sendmsg_locked(sk, &msghdr, sg->length); + if (ret < 0) return ret; - } - - if (ret != size) { - offset += ret; - size -= ret; - goto retry; - } - done++; - put_page(p); - sk_mem_uncharge(sk, sg->length); - sg = sg_next(sg); - } while (sg); + sk_msg_free_partial(sk, skmsg, ret); + } while (skmsg->sg.size); memset(emsg, 0, sizeof(*emsg)); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 630f3dd31cc5..f153bf695b9d 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -182,7 +182,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur err = x->type_offload->xmit(x, skb, esp_features); if (err) { if (err == -EINPROGRESS) - return NULL; + return ERR_PTR(-EINPROGRESS); XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); kfree_skb(skb); @@ -224,7 +224,15 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur pskb = skb2; } - return skb; + /* skb_gso_segment() set skb->prev to the last segment, but async + * crypto may have stolen it above without updating ->prev. Repoint + * it at the last retained segment so validate_xmit_skb_list() does + * not chain onto a segment now owned by the crypto engine. + */ + if (skb) + skb->prev = pskb; + + return skb ? skb : ERR_PTR(-EINPROGRESS); } EXPORT_SYMBOL_GPL(validate_xmit_xfrm); @@ -313,6 +321,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } xso->dev = dev; + xso->ifindex = dev->ifindex; netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC); if (xuo->flags & XFRM_OFFLOAD_INBOUND) diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index e4c2cd24936d..eecab337bd0a 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -467,6 +467,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) { const struct xfrm_state_afinfo *afinfo; struct net *net = dev_net(skb->dev); + struct net_device *dev = skb->dev; int err; __be32 seq; __be32 seq_hi; @@ -493,7 +494,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) LINUX_MIB_XFRMINSTATEINVALID); if (encap_type == -1) - dev_put(skb->dev); + dev_put(dev); goto drop; } @@ -655,16 +656,16 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (!crypto_done) { spin_unlock(&x->lock); - dev_hold(skb->dev); + dev_hold(dev); nexthdr = x->type->input(x, skb); if (nexthdr == -EINPROGRESS) { if (async) - dev_put(skb->dev); + dev_put(dev); return 0; } - dev_put(skb->dev); + dev_put(dev); spin_lock(&x->lock); } resume: @@ -699,7 +700,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) err = xfrm_inner_mode_input(x, skb); if (err == -EINPROGRESS) { if (async) - dev_put(skb->dev); + dev_put(dev); return 0; } else if (err) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); @@ -726,9 +727,12 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) crypto_done = false; } while (!err); + rcu_read_lock(); err = xfrm_rcv_cb(skb, family, x->type->proto, 0); - if (err) + if (err) { + rcu_read_unlock(); goto drop; + } nf_reset_ct(skb); @@ -739,8 +743,9 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) if (skb_valid_dst(skb)) skb_dst_drop(skb); if (async) - dev_put(skb->dev); + dev_put(dev); gro_cells_receive(&gro_cells, skb); + rcu_read_unlock(); return 0; } else { xo = xfrm_offload(skb); @@ -748,23 +753,21 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) xfrm_gro = xo->flags & XFRM_GRO; err = -EAFNOSUPPORT; - rcu_read_lock(); afinfo = xfrm_state_afinfo_get_rcu(x->props.family); if (likely(afinfo)) err = afinfo->transport_finish(skb, xfrm_gro || async); - rcu_read_unlock(); if (xfrm_gro) { sp = skb_sec_path(skb); if (sp) sp->olen = 0; if (skb_valid_dst(skb)) skb_dst_drop(skb); - if (async) - dev_put(skb->dev); gro_cells_receive(&gro_cells, skb); - return err; } + if (async) + dev_put(dev); + rcu_read_unlock(); return err; } @@ -772,7 +775,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) spin_unlock(&x->lock); drop: if (async) - dev_put(skb->dev); + dev_put(dev); xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); kfree_skb(skb); return 0; diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c index 330a05286a56..688306bf62c5 100644 --- a/net/xfrm/xfrm_interface_core.c +++ b/net/xfrm/xfrm_interface_core.c @@ -869,6 +869,9 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], struct net *net = xi->net; struct xfrm_if_parms p = {}; + if (!rtnl_dev_link_net_capable(dev, net)) + return -EPERM; + xfrmi_netlink_parms(data, &p); if (!p.if_id) { NL_SET_ERR_MSG(extack, "if_id must be non zero"); diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 458931062a04..eb1b6f67739e 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb, ka->encap_sport, sock_net_uid(net, NULL)); rt = ip_route_output_key(net, &fl4); - if (IS_ERR(rt)) + if (IS_ERR(rt)) { + kfree_skb(skb); return PTR_ERR(rt); + } skb_dst_set(skb, &rt->dst); @@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb, dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); if (IS_ERR(dst)) { local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock); + kfree_skb(skb); return PTR_ERR(dst); } @@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka) sizeof(struct ipv6hdr)) + sizeof(struct udphdr); const u8 nat_ka_payload = 0xFF; - int err = -EAFNOSUPPORT; struct sk_buff *skb; struct udphdr *uh; @@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka) switch (ka->family) { case AF_INET: - err = nat_keepalive_send_ipv4(skb, ka); + nat_keepalive_send_ipv4(skb, ka); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - err = nat_keepalive_send_ipv6(skb, ka, uh); + nat_keepalive_send_ipv6(skb, ka, uh); break; #endif - } - if (err) + default: kfree_skb(skb); + break; + } } struct nat_keepalive_work_ctx { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 0ea015e1880b..7ef861a0e823 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -242,6 +242,9 @@ __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl, unsigned short family) { + if (family != sel->family && sel->family != AF_UNSPEC) + return false; + switch (family) { case AF_INET: return __xfrm4_selector_match(sel, fl); @@ -685,7 +688,7 @@ static void xfrm_byidx_resize(struct net *net) static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total) { - unsigned int cnt = net->xfrm.policy_count[dir]; + unsigned int cnt = READ_ONCE(net->xfrm.policy_count[dir]); unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; if (total) @@ -711,12 +714,12 @@ static inline int xfrm_byidx_should_resize(struct net *net, int total) void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si) { - si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; - si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; - si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; - si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; - si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; - si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; + si->incnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_IN]); + si->outcnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]); + si->fwdcnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_FWD]); + si->inscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]); + si->outscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]); + si->fwdscnt = READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]); si->spdhcnt = net->xfrm.policy_idx_hmask; si->spdhmcnt = xfrm_policy_hashmax; } @@ -2318,7 +2321,7 @@ static void __xfrm_policy_link(struct xfrm_policy *pol, int dir) } list_add(&pol->walk.all, &net->xfrm.policy_all); - net->xfrm.policy_count[dir]++; + WRITE_ONCE(net->xfrm.policy_count[dir], net->xfrm.policy_count[dir] + 1); xfrm_pol_hold(pol); } @@ -2337,7 +2340,7 @@ static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, } list_del_init(&pol->walk.all); - net->xfrm.policy_count[dir]--; + WRITE_ONCE(net->xfrm.policy_count[dir], net->xfrm.policy_count[dir] - 1); return pol; } @@ -3222,7 +3225,7 @@ struct dst_entry *xfrm_lookup_with_ifid(struct net *net, /* To accelerate a bit... */ if (!if_id && ((dst_orig->flags & DST_NOXFRM) || - !net->xfrm.policy_count[XFRM_POLICY_OUT])) + !READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]))) goto nopol; xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id); @@ -3296,7 +3299,7 @@ struct dst_entry *xfrm_lookup_with_ifid(struct net *net, nopol: if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && - net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { + READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_BLOCK) { err = -EPERM; goto error; } @@ -3750,7 +3753,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, const bool is_crypto_offload = sp && (xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO); - if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { + if (READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_BLOCK) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); return 0; } diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 1df017871651..10e5a1a95fe3 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1207,9 +1207,11 @@ struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark, struct hlist_head *state_cache_input; struct xfrm_state *x = NULL; + /* BH is always disabled on the input path. */ + lockdep_assert_in_softirq(); + state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input); - rcu_read_lock(); hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) { if (x->props.family != family || x->id.spi != spi || @@ -1227,20 +1229,25 @@ struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark, xfrm_hash_ptrs_get(net, &state_ptrs); x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family); - - if (x && x->km.state == XFRM_STATE_VALID) { - spin_lock_bh(&net->xfrm.xfrm_state_lock); - if (hlist_unhashed(&x->state_cache_input)) { + if (x) { + spin_lock(&net->xfrm.xfrm_state_lock); + if (x->km.state != XFRM_STATE_VALID) { + /* + * The state is about to be destroyed. + * + * Don't add it to the cache but still + * return it to the caller. + */ + } else if (hlist_unhashed(&x->state_cache_input)) { hlist_add_head_rcu(&x->state_cache_input, state_cache_input); } else { hlist_del_rcu(&x->state_cache_input); hlist_add_head_rcu(&x->state_cache_input, state_cache_input); } - spin_unlock_bh(&net->xfrm.xfrm_state_lock); + spin_unlock(&net->xfrm.xfrm_state_lock); } out: - rcu_read_unlock(); return x; } EXPORT_SYMBOL(xfrm_input_state_lookup); @@ -1540,6 +1547,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr, xso->type = XFRM_DEV_OFFLOAD_PACKET; xso->dir = xdo->dir; xso->dev = dev; + xso->ifindex = dev->ifindex; xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC); error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, @@ -3003,7 +3011,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (sockptr_is_null(optval) && !optlen) { xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); - __sk_dst_reset(sk); + sk_dst_reset(sk); return 0; } @@ -3014,7 +3022,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (IS_ERR(data)) return PTR_ERR(data); - if (in_compat_syscall()) { + if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) { struct xfrm_translator *xtr = xfrm_get_translator(); if (!xtr) { @@ -3043,7 +3051,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (err >= 0) { xfrm_sk_policy_insert(sk, err, pol); xfrm_pol_put(pol); - __sk_dst_reset(sk); + sk_dst_reset(sk); err = 0; } diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 5975849e5893..d6db63304ba6 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1201,17 +1201,26 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb) return 0; } -static void xso_to_xuo(const struct xfrm_dev_offload *xso, - struct xfrm_user_offload *xuo) +static void xso_to_xuo_ifindex(const struct xfrm_dev_offload *xso, int ifindex, + struct xfrm_user_offload *xuo) { - xuo->ifindex = xso->dev->ifindex; + xuo->ifindex = ifindex; if (xso->dir == XFRM_DEV_OFFLOAD_IN) xuo->flags = XFRM_OFFLOAD_INBOUND; if (xso->type == XFRM_DEV_OFFLOAD_PACKET) xuo->flags |= XFRM_OFFLOAD_PACKET; } -static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +#ifdef CONFIG_XFRM_MIGRATE +static void xso_to_xuo(const struct xfrm_dev_offload *xso, + struct xfrm_user_offload *xuo) +{ + xso_to_xuo_ifindex(xso, xso->dev->ifindex, xuo); +} +#endif + +static int copy_user_offload_ifindex(const struct xfrm_dev_offload *xso, + int ifindex, struct sk_buff *skb) { struct xfrm_user_offload *xuo; struct nlattr *attr; @@ -1222,11 +1231,22 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) xuo = nla_data(attr); memset(xuo, 0, sizeof(*xuo)); - xso_to_xuo(xso, xuo); + xso_to_xuo_ifindex(xso, ifindex, xuo); return 0; } +static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, xso->dev->ifindex, skb); +} + +static int copy_user_state_offload(const struct xfrm_dev_offload *xso, + struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, READ_ONCE(xso->ifindex), skb); +} + static bool xfrm_redact(void) { return IS_ENABLED(CONFIG_SECURITY) && @@ -1433,8 +1453,8 @@ static int copy_to_user_state_extra(struct xfrm_state *x, &x->replay); if (ret) goto out; - if(x->xso.dev) - ret = copy_user_offload(&x->xso, skb); + if (READ_ONCE(x->xso.dev)) + ret = copy_user_state_offload(&x->xso, skb); if (ret) goto out; if (x->if_id) { @@ -2104,13 +2124,12 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family, switch (ut[i].mode) { case XFRM_MODE_TUNNEL: case XFRM_MODE_BEET: + case XFRM_MODE_IPTFS: if (ut[i].optional && dir == XFRM_POLICY_OUT) { NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy"); return -EINVAL; } break; - case XFRM_MODE_IPTFS: - break; default: if (ut[i].family != prev_family) { NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change"); @@ -2511,9 +2530,9 @@ static int xfrm_notify_userpolicy(struct net *net) } up = nlmsg_data(nlh); - up->in = net->xfrm.policy_default[XFRM_POLICY_IN]; - up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD]; - up->out = net->xfrm.policy_default[XFRM_POLICY_OUT]; + up->in = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN]); + up->fwd = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD]); + up->out = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]); nlmsg_end(skb, nlh); @@ -2537,13 +2556,13 @@ static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh, struct xfrm_userpolicy_default *up = nlmsg_data(nlh); if (xfrm_userpolicy_is_valid(up->in)) - net->xfrm.policy_default[XFRM_POLICY_IN] = up->in; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN], up->in); if (xfrm_userpolicy_is_valid(up->fwd)) - net->xfrm.policy_default[XFRM_POLICY_FWD] = up->fwd; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD], up->fwd); if (xfrm_userpolicy_is_valid(up->out)) - net->xfrm.policy_default[XFRM_POLICY_OUT] = up->out; + WRITE_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT], up->out); rt_genid_bump_all(net); @@ -2573,9 +2592,9 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh, } r_up = nlmsg_data(r_nlh); - r_up->in = net->xfrm.policy_default[XFRM_POLICY_IN]; - r_up->fwd = net->xfrm.policy_default[XFRM_POLICY_FWD]; - r_up->out = net->xfrm.policy_default[XFRM_POLICY_OUT]; + r_up->in = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_IN]); + r_up->fwd = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_FWD]); + r_up->out = READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]); nlmsg_end(r_skb, r_nlh); return nlmsg_unicast(xfrm_net_nlsk(net, skb), r_skb, portid); @@ -3835,7 +3854,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, if (!netlink_net_capable(skb, CAP_NET_ADMIN)) return -EPERM; - if (in_compat_syscall()) { + if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) { struct xfrm_translator *xtr = xfrm_get_translator(); if (!xtr) @@ -4046,8 +4065,8 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) l += nla_total_size(sizeof(*x->coaddr)); if (x->props.extra_flags) l += nla_total_size(sizeof(x->props.extra_flags)); - if (x->xso.dev) - l += nla_total_size(sizeof(struct xfrm_user_offload)); + if (READ_ONCE(x->xso.dev)) + l += nla_total_size(sizeof(struct xfrm_user_offload)); if (x->props.smark.v | x->props.smark.m) { l += nla_total_size(sizeof(x->props.smark.v)); l += nla_total_size(sizeof(x->props.smark.m)); diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 854e7c471434..812f8e5a08d5 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -27,6 +27,10 @@ #[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] #[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] mod bindings_raw { use pin_init::{MaybeZeroable, Zeroable}; diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index d9625b9ee046..6e59cc9c665c 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -14,7 +14,7 @@ rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) return copy_to_user(to, from, n); } -#ifdef INLINE_COPY_FROM_USER +#ifdef INLINE_COPY_USER __rust_helper unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) { diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index 554a5a2ff0ab..35ede53f2b8e 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -535,6 +535,7 @@ macro_rules! bitfield { // `Debug` implementation. (@debug $name:ident { $($field:ident;)* }) => { impl ::kernel::fmt::Debug for $name { + #[inline] fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { f.debug_struct(stringify!($name)) .field("", &::kernel::prelude::fmt!("{:#x}", self.inner)) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 477cf771fb10..7ad124327a83 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -102,7 +102,7 @@ macro_rules! drm_legacy_fields { /// what stage of the process the [`Device`] is currently in. This means for instance that a /// `&Device` may actually be registered with userspace, it just wasn't known to be /// registered at the time the reference was taken. -pub trait DeviceContext: Sealed + Send + Sync {} +pub trait DeviceContext: Sealed + Send + Sync + 'static {} /// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point. /// diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index c8b66d816871..48fa6e96dfe7 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -85,7 +85,7 @@ pub type DriverAllocImpl = <::Driver as drm::Driver>::Object; /// GEM object functions, which must be implemented by drivers. -pub trait DriverObject: Sync + Send + Sized { +pub trait DriverObject: Sync + Send + Sized + 'static { /// Parent `Driver` for this object. type Driver: drm::Driver; diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 34af402899a0..3ee19ef6264e 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -11,6 +11,11 @@ use crate::{ container_of, + device::{ + self, + Bound, // + }, + devres::*, drm::{ driver, gem, @@ -19,20 +24,46 @@ use crate::{ DeviceContext, Registered, // }, - error::to_result, + error::{ + from_err_ptr, + to_result, // + }, + io::{ + Io, + IoCapable, + IoKnownSize, // + }, prelude::*, - sync::aref::ARef, - types::Opaque, // + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, }; use core::{ + ffi::c_void, marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, ops::{ Deref, DerefMut, // }, - ptr::NonNull, // + ptr::{ + self, + NonNull, // + }, }; use gem::{ + BaseObject, BaseObjectPrivate, DriverObject, IntoGEMObject, // @@ -42,7 +73,6 @@ use gem::{ /// /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. -#[derive(Default)] pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { /// Whether to set the write-combine map flag. pub map_wc: bool, @@ -53,6 +83,16 @@ pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { pub parent_resv_obj: Option<&'a Object>, } +impl<'a, T: DriverObject, C: DeviceContext> Default for ObjectConfig<'a, T, C> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } +} + /// A shmem-backed GEM object. /// /// # Invariants @@ -67,6 +107,11 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, #[pin] inner: T, _ctx: PhantomData, @@ -125,6 +170,8 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), inner <- T::new(dev, size, args), _ctx: PhantomData::, }), @@ -169,22 +216,143 @@ impl Object { // - DRM always passes a valid gem object here // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that // `obj` is contained within a drm_gem_shmem_object - let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; - - // SAFETY: - // - We're in free_callback - so this function is safe to call. - // - We won't be using the gem resources on `this` after this call. - unsafe { bindings::drm_gem_shmem_release(this) }; + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; // SAFETY: // - We verified above that `obj` is valid, which makes `this` valid // - This function is set in AllocOps, so we know that `this` is contained within a // `Object` - let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; // SAFETY: We're recovering the Kbox<> we created in gem_create_object() let _ = unsafe { KBox::from_raw(this) }; } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result> + where + R: Deref + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap(&self) -> Result> { + self.make_vmap() + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap(&self) -> Result> { + self.make_vmap() + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { + if dev.as_raw() != self.dev().as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } } impl Deref for Object { @@ -235,3 +403,369 @@ impl driver::AllocImpl for Object { dumb_map_offset: None, }; } + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject, C: DeviceContext = Registered>( + &'a Object, + NotThreadSafe, +); + +impl<'a, T: DriverObject, C: DeviceContext> DmaResvGuard<'a, T, C> { + #[inline] + fn new(obj: &'a Object) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject, C: DeviceContext> Drop for DmaResvGuard<'a, T, C> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, C, const SIZE: usize = 0> = VMap, C, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned = VMap>, C, SIZE>; + +impl VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl Drop for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl Send for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl Sync for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref> + Sync, +{ +} + +impl Io for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + #[inline] + fn addr(&self) -> usize { + self.addr as usize + } + + #[inline] + fn maxsize(&self) -> usize { + self.owner.size() + } +} + +impl IoKnownSize for VMap +where + D: DriverObject, + C: DeviceContext, + R: Deref>, +{ + const MIN_SIZE: usize = SIZE; +} + +macro_rules! impl_vmap_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for VMap + where + D: DriverObject, + C: DeviceContext, + R: Deref>, + { + #[inline] + unsafe fn io_read(&self, address: usize) -> $ty { + let ptr = address as *mut $ty; + + // SAFETY: The safety contract of `io_read` guarantees that address is a valid + // address within the bounds of `Self` of at least the size of $ty, and is properly + // aligned. + unsafe { ptr::read_volatile(ptr) } + } + + #[inline] + unsafe fn io_write(&self, value: $ty, address: usize) { + let ptr = address as *mut $ty; + + // SAFETY: The safety contract of `io_write` guarantees that address is a valid + // address within the bounds of `Self` of at least the size of $ty, and is properly + // aligned. + unsafe { ptr::write_volatile(ptr, value) } + } + } + }; +} + +impl_vmap_io_capable!(u8); +impl_vmap_io_capable!(u16); +impl_vmap_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_vmap_io_capable!(u64); + +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, +} + +impl Deref for SGTableMap { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl Drop for SGTableMap { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device) -> Result>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device, + _size: usize, + _args: Self::Args, + ) -> impl PinInit { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type File = KunitFile; + type Object = Object; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let dev = faux::Registration::new(c"Kunit", None)?; + let drm = UnregisteredDevice::new(dev.as_ref(), data)?; + + Ok((dev, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the max size matches the actual object size + assert_eq!(vmap.maxsize(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.read8(offset), expected); + } + + Ok(()) + } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let wrong_dev = faux::Registration::new(c"EvilKunit", None)?; + + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } +} diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ae58f6f667c1..20a08b3defeb 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -116,9 +116,9 @@ impl GpuVm { /// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] - pub fn new( + pub fn new( name: &'static CStr, - dev: &drm::Device, + dev: &drm::Device, r_obj: &T::Object, range: Range, reserve_range: Range, @@ -252,10 +252,10 @@ impl GpuVm { /// The manager for a GPUVM. pub trait DriverGpuVm: Sized + Send { /// Parent `Driver` for this object. - type Driver: drm::Driver; + type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: IntoGEMObject; + type Object: drm::driver::AllocImpl; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). type VaData; @@ -264,7 +264,9 @@ pub trait DriverGpuVm: Sized + Send { type VmBoData; /// The private data passed to callbacks. - type SmContext<'ctx>; + type SmContext<'ctx> + where + Self: 'ctx; /// Indicates that a new mapping should be created. fn sm_step_map<'op, 'ctx>( diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs index 69a8e5ab2821..742c151b2540 100644 --- a/rust/kernel/drm/gpuvm/sm_ops.rs +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -3,7 +3,7 @@ use super::*; /// The actual data that gets threaded through the callbacks. -struct SmData<'a, 'ctx, T: DriverGpuVm> { +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { gpuvm: &'a mut UniqueRefGpuVm, user_context: &'a mut T::SmContext<'ctx>, } @@ -20,7 +20,7 @@ struct SmMapData<'a, 'ctx, T: DriverGpuVm> { } /// The argument for [`UniqueRefGpuVm::sm_map`]. -pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> { +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { /// Address in GPU virtual address space. pub addr: u64, /// Length of mapping to create. diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 43b4974f48cd..36c92ae2943c 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -25,7 +25,8 @@ use core::ptr::{ /// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. /// /// [`struct faux_device`]: srctree/include/linux/device/faux.h pub struct Registration(NonNull); @@ -59,10 +60,17 @@ impl Registration { } } -impl AsRef for Registration { - fn as_ref(&self) -> &device::Device { - // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be - // a valid initialized `device`. +impl AsRef> for Registration { + fn as_ref(&self) -> &device::Device { + // SAFETY: + // - The underlying `device` in `faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } } } diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 624b971ca8b0..7655d61daac8 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -404,6 +404,7 @@ impl I2cAdapter { } /// Gets pointer to an `i2c_adapter` by index. + #[inline] pub fn get(index: i32) -> Result> { // SAFETY: `index` must refer to a valid I2C adapter; the kernel // guarantees that `i2c_get_adapter(index)` returns either a valid @@ -425,11 +426,13 @@ kernel::impl_device_context_into_aref!(I2cAdapter); // SAFETY: Instances of `I2cAdapter` are always reference-counted. unsafe impl AlwaysRefCounted for I2cAdapter { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::i2c_get_adapter(self.index()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index f924c7c7c1db..66449377a0b7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -448,7 +448,7 @@ where /// /// register! { /// FIXED_REG(u32) @ 0x100 { -/// 16:8 high_byte; +/// 15:8 high_byte; /// 7:0 low_byte; /// } /// } diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index cdee5f27bd7f..91eaff8c186a 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -288,6 +288,7 @@ macro_rules! kunit_unsafe_test_suite { log: ::core::ptr::null_mut(), suite_init_err: 0, is_init: false, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, }; #[used(compiler)] diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index c3c2052c9206..83ce50def5ac 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -11,16 +11,27 @@ use crate::{ bindings, device::Device, - error::{to_result, Error, Result, VTABLE_DEFAULT_ERROR}, - ffi::{c_int, c_long, c_uint, c_ulong}, - fs::{File, Kiocb}, - iov::{IovIterDest, IovIterSource}, + error::{ + to_result, + VTABLE_DEFAULT_ERROR, // + }, + fs::{ + File, + Kiocb, // + }, + iov::{ + IovIterDest, + IovIterSource, // + }, mm::virt::VmaNew, prelude::*, seq_file::SeqFile, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + Opaque, // + }, }; -use core::{marker::PhantomData, pin::Pin}; +use core::marker::PhantomData; /// Options for creating a misc device. #[derive(Copy, Clone)] diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 8a6da92e8da6..ca396f1f78a6 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -61,10 +61,16 @@ pub use pin_init::{ }; #[doc(no_inline)] -pub use zerocopy::FromBytes; +pub use zerocopy::{ + FromBytes, + IntoBytes, // +}; #[doc(no_inline)] -pub use zerocopy_derive::FromBytes; +pub use zerocopy_derive::{ + FromBytes, + IntoBytes, // +}; #[doc(no_inline)] pub use super::{ diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index b8a515de31ca..797ead5b5626 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -24,6 +24,10 @@ unsafe_op_in_unsafe_fn )] #![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#![cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] #![feature(cfi_encoding)] // Manual definition of blocklisted types. diff --git a/samples/damon/Kconfig b/samples/damon/Kconfig index cbf96fd8a8bf..00be3e6bdd65 100644 --- a/samples/damon/Kconfig +++ b/samples/damon/Kconfig @@ -31,7 +31,7 @@ config SAMPLE_DAMON_MTIER bool "DAMON sample module for memory tiering" depends on DAMON && DAMON_PADDR help - Thps builds DAMON sample module for memory tierign. + This builds DAMON sample module for memory tiering. The module assumes the system is constructed with two NUMA nodes, which seems as local and remote nodes to all CPUs. For example, diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index eb1143de8df1..e567f4edd80e 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -120,6 +120,9 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) addr.end = promote ? node1_end_addr : node0_end_addr; } + if (addr.start >= addr.end) + goto free_out; + range.start = addr.start; range.end = addr.end; @@ -174,6 +177,7 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) static int damon_sample_mtier_start(void) { struct damon_ctx *ctx; + int err; ctx = damon_sample_mtier_build_ctx(true); if (!ctx) @@ -185,12 +189,21 @@ static int damon_sample_mtier_start(void) return -ENOMEM; } ctxs[1] = ctx; - return damon_start(ctxs, 2, true); + err = damon_start(ctxs, 2, true); + if (!err) + return 0; + + if (damon_is_running(ctxs[0])) + damon_stop(ctxs, 1); + damon_destroy_ctx(ctxs[0]); + damon_destroy_ctx(ctxs[1]); + return err; } static void damon_sample_mtier_stop(void) { - damon_stop(ctxs, 2); + damon_stop(ctxs, 1); + damon_stop(&ctxs[1], 1); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); } diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index b7c50f2656ce..edeae145c4a8 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -106,11 +106,18 @@ static int damon_sample_prcl_start(void) damon_set_schemes(ctx, &scheme, 1); err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_prcl_stop(void) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 799ad4443943..ff5e8a890f44 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -87,10 +87,17 @@ static int damon_sample_wsse_start(void) target->pid = target_pidp; err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_wsse_stop(void) diff --git a/samples/rust/rust_misc_device.rs b/samples/rust/rust_misc_device.rs index 87a1fe63533a..41e26c825060 100644 --- a/samples/rust/rust_misc_device.rs +++ b/samples/rust/rust_misc_device.rs @@ -97,14 +97,36 @@ use kernel::{ device::Device, - fs::{File, Kiocb}, - ioctl::{_IO, _IOC_SIZE, _IOR, _IOW}, - iov::{IovIterDest, IovIterSource}, - miscdevice::{MiscDevice, MiscDeviceOptions, MiscDeviceRegistration}, + fs::{ + File, + Kiocb, // + }, + ioctl::{ + _IO, + _IOC_SIZE, + _IOR, + _IOW, // + }, + iov::{ + IovIterDest, + IovIterSource, // + }, + miscdevice::{ + MiscDevice, + MiscDeviceOptions, + MiscDeviceRegistration, // + }, new_mutex, prelude::*, - sync::{aref::ARef, Mutex}, - uaccess::{UserSlice, UserSliceReader, UserSliceWriter}, + sync::{ + aref::ARef, + Mutex, // + }, + uaccess::{ + UserSlice, + UserSliceReader, + UserSliceWriter, // + }, }; const RUST_MISC_DEV_HELLO: u32 = _IO('|' as u32, 0x80); diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c index ecc7db237f2e..ca5d98c360cb 100644 --- a/samples/trace_events/trace-events-sample.c +++ b/samples/trace_events/trace-events-sample.c @@ -94,6 +94,20 @@ static int simple_thread_fn(void *arg) static DEFINE_MUTEX(thread_mutex); static int simple_thread_cnt; +static struct foo_timer_data *foo_timer_data; + +static void sample_timer_cb(struct timer_list *t) +{ + struct foo_timer_data *data = container_of(t, struct foo_timer_data, timer); + + get_cpu(); + trace_foo_timer_fn(data); + (*this_cpu_ptr(data->counter))++; + put_cpu(); + + mod_timer(t, jiffies + HZ); +} + int foo_bar_reg(void) { mutex_lock(&thread_mutex); @@ -107,6 +121,10 @@ int foo_bar_reg(void) * for consistency sake, we still take the thread_mutex. */ simple_tsk_fn = kthread_run(simple_thread_fn, NULL, "event-sample-fn"); + if (IS_ERR_OR_NULL(simple_tsk_fn)) { + pr_err("Failed to create simple_thread_fn\n"); + simple_tsk_fn = NULL; + } out: mutex_unlock(&thread_mutex); return 0; @@ -128,9 +146,27 @@ void foo_bar_unreg(void) static int __init trace_event_init(void) { + foo_timer_data = kzalloc_obj(*foo_timer_data, GFP_KERNEL); + if (!foo_timer_data) + return -ENOMEM; + + foo_timer_data->name = "sample_timer_counter"; + foo_timer_data->counter = alloc_percpu(int); + if (!foo_timer_data->counter) { + kfree(foo_timer_data); + return -ENOMEM; + } + + timer_setup(&foo_timer_data->timer, sample_timer_cb, 0); + mod_timer(&foo_timer_data->timer, jiffies + HZ); + simple_tsk = kthread_run(simple_thread, NULL, "event-sample"); - if (IS_ERR(simple_tsk)) - return -1; + if (IS_ERR(simple_tsk)) { + timer_shutdown_sync(&foo_timer_data->timer); + free_percpu(foo_timer_data->counter); + kfree(foo_timer_data); + return PTR_ERR(simple_tsk); + } return 0; } @@ -143,6 +179,10 @@ static void __exit trace_event_exit(void) kthread_stop(simple_tsk_fn); simple_tsk_fn = NULL; mutex_unlock(&thread_mutex); + + timer_shutdown_sync(&foo_timer_data->timer); + free_percpu(foo_timer_data->counter); + kfree(foo_timer_data); } module_init(trace_event_init); diff --git a/samples/trace_events/trace-events-sample.h b/samples/trace_events/trace-events-sample.h index 1a05fc153353..816848a456a2 100644 --- a/samples/trace_events/trace-events-sample.h +++ b/samples/trace_events/trace-events-sample.h @@ -247,12 +247,14 @@ */ /* - * It is OK to have helper functions in the file, but they need to be protected - * from being defined more than once. Remember, this file gets included more - * than once. + * It is OK to have helper functions and data structures in the file, but they + * need to be protected from being defined more than once. Remember, this file + * gets included more than once. */ #ifndef __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS #define __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS +#include + static inline int __length_of(const int *list) { int i; @@ -270,6 +272,13 @@ enum { TRACE_SAMPLE_BAR = 4, TRACE_SAMPLE_ZOO = 8, }; + +struct foo_timer_data { + const char *name; + struct timer_list timer; + int __percpu *counter; +}; + #endif /* @@ -595,6 +604,25 @@ TRACE_EVENT(foo_rel_loc, __get_rel_bitmask(bitmask), __get_rel_cpumask(cpumask)) ); + +TRACE_EVENT(foo_timer_fn, + + TP_PROTO(struct foo_timer_data *data), + + TP_ARGS(data), + + TP_STRUCT__entry( + __string( name, data->name ) + __field( int, count ) + ), + + TP_fast_assign( + __assign_str(name); + __entry->count = *this_cpu_ptr(data->counter); + ), + + TP_printk("name=%s count=%d", __get_str(name), __entry->count) +); #endif /***** NOTICE! The #if protection ends here. *****/ diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 911745743246..d432693e5367 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -159,10 +159,10 @@ targets += $(targets-for-builtin) $(targets-for-modules) # Linus' kernel sanity checking tool ifeq ($(KBUILD_CHECKSRC),1) - quiet_cmd_checksrc = CHECK $< + quiet_cmd_checksrc = CHECK $(patsubst $(srctree)/%,%,$<) cmd_checksrc = $(CHECK) $(CHECKFLAGS) $(c_flags) $< else ifeq ($(KBUILD_CHECKSRC),2) - quiet_cmd_force_checksrc = CHECK $< + quiet_cmd_force_checksrc = CHECK $(patsubst $(srctree)/%,%,$<) cmd_force_checksrc = $(CHECK) $(CHECKFLAGS) $(c_flags) $< endif diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter index 9b4fb996d95b..5868a8b11b0f 100755 --- a/scripts/bloat-o-meter +++ b/scripts/bloat-o-meter @@ -43,6 +43,7 @@ def getsizes(file, format): if name.startswith("__se_compat_sys"): continue if name.startswith("__addressable_"): continue if name.startswith("__noinstr_text_start"): continue + if name.startswith("_sdata"): continue if name == "linux_banner": continue if name == "vermagic": continue # statics and some other optimizations adds random .NUMBER diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index cc5bbd70cb84..2b7a42bbdd94 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -57,8 +57,12 @@ my @ignore = (); my $help = 0; my $configuration_file = ".checkpatch.conf"; +my $def_configuration_dirs_help = '.:$HOME:.scripts'; +(my $def_configuration_dirs = $def_configuration_dirs_help) =~ s/\$(\w+)/$ENV{$1}/g; +my $env_config_dir = 'CHECKPATCH_CONFIG_DIR'; my $max_line_length = 100; my $ignore_perl_version = 0; +my $spdx_cxx_comments = 0; my $minimum_perl_version = 5.10.0; my $min_conf_desc_length = 4; my $spelling_file = "$D/spelling.txt"; @@ -135,6 +139,10 @@ sub help { file. It's your fault if there's no backup or git --ignore-perl-version override checking of perl version. expect runtime errors. + --spdx-cxx-comments don't force C comments (/* */) for SPDX license + (required by old toolchains), allow also C++ + comments (//). + NOTE: it should *not* be used for Linux mainline. --codespell Use the codespell dictionary for spelling/typos (default:$codespellfile) --codespellfile Use this codespell dictionary @@ -146,6 +154,11 @@ sub help { -h, --help, --version display this help and exit When FILE is - read standard input. + +CONFIGURATION FILE +Default configuration options can be stored in $configuration_file, +search path: '$def_configuration_dirs_help' or in a directory specified by +\$$env_config_dir environment variable (fallback to the default search path). EOM exit($exitcode); @@ -237,7 +250,7 @@ sub list_types { exit($exitcode); } -my $conf = which_conf($configuration_file); +my $conf = which_conf($configuration_file, $env_config_dir, $def_configuration_dirs); if (-f $conf) { my @conf_args; open(my $conffile, '<', "$conf") @@ -339,6 +352,7 @@ sub load_docs { 'fix!' => \$fix, 'fix-inplace!' => \$fix_inplace, 'ignore-perl-version!' => \$ignore_perl_version, + 'spdx-cxx-comments!' => \$spdx_cxx_comments, 'debug=s' => \%debug, 'test-only=s' => \$tst_only, 'codespell!' => \$codespell, @@ -1529,9 +1543,15 @@ sub which { } sub which_conf { - my ($conf) = @_; + my ($conf, $env_key, $paths) = @_; + my $env_dir = $ENV{$env_key}; + + if (defined($env_dir) && $env_dir ne "") { + return "$env_dir/$conf" if (-e "$env_dir/$conf"); + warn "$P: Can't find a readable $conf in '$env_dir', falling back to default search paths\n"; + } - foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) { + foreach my $path (split(/:/, $paths)) { if (-e "$path/$conf") { return "$path/$conf"; } @@ -3231,10 +3251,10 @@ sub process { if ($sign_off =~ /^reported(?:|-and-tested)-by:$/i) { if (!defined $lines[$linenr]) { WARN("BAD_REPORTED_BY_LINK", - "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . "\n"); - } elsif ($rawlines[$linenr] !~ /^closes:\s*/i) { + "Reported-by: should be immediately followed by Closes: or Link: with a URL to the report\n" . $herecurr . "\n"); + } elsif ($rawlines[$linenr] !~ /^(closes|link):\s*/i) { WARN("BAD_REPORTED_BY_LINK", - "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); + "Reported-by: should be immediately followed by Closes: or Link: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); } } } @@ -3799,26 +3819,33 @@ sub process { $checklicenseline = 2; } elsif ($rawline =~ /^\+/) { my $comment = ""; - if ($realfile =~ /\.(h|s|S)$/) { - $comment = '/*'; - } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { + if ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; } elsif ($realfile =~ /\.rst$/) { $comment = '..'; } + my $pattern = qr{\Q$comment\E}; + if ($realfile =~ /\.(h|s|S)$/) { + $comment = '/*'; + $pattern = qr{/\*}; + if ($spdx_cxx_comments) { + $comment = '// or /*'; + $pattern = qr{//|/\*}; + } + } # check SPDX comment style for .[chsS] files if ($realfile =~ /\.[chsS]$/ && $rawline =~ /SPDX-License-Identifier:/ && - $rawline !~ m@^\+\s*\Q$comment\E\s*@) { + $rawline !~ m@^\+\s*$pattern\s*@) { WARN("SPDX_LICENSE_TAG", "Improper SPDX comment style for '$realfile', please use '$comment' instead\n" . $herecurr); } if ($comment !~ /^$/ && - $rawline !~ m@^\+\Q$comment\E SPDX-License-Identifier: @) { + $rawline !~ m@^\+$pattern SPDX-License-Identifier: @) { WARN("SPDX_LICENSE_TAG", "Missing or malformed SPDX-License-Identifier tag in line $checklicenseline\n" . $herecurr); } elsif ($rawline =~ /(SPDX-License-Identifier: .*)/) { @@ -4154,7 +4181,7 @@ sub process { $pl =~ s/\b(?:$Attribute|$Sparse)\b//g; if (($pl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $pl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros @@ -4168,7 +4195,7 @@ sub process { # looks like a declaration !($sl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $sl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py index 8e713a09cfe7..eeabaeed438b 100644 --- a/scripts/gdb/linux/page_owner.py +++ b/scripts/gdb/linux/page_owner.py @@ -34,6 +34,7 @@ class DumpPageOwner(gdb.Command): max_pfn = None p_ops = None migrate_reason_names = None + mr_never = None def __init__(self): super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) @@ -65,6 +66,7 @@ def get_page_owner_info(self): self.max_pfn = int(gdb.parse_and_eval("max_pfn")) self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + self.mr_never = int(gdb.parse_and_eval('MR_NEVER')) def page_ext_invalid(self, page_ext): if page_ext == gdb.Value(0): @@ -138,7 +140,7 @@ def read_page_owner_by_addr(self, struct_page_addr): else: gdb.write('page last free stack trace:\n') stackdepot.stack_depot_print(page_owner["free_handle"]) - if page_owner['last_migrate_reason'] != -1: + if page_owner['last_migrate_reason'] != self.mr_never: gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) def read_page_owner(self): diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index dc1219736f77..621895bae128 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -250,6 +250,7 @@ def append_sysroot_crate( "macros", srctree / "rust" / "macros" / "lib.rs", [std, proc_macro, proc_macro2, quote, syn], + cfg=generated_cfg, ) zerocopy_derive = append_proc_macro_crate( diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl index f0ca0db6ddc2..16b80a700d4a 100755 --- a/scripts/get_maintainer.pl +++ b/scripts/get_maintainer.pl @@ -21,6 +21,7 @@ use File::Find; use File::Spec::Functions; use open qw(:std :encoding(UTF-8)); +use JSON::PP; my $cur_path = fastgetcwd() . '/'; my $lk_path = "./"; @@ -68,6 +69,7 @@ my $self_test = undef; my $version = 0; my $help = 0; +my $json = 0; my $find_maintainer_files = 0; my $maintainer_path; my $vcs_used = 0; @@ -285,6 +287,7 @@ 'find-maintainer-files' => \$find_maintainer_files, 'mpath|maintainer-path=s' => \$maintainer_path, 'self-test:s' => \$self_test, + 'json!' => \$json, 'v|version' => \$version, 'h|help|usage' => \$help, )) { @@ -650,39 +653,48 @@ sub read_mailmap { my %deduplicate_address_hash = (); my @maintainers = get_maintainers(); -if (@maintainers) { - @maintainers = merge_email(@maintainers); - output(@maintainers); -} - -if ($scm) { - @scm = uniq(@scm); - output(@scm); -} - -if ($output_substatus) { - @substatus = uniq(@substatus); - output(@substatus); -} - -if ($status) { - @status = uniq(@status); - output(@status); -} -if ($subsystem) { - @subsystem = uniq(@subsystem); - output(@subsystem); -} +@maintainers = merge_email(@maintainers) if (@maintainers); +@scm = uniq(@scm) if ($scm); +@substatus = uniq(@substatus) if ($output_substatus); +@status = uniq(@status) if ($status); +@subsystem = uniq(@subsystem) if ($subsystem); +@web = uniq(@web) if ($web); +@bug = uniq(@bug) if ($bug); + +if ($json) { + my @json_maintainers; + for my $m (@maintainers) { + my ($addr, $role); + if ($output_roles && $m =~ /^(.*?)\s+\((.+)\)\s*$/) { + $addr = $1; + $role = $2; + } else { + $addr = $m; + } + my ($name, $email_addr) = parse_email($addr); + my %entry = (name => $name, email => $email_addr); + $entry{role} = $role if (defined $role && $role ne ''); + push(@json_maintainers, \%entry); + } -if ($web) { - @web = uniq(@web); - output(@web); -} + my %result = (maintainers => \@json_maintainers); + $result{scm} = \@scm if ($scm); + $result{status} = \@status if ($status); + $result{subsystem} = \@subsystem if ($subsystem); + $result{web} = \@web if ($web); + $result{bug} = \@bug if ($bug); -if ($bug) { - @bug = uniq(@bug); - output(@bug); + my $json_encoder = JSON::PP->new->canonical->utf8; + print($json_encoder->encode(\%result) . "\n"); +} else { + output(@maintainers) if (@maintainers); + output(@scm) if ($scm); + output(@substatus) if ($output_substatus); + output(@status) if ($status); + output(@subsystem) if ($subsystem); + output(@web) if ($web); + output(@bug) if ($bug); } exit($exit); @@ -1104,6 +1116,7 @@ sub usage { --separator [, ] => separator for multiple entries on 1 line using --separator also sets --nomultiline if --separator is not [, ] --multiline => print 1 entry per line + --json => output results as JSON Other options: --pattern-depth => Number of pattern directory traversals (default: 0 (all)) diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index 9599a0408862..4234a51d16fd 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -206,6 +206,78 @@ static void conf_message(const char *fmt, ...) va_end(ap); } +static void conf_changed_input_warning(const char *s) +{ + fputs(s, stderr); +} + +static bool conf_warn_changed_input_enabled(void) +{ + const char *env = getenv("KCONFIG_WARN_CHANGED_INPUT"); + + return env && *env; +} + +static const char *sym_get_user_value_string(struct symbol *sym) +{ + switch (sym->type) { + case S_BOOLEAN: + case S_TRISTATE: + switch (sym->def[S_DEF_USER].tri) { + case yes: + return "y"; + case mod: + return "m"; + default: + return "n"; + } + default: + return sym->def[S_DEF_USER].val ?: ""; + } +} + +static bool sym_user_value_changed(struct symbol *sym) +{ + if (!sym_has_value(sym) || sym->type == S_UNKNOWN) + return false; + + switch (sym->type) { + case S_BOOLEAN: + case S_TRISTATE: + return sym->def[S_DEF_USER].tri != sym_get_tristate_value(sym); + default: + return strcmp(sym_get_user_value_string(sym), + sym_get_string_value(sym)); + } +} + +static void conf_clear_written_flags(void) +{ + struct symbol *sym; + + for_all_symbols(sym) + sym->flags &= ~SYMBOL_WRITTEN; +} + +static void conf_append_changed_input_warning(struct gstr *gs, + struct symbol *sym, + bool *changed_input_found) +{ + if (!sym_user_value_changed(sym)) + return; + + if (!*changed_input_found) { + str_printf(gs, + "warning: user-provided values changed by Kconfig:\n"); + *changed_input_found = true; + } + + str_printf(gs, " %s%s: %s -> %s\n", + CONFIG_, sym->name, + sym_get_user_value_string(sym), + sym_get_string_value(sym)); +} + const char *conf_get_configname(void) { char *name = getenv("KCONFIG_CONFIG"); @@ -759,11 +831,15 @@ int conf_write_defconfig(const char *filename) { struct symbol *sym; struct menu *menu; + struct gstr gs; FILE *out; + bool warn_changed_input = conf_warn_changed_input_enabled(); + bool changed_input_found = false; out = fopen(filename, "w"); if (!out) return 1; + gs = str_new(); sym_clear_all_valid(); @@ -772,10 +848,14 @@ int conf_write_defconfig(const char *filename) sym = menu->sym; - if (!sym || sym_is_choice(sym)) + if (!sym || sym_is_choice(sym) || sym->flags & SYMBOL_WRITTEN) continue; sym_calc_value(sym); + if (warn_changed_input) + conf_append_changed_input_warning(&gs, sym, + &changed_input_found); + sym->flags |= SYMBOL_WRITTEN; if (!(sym->flags & SYMBOL_WRITE)) continue; sym->flags &= ~SYMBOL_WRITE; @@ -798,6 +878,13 @@ int conf_write_defconfig(const char *filename) print_symbol_for_dotconfig(out, sym); } fclose(out); + + conf_clear_written_flags(); + + if (changed_input_found) + conf_changed_input_warning(str_get(&gs)); + + str_free(&gs); return 0; } @@ -809,7 +896,10 @@ int conf_write(const char *name) const char *str; char tmpname[PATH_MAX + 1], oldname[PATH_MAX + 1]; char *env; + struct gstr gs; bool need_newline = false; + bool warn_changed_input = conf_warn_changed_input_enabled(); + bool changed_input_found = false; if (!name) name = conf_get_configname(); @@ -838,6 +928,7 @@ int conf_write(const char *name) } if (!out) return 1; + gs = str_new(); conf_write_heading(out, &comment_style_pound); @@ -859,13 +950,16 @@ int conf_write(const char *name) } else if (!sym_is_choice(sym) && !(sym->flags & SYMBOL_WRITTEN)) { sym_calc_value(sym); + if (warn_changed_input) + conf_append_changed_input_warning(&gs, sym, + &changed_input_found); + sym->flags |= SYMBOL_WRITTEN; if (!(sym->flags & SYMBOL_WRITE)) goto next; if (need_newline) { fprintf(out, "\n"); need_newline = false; } - sym->flags |= SYMBOL_WRITTEN; print_symbol_for_dotconfig(out, sym); } @@ -892,8 +986,12 @@ int conf_write(const char *name) } fclose(out); - for_all_symbols(sym) - sym->flags &= ~SYMBOL_WRITTEN; + conf_clear_written_flags(); + + if (changed_input_found) + conf_changed_input_warning(str_get(&gs)); + + str_free(&gs); if (*tmpname) { if (is_same(name, tmpname)) { diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index b2d8d4e11e07..8c280292f9cd 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -242,13 +242,33 @@ static int menu_validate_number(struct symbol *sym, struct symbol *sym2) static void sym_check_prop(struct symbol *sym) { - struct property *prop; + struct property *prev, *prop; struct symbol *sym2; char *use; for (prop = sym->prop; prop; prop = prop->next) { switch (prop->type) { case P_DEFAULT: + for_all_defaults(sym, prev) { + if (prev == prop) + break; + if (expr_is_yes(prev->visible.expr)) { + if (!expr_eq(prev->expr, prop->expr)) + prop_warn(prop, + "default for '%s' is unreachable: earlier default at %s:%d is unconditional", + sym->name ? sym->name : "", + prev->filename, prev->lineno); + break; + } + if (expr_eq(prev->visible.expr, prop->visible.expr)) { + if (!expr_eq(prev->expr, prop->expr)) + prop_warn(prop, + "default for '%s' has the same condition as the earlier default at %s:%d", + sym->name ? sym->name : "", + prev->filename, prev->lineno); + break; + } + } if ((sym->type == S_STRING || sym->type == S_INT || sym->type == S_HEX) && prop->expr->type != E_SYMBOL) prop_warn(prop, diff --git a/scripts/kconfig/tests/conftest.py b/scripts/kconfig/tests/conftest.py index d94b79e012c0..66f95e4ed58c 100644 --- a/scripts/kconfig/tests/conftest.py +++ b/scripts/kconfig/tests/conftest.py @@ -37,7 +37,8 @@ def __init__(self, request): # runners def _run_conf(self, mode, dot_config=None, out_file='.config', - interactive=False, in_keys=None, extra_env={}): + interactive=False, in_keys=None, extra_env={}, + silent=False): """Run text-based Kconfig executable and save the result. mode: input mode option (--oldaskconfig, --defconfig= etc.) @@ -48,7 +49,10 @@ def _run_conf(self, mode, dot_config=None, out_file='.config', extra_env: additional environments returncode: exit status of the Kconfig executable """ - command = [CONF_PATH, mode, 'Kconfig'] + command = [CONF_PATH] + if silent: + command.append('-s') + command += [mode, 'Kconfig'] # Override 'srctree' environment to make the test as the top directory extra_env['srctree'] = self._test_dir diff --git a/scripts/kconfig/tests/warn_changed_input/Kconfig b/scripts/kconfig/tests/warn_changed_input/Kconfig new file mode 100644 index 000000000000..69845e2f3fb3 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +config DEP + bool "DEP" + help + Test dependency symbol for Kconfig warning coverage. + This is used by the warn_changed_input selftest. + It intentionally stays unset in the input fragment. + The test checks how dependent user input is adjusted. + +config A + bool "A" + depends on DEP + help + Test bool symbol for changed-input diagnostics. + The input fragment requests this symbol as built-in. + The unmet dependency on DEP forces the final value to n. + The warning should report that downgrade. + +config NUM + int "NUM" + range 10 20 + help + Test integer symbol for changed-input diagnostics. + The input fragment requests a value outside the allowed range. + Kconfig resolves it to the constrained in-range value. + The warning should report that adjustment. + +config DUP + bool "DUP" + depends on DEP + help + Test duplicate-definition handling for changed-input diagnostics. + The input fragment requests this symbol as built-in. + The duplicate definition below must not produce a duplicate warning. + This keeps the warning output stable for repeated menu entries. + +config DUP + bool + depends on DEP diff --git a/scripts/kconfig/tests/warn_changed_input/__init__.py b/scripts/kconfig/tests/warn_changed_input/__init__.py new file mode 100644 index 000000000000..4c3bca6af846 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/__init__.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Test optional warnings for user-provided values changed by Kconfig. + +Warnings should stay disabled by default, and should only appear when +KCONFIG_WARN_CHANGED_INPUT is enabled. +""" + + +def test(conf): + assert conf.olddefconfig('config') == 0 + assert 'user-provided values changed by Kconfig' not in conf.stderr + + assert conf._run_conf('--olddefconfig', dot_config='config', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }) == 0 + assert conf.stderr_contains('expected_stderr') + assert conf.config_matches('expected_config') + + assert conf._run_conf('--olddefconfig', dot_config='config', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }, silent=True) == 0 + assert conf.stderr_contains('expected_stderr') + + assert conf._run_conf('--savedefconfig=defconfig', dot_config='config', + out_file='defconfig', + extra_env={ + 'KCONFIG_WARN_CHANGED_INPUT': '1', + }) == 0 + assert conf.stderr_contains('expected_stderr') + assert conf.config_matches('expected_defconfig') diff --git a/scripts/kconfig/tests/warn_changed_input/config b/scripts/kconfig/tests/warn_changed_input/config new file mode 100644 index 000000000000..dbe93ff26408 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/config @@ -0,0 +1,3 @@ +CONFIG_A=y +CONFIG_NUM=30 +CONFIG_DUP=y diff --git a/scripts/kconfig/tests/warn_changed_input/expected_config b/scripts/kconfig/tests/warn_changed_input/expected_config new file mode 100644 index 000000000000..fe8bbec66c53 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_config @@ -0,0 +1,6 @@ +# +# Automatically generated file; DO NOT EDIT. +# Main menu +# +# CONFIG_DEP is not set +CONFIG_NUM=20 diff --git a/scripts/kconfig/tests/warn_changed_input/expected_defconfig b/scripts/kconfig/tests/warn_changed_input/expected_defconfig new file mode 100644 index 000000000000..af9e34851d2a --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_defconfig @@ -0,0 +1 @@ +CONFIG_NUM=20 diff --git a/scripts/kconfig/tests/warn_changed_input/expected_stderr b/scripts/kconfig/tests/warn_changed_input/expected_stderr new file mode 100644 index 000000000000..9ec8446b4ac2 --- /dev/null +++ b/scripts/kconfig/tests/warn_changed_input/expected_stderr @@ -0,0 +1,4 @@ +warning: user-provided values changed by Kconfig: + CONFIG_A: y -> n + CONFIG_NUM: 30 -> 20 + CONFIG_DUP: y -> n diff --git a/scripts/kconfig/tests/warn_dead_default/Kconfig b/scripts/kconfig/tests/warn_dead_default/Kconfig new file mode 100644 index 000000000000..adf421d73dbd --- /dev/null +++ b/scripts/kconfig/tests/warn_dead_default/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +config A + bool + +config B + bool + +config UNCONDITIONAL + int + default 1 + default 2 + +config CONDITIONAL + int + default 1 if A + default 2 if A + default 3 if B + +config CONDITIONAL_COMMUTATIVE + int + default 1 if A && B + default 2 if B && A + +config CONTROL + int + default 1 if A + default 2 if B + default 3 + +choice + prompt "test choice" + default C + default D + + config C + bool "C" + config D + bool "D" +endchoice diff --git a/scripts/kconfig/tests/warn_dead_default/__init__.py b/scripts/kconfig/tests/warn_dead_default/__init__.py new file mode 100644 index 000000000000..911b30ce19fe --- /dev/null +++ b/scripts/kconfig/tests/warn_dead_default/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Test detection of dead defaults (different defaults that can never be active). +""" + +def test(conf): + assert conf.olddefconfig() == 0 + assert conf.stderr_contains('expected_stderr') diff --git a/scripts/kconfig/tests/warn_dead_default/expected_stderr b/scripts/kconfig/tests/warn_dead_default/expected_stderr new file mode 100644 index 000000000000..baa20bf33910 --- /dev/null +++ b/scripts/kconfig/tests/warn_dead_default/expected_stderr @@ -0,0 +1,4 @@ +Kconfig:12:warning: default for 'UNCONDITIONAL' is unreachable: earlier default at Kconfig:11 is unconditional +Kconfig:17:warning: default for 'CONDITIONAL' has the same condition as the earlier default at Kconfig:16 +Kconfig:23:warning: default for 'CONDITIONAL_COMMUTATIVE' has the same condition as the earlier default at Kconfig:22 +Kconfig:34:warning: default for '' is unreachable: earlier default at Kconfig:33 is unconditional diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 2ad87a74bb03..8d36c74dec2d 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1280,6 +1280,8 @@ static void do_tee_entry(struct module *mod, void *symval) static void do_wmi_entry(struct module *mod, void *symval) { DEF_FIELD_ADDR(symval, wmi_device_id, guid_string); + char result[sizeof(*guid_string)]; + int i; if (strlen(*guid_string) != UUID_STRING_LEN) { warn("Invalid WMI device id 'wmi:%s' in '%s'\n", @@ -1287,7 +1289,31 @@ static void do_wmi_entry(struct module *mod, void *symval) return; } - module_alias_printf(mod, false, WMI_MODULE_PREFIX "%s", *guid_string); + for (i = 0; i < UUID_STRING_LEN; i++) { + char value = (*guid_string)[i]; + bool valid = false; + + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (value == '-') + valid = true; + } else { + if (isxdigit(value)) + valid = true; + } + + if (!valid) { + warn("Invalid character %c inside WMI GUID string '%s' in '%s'\n", + value, *guid_string, mod->name); + return; + } + + /* Some GUIDs from BMOF definitions contain lowercase characters */ + result[i] = toupper(value); + } + + result[i] = '\0'; + + module_alias_printf(mod, false, WMI_MODULE_PREFIX "%s", result); } /* Looks like: mhi:S */ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index d592548cbd60..a7b72a81d248 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -969,7 +969,7 @@ static int secref_whitelist(const char *fromsec, const char *fromsym, /* symbols in data sections that may refer to any init/exit sections */ if (match(fromsec, PATTERNS(DATA_SECTIONS)) && match(tosec, PATTERNS(ALL_INIT_SECTIONS, ALL_EXIT_SECTIONS)) && - match(fromsym, PATTERNS("*_ops", "*_console"))) + match(fromsym, PATTERNS("*_ops", "*_ops.llvm.*", "*_console"))) return 0; /* Check for pattern 3 */ diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs index f7540bcf595a..df864437cef7 100644 --- a/scripts/rustdoc_test_builder.rs +++ b/scripts/rustdoc_test_builder.rs @@ -28,7 +28,7 @@ fn main() { // // ``` // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_28_0() { - // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl ::core::fmt::Debug> { + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl core::fmt::Debug> { // ``` // // It should be unlikely that doctest code matches such lines (when code is formatted properly). @@ -47,12 +47,16 @@ fn main() { }) .expect("No test function found in `rustdoc`'s output."); - // Qualify `Result` to avoid the collision with our own `Result` coming from the prelude. + // Replicate `rustdoc` 1.87+ behaviour [1] by fully qualifying `Result` to avoid the collision + // with our own `Result` coming from the prelude. + // + // [1]: https://github.com/rust-lang/rust/pull/137807 + // + // TODO: Remove this when MSRV is bumped above 1.87. let body = body.replace( - &format!("{rustdoc_function_name}() -> Result<(), impl ::core::fmt::Debug> {{"), - &format!( - "{rustdoc_function_name}() -> ::core::result::Result<(), impl ::core::fmt::Debug> {{" - ), + &format!("{rustdoc_function_name}() -> Result<(), impl core::fmt::Debug> {{"), + // This intentionally does not use absolute paths to match `rustdoc` 1.87 behaviour. + &format!("{rustdoc_function_name}() -> core::result::Result<(), impl core::fmt::Debug> {{"), ); // For tests that get generated with `Result`, like above, `rustdoc` generates an `unwrap()` on diff --git a/scripts/sbom/sbom.py b/scripts/sbom/sbom.py new file mode 100644 index 000000000000..764175b9c893 --- /dev/null +++ b/scripts/sbom/sbom.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +""" +Compute software bill of materials in SPDX format describing a kernel build. +""" + +import json +import logging +import os +import sys +import time +import uuid +import sbom.sbom_logging as sbom_logging +from sbom.config import get_config +from sbom.path_utils import is_relative_to +from sbom.spdx import JsonLdSpdxDocument, SpdxIdGenerator +from sbom.spdx.core import CreationInfo, SpdxDocument +from sbom.spdx_graph import SpdxIdGeneratorCollection, build_spdx_graphs +from sbom.cmd_graph import CmdGraph + + +def _exit_with_summary(write_output_on_error: bool = False) -> None: + warning_summary = sbom_logging.summarize_warnings() + error_summary = sbom_logging.summarize_errors() + if warning_summary: + logging.warning(warning_summary) + if error_summary: + logging.error(error_summary) + if not write_output_on_error: + logging.info( + "Use --write-output-on-error to generate output documents even when errors occur. " + "Note that in this case the generated documents may be incomplete." + ) + sys.exit(1) + + +def main(): + # Read config + config = get_config() + + # Configure logging + logging.basicConfig( + level=logging.DEBUG if config.debug else logging.INFO, + format="[%(levelname)s] %(message)s", + ) + + # Build cmd graph + logging.debug("Start building cmd graph") + start_time = time.time() + cmd_graph = CmdGraph.create(config.root_paths, config) + logging.debug(f"Built cmd graph in {time.time() - start_time} seconds") + + # Save used files document + if config.generate_used_files: + if config.src_tree == config.obj_tree: + logging.info( + f"Extracting all files from the cmd graph to {config.used_files_file_name} " + "instead of only source files because source files cannot be " + "reliably classified when the source and object trees are identical.", + ) + used_files = [os.path.relpath(node.absolute_path, config.src_tree) for node in cmd_graph] + logging.debug(f"Found {len(used_files)} files in cmd graph.") + else: + used_files = [ + os.path.relpath(node.absolute_path, config.src_tree) + for node in cmd_graph + if is_relative_to(node.absolute_path, config.src_tree) + and not is_relative_to(node.absolute_path, config.obj_tree) + ] + logging.debug(f"Found {len(used_files)} source files in cmd graph") + if not sbom_logging.has_errors() or config.write_output_on_error: + used_files_path = os.path.join(config.output_directory, config.used_files_file_name) + with open(used_files_path, "w", encoding="utf-8") as f: + f.write("\n".join(str(file_path) for file_path in used_files)) + logging.debug(f"Successfully saved {used_files_path}") + + if config.generate_spdx is False: + _exit_with_summary(config.write_output_on_error) + return + + # Build SPDX Documents + logging.debug("Start generating SPDX graph based on cmd graph") + start_time = time.time() + + # The real uuid will be generated based on the content of the SPDX graphs + # to ensure that the same SPDX document is always assigned the same uuid. + PLACEHOLDER_UUID = "00000000-0000-0000-0000-000000000000" + spdx_id_base_namespace = f"{config.spdxId_prefix}{PLACEHOLDER_UUID}/" + spdx_id_generators = SpdxIdGeneratorCollection( + base=SpdxIdGenerator(prefix="p", namespace=spdx_id_base_namespace), + source=SpdxIdGenerator(prefix="s", namespace=f"{spdx_id_base_namespace}source/"), + build=SpdxIdGenerator(prefix="b", namespace=f"{spdx_id_base_namespace}build/"), + output=SpdxIdGenerator(prefix="o", namespace=f"{spdx_id_base_namespace}output/"), + ) + + spdx_graphs = build_spdx_graphs( + cmd_graph, + spdx_id_generators, + config, + ) + spdx_id_uuid = uuid.uuid5( + uuid.NAMESPACE_URL, + "".join( + json.dumps(element.to_dict()) for spdx_graph in spdx_graphs.values() for element in spdx_graph.to_list() + ), + ) + logging.debug(f"Generated SPDX graph in {time.time() - start_time} seconds") + + if not sbom_logging.has_errors() or config.write_output_on_error: + for kernel_sbom_kind, spdx_graph in spdx_graphs.items(): + spdx_graph_objects = spdx_graph.to_list() + # Add warning and error summary to creation info comment + creation_info = next(element for element in spdx_graph_objects if isinstance(element, CreationInfo)) + creation_info.comment = "\n".join([ + sbom_logging.summarize_warnings(), + sbom_logging.summarize_errors(), + ]).strip() + # Replace Placeholder uuid with real uuid for spdxIds + spdx_document = next(element for element in spdx_graph_objects if isinstance(element, SpdxDocument)) + for namespaceMap in spdx_document.namespaceMap: + namespaceMap.namespace = namespaceMap.namespace.replace(PLACEHOLDER_UUID, str(spdx_id_uuid)) + # Serialize SPDX graph to JSON-LD + spdx_doc = JsonLdSpdxDocument(graph=spdx_graph_objects) + save_path = os.path.join(config.output_directory, config.spdx_file_names[kernel_sbom_kind]) + spdx_doc.save(save_path, config.prettify_json) + logging.debug(f"Successfully saved {save_path}") + + _exit_with_summary(config.write_output_on_error) + + +# Call main method +if __name__ == "__main__": + main() diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/sbom/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/sbom/sbom/cmd_graph/__init__.py b/scripts/sbom/sbom/cmd_graph/__init__.py new file mode 100644 index 000000000000..9d661a5c3d93 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .cmd_graph import CmdGraph +from .cmd_graph_node import CmdGraphNode, CmdGraphNodeConfig + +__all__ = ["CmdGraph", "CmdGraphNode", "CmdGraphNodeConfig"] diff --git a/scripts/sbom/sbom/cmd_graph/cmd_file.py b/scripts/sbom/sbom/cmd_graph/cmd_file.py new file mode 100644 index 000000000000..dcd63e284a38 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_file.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +import re +from dataclasses import dataclass, field +from sbom.cmd_graph.deps_parser import parse_cmd_file_deps +from sbom.cmd_graph.savedcmd_parser import parse_inputs_from_commands +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr + +SAVEDCMD_PATTERN = re.compile(r"^(saved)?cmd_.*?:=\s*(?P.+)$") +SOURCE_PATTERN = re.compile(r"^source.*?:=\s*(?P.+)$") + + +@dataclass +class CmdFile: + cmd_file_path: PathStr + savedcmd: str + source: PathStr | None = None + deps: list[str] = field(default_factory=list) + make_rules: list[str] = field(default_factory=list) + + @classmethod + def create(cls, cmd_file_path: PathStr) -> "CmdFile | None": + """ + Parses a .cmd file. + .cmd files are assumed to have one of the following structures: + 1. Full Cmd File + (saved)?cmd_ := + source_ := + deps_ := \ + + := $(deps_) + $(deps_): + + 2. Command Only Cmd File + (saved)?cmd_ := + + 3. Single Dependency Cmd File + (saved)?cmd_ := + : + + Args: + cmd_file_path (Path): absolute Path to a .cmd file + + Returns: + cmd_file (CmdFile): Parsed cmd file. + """ + with open(cmd_file_path, "rt", encoding="utf-8") as f: + lines = [line.strip() for line in f.readlines() if line.strip() != "" and not line.startswith("#")] + + # savedcmd + match = SAVEDCMD_PATTERN.match(lines[0] if lines else "") + if match is None: + sbom_logging.error( + "Skip parsing '{cmd_file_path}' because no 'savedcmd_' command was found.", cmd_file_path=cmd_file_path + ) + return None + savedcmd = match.group("full_command") + + # Command Only Cmd File + if len(lines) == 1: + return CmdFile(cmd_file_path, savedcmd) + + # Single Dependency Cmd File + if len(lines) == 2: + parts = lines[1].split(":", 1) + if len(parts) != 2: + sbom_logging.error( + "Skip parsing '{cmd_file_path}'. Expected dependency line ': ' but got {second_line}", cmd_file_path=cmd_file_path, second_line=lines[1] + ) + return None + dep = parts[1].strip() + return CmdFile(cmd_file_path, savedcmd, deps=[dep]) + + # Full Cmd File + # source + line1 = SOURCE_PATTERN.match(lines[1]) + if line1 is None: + sbom_logging.error( + "Skip parsing '{cmd_file_path}' because no 'source_' entry was found.", cmd_file_path=cmd_file_path + ) + return CmdFile(cmd_file_path, savedcmd) + source = line1.group("source_file") + + # deps + deps: list[str] = [] + i = 3 # lines[2] includes the variable assignment but no actual dependency, so we need to start at lines[3]. + while i < len(lines): + if not lines[i].endswith("\\"): + break + deps.append(lines[i][:-1].strip()) + i += 1 + + # make_rules + make_rules = lines[i:] + + return CmdFile(cmd_file_path, savedcmd, source, deps, make_rules) + + def get_dependencies( + self: "CmdFile", target_path: PathStr, obj_tree: PathStr, fail_on_unknown_build_command: bool + ) -> list[PathStr]: + """ + Parses all dependencies required to build a target file from its cmd file. + + Args: + target_path: path to the target file relative to `obj_tree`. + obj_tree: absolute path to the object tree. + fail_on_unknown_build_command: Whether to fail if an unknown build command is encountered. + + Returns: + list[PathStr]: dependency file paths relative to `obj_tree`. + """ + input_files: list[PathStr] = [ + str(p) for p in parse_inputs_from_commands(self.savedcmd, fail_on_unknown_build_command) + ] + if self.deps: + input_files += [str(p) for p in parse_cmd_file_deps(self.deps)] + input_files = _expand_resolve_files(input_files, obj_tree) + + cmd_file_dependencies: list[PathStr] = [] + for input_file in input_files: + # input files are either absolute or relative to the object tree + if os.path.isabs(input_file): + input_file = os.path.relpath(input_file, obj_tree) + if input_file == target_path: + # Skip target file to prevent cycles. This is necessary because some multi stage commands first create an output and then pass it as input to the next command, e.g., objcopy. + continue + cmd_file_dependencies.append(input_file) + unique_cmd_file_dependencies = list(dict.fromkeys(cmd_file_dependencies)) + return unique_cmd_file_dependencies + + +def _expand_resolve_files(input_files: list[PathStr], obj_tree: PathStr) -> list[PathStr]: + """ + Expands resolve files which may reference additional files via '@' notation. + + Args: + input_files (list[PathStr]): List of file paths relative to the object tree, where paths starting with '@' refer to files + containing further file paths, each on a separate line. + obj_tree: Absolute path to the root of the object tree. + + Returns: + list[PathStr]: Flattened list of all input file paths, with any nested '@' file references resolved recursively. + """ + expanded_input_files: list[PathStr] = [] + for input_file in input_files: + if not input_file.startswith("@"): + expanded_input_files.append(input_file) + continue + resolve_file_path = os.path.join(obj_tree, input_file.removeprefix("@")) + if not os.path.exists(resolve_file_path): + sbom_logging.error( + "Skip resolving '{resolve_file_path}' because the response file does not exist.", + resolve_file_path=resolve_file_path, + ) + continue + with open(resolve_file_path, "rt", encoding="utf-8") as f: + resolve_file_content = [line_stripped for line in f.readlines() if (line_stripped := line.strip())] + expanded_input_files += _expand_resolve_files(resolve_file_content, obj_tree) + return expanded_input_files diff --git a/scripts/sbom/sbom/cmd_graph/cmd_graph.py b/scripts/sbom/sbom/cmd_graph/cmd_graph.py new file mode 100644 index 000000000000..2f57965237f4 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_graph.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from collections import deque +from dataclasses import dataclass, field +from typing import Iterator + +from sbom.cmd_graph.cmd_graph_node import CmdGraphNode, CmdGraphNodeConfig +from sbom.path_utils import PathStr + + +@dataclass +class CmdGraph: + """Directed acyclic graph of build dependencies primarily inferred from .cmd files produced during kernel builds""" + + roots: list[CmdGraphNode] = field(default_factory=list) + + @classmethod + def create(cls, root_paths: list[PathStr], config: CmdGraphNodeConfig) -> "CmdGraph": + """ + Recursively builds a dependency graph starting from `root_paths`. + Dependencies are mainly discovered by parsing the `.cmd` files. + + Args: + root_paths (list[PathStr]): List of paths to root outputs relative to obj_tree + config (CmdGraphNodeConfig): Configuration options + + Returns: + CmdGraph: A graph of all build dependencies for the given root files. + """ + node_cache: dict[PathStr, CmdGraphNode] = {} + root_nodes = [CmdGraphNode.create(root_path, config, node_cache) for root_path in root_paths] + return CmdGraph(root_nodes) + + def __iter__(self) -> Iterator[CmdGraphNode]: + """Traverse the graph in breadth-first order, yielding each unique node.""" + visited: set[PathStr] = set() + node_stack: deque[CmdGraphNode] = deque(self.roots) + while len(node_stack) > 0: + node = node_stack.popleft() + if node.absolute_path in visited: + continue + + visited.add(node.absolute_path) + node_stack.extend(node.children) + yield node diff --git a/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py b/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py new file mode 100644 index 000000000000..61f3a8140cea --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/cmd_graph_node.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from itertools import chain +import logging +import os +from typing import Iterator, Protocol + +from sbom import sbom_logging +from sbom.cmd_graph.cmd_file import CmdFile +from sbom.cmd_graph.hardcoded_dependencies import get_hardcoded_dependencies +from sbom.cmd_graph.incbin_parser import parse_incbin_statements +from sbom.path_utils import PathStr, has_link, is_relative_to + + +@dataclass +class IncbinDependency: + node: "CmdGraphNode" + full_statement: str + + +class CmdGraphNodeConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + fail_on_unknown_build_command: bool + + +@dataclass +class CmdGraphNode: + """A node in the cmd graph representing a single file and its dependencies.""" + + absolute_path: PathStr + """Absolute path to the file this node represents.""" + + cmd_file: CmdFile | None = None + """Parsed .cmd file describing how the file at absolute_path was built, or None if not available.""" + + cmd_file_dependencies: list["CmdGraphNode"] = field(default_factory=list) + incbin_dependencies: list[IncbinDependency] = field(default_factory=list) + hardcoded_dependencies: list["CmdGraphNode"] = field(default_factory=list) + + @property + def children(self) -> Iterator["CmdGraphNode"]: + seen: set[PathStr] = set() + for node in chain( + self.cmd_file_dependencies, + (dep.node for dep in self.incbin_dependencies), + self.hardcoded_dependencies, + ): + if node.absolute_path not in seen: + seen.add(node.absolute_path) + yield node + + @classmethod + def create( + cls, + target_path: PathStr, + config: CmdGraphNodeConfig, + cache: dict[PathStr, "CmdGraphNode"] | None = None, + depth: int = 0, + ) -> "CmdGraphNode": + """ + Recursively builds a dependency graph starting from `target_path`. + Dependencies are mainly discovered by parsing the `..cmd` file. + + Args: + target_path: Path to the target file relative to obj_tree. + config: Config options + cache: Tracks processed nodes to prevent cycles. + depth: Internal parameter to track the current recursion depth. + + Returns: + CmdGraphNode: cmd graph node representing the target file + """ + if cache is None: + cache = {} + + target_path_absolute = ( + os.path.realpath(p) + if has_link(p:=os.path.join(config.obj_tree, target_path)) + else os.path.normpath(p) + ) + + if target_path_absolute in cache: + return cache[target_path_absolute] + + if depth == 0: + logging.debug(f"Build node: {target_path}") + + cmd_file_path = _to_cmd_path(target_path_absolute) + cmd_file = CmdFile.create(cmd_file_path) if os.path.exists(cmd_file_path) else None + node = CmdGraphNode(target_path_absolute, cmd_file) + cache[target_path_absolute] = node + + if not os.path.exists(target_path_absolute): + error_or_warning = ( + sbom_logging.error + if is_relative_to(target_path_absolute, config.obj_tree) + or is_relative_to(target_path_absolute, config.src_tree) + else sbom_logging.warning + ) + error_or_warning( + "Skip parsing '{target_path_absolute}' because file does not exist", + target_path_absolute=target_path_absolute, + ) + return node + + # Search for dependencies to add to the graph as child nodes. Child paths are always relative to the output tree. + def _build_child_node(child_path: PathStr) -> "CmdGraphNode": + return CmdGraphNode.create(child_path, config, cache, depth + 1) + + node.hardcoded_dependencies = [ + _build_child_node(hardcoded_dependency_path) + for hardcoded_dependency_path in get_hardcoded_dependencies( + target_path_absolute, config.obj_tree, config.src_tree + ) + ] + + if cmd_file is not None: + node.cmd_file_dependencies = [ + _build_child_node(cmd_file_dependency_path) + for cmd_file_dependency_path in cmd_file.get_dependencies( + target_path, config.obj_tree, config.fail_on_unknown_build_command + ) + ] + + if node.absolute_path.endswith(".S"): + node.incbin_dependencies = [ + IncbinDependency( + node=_build_child_node(incbin_statement.path), + full_statement=incbin_statement.full_statement, + ) + for incbin_statement in parse_incbin_statements(node.absolute_path) + ] + + return node + + +def _to_cmd_path(path: PathStr) -> PathStr: + name = os.path.basename(path) + return path.removesuffix(name) + f".{name}.cmd" diff --git a/scripts/sbom/sbom/cmd_graph/deps_parser.py b/scripts/sbom/sbom/cmd_graph/deps_parser.py new file mode 100644 index 000000000000..6a2d92f0778c --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/deps_parser.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr + +# Match dependencies on config files +# Example match: "$(wildcard include/config/CONFIG_SOMETHING)" +CONFIG_PATTERN = re.compile(r"\$\(wildcard (include/config/[^)]+)\)") + +# Match dependencies on the objtool binary +# Example match: "$(wildcard ./tools/objtool/objtool)" +OBJTOOL_PATTERN = re.compile(r"\$\(wildcard \./tools/objtool/objtool\)") + +# Match any Makefile wildcard reference +# Example match: "$(wildcard path/to/file)" +WILDCARD_PATTERN = re.compile(r"\$\(wildcard (?P[^)]+)\)") + +# Match ordinary paths: +# - ^(\/)?: Optionally starts with a '/' +# - (([\w\-\.,+~=@ ]*)\/)*: Zero or more directory levels +# - [\w\-\.,+~=@ ]+$: Path component (file or directory) +# Example matches: "/foo/bar.c", "dir1/dir2/file.txt", "plainfile" +VALID_PATH_PATTERN = re.compile(r"^(\/)?(([\w\-\.,+~=@ ]*)\/)*[\w\-\.,+~=@ ]+$") + + +def parse_cmd_file_deps(deps: list[str]) -> list[PathStr]: + """ + Parse dependency strings of a .cmd file and return valid input file paths. + + Args: + deps: List of dependency strings as found in `.cmd` files. + + Returns: + input_files: List of input file paths + """ + input_files: list[PathStr] = [] + for dep in deps: + dep = dep.strip() + match dep: + case _ if CONFIG_PATTERN.match(dep) or OBJTOOL_PATTERN.match(dep): + # config paths like include/config/ should not be included in the graph + continue + case _ if match := WILDCARD_PATTERN.match(dep): + path = match.group("path") + input_files.append(path) + case _ if VALID_PATH_PATTERN.match(dep): + input_files.append(dep) + case _: + sbom_logging.error("Skip parsing dependency {dep} because of unrecognized format", dep=dep) + return input_files diff --git a/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py b/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py new file mode 100644 index 000000000000..2eb04d30f4e6 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/hardcoded_dependencies.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +from typing import Callable +import sbom.sbom_logging as sbom_logging +from sbom.path_utils import PathStr, is_relative_to +from sbom.environment import Environment + +HARDCODED_DEPENDENCIES: dict[str, list[str]] = { + # defined in linux/Kbuild + "include/generated/rq-offsets.h": ["kernel/sched/rq-offsets.s"], + "kernel/sched/rq-offsets.s": ["include/generated/asm-offsets.h"], + "include/generated/bounds.h": ["kernel/bounds.s"], + "include/generated/asm-offsets.h": ["arch/{arch}/kernel/asm-offsets.s"], +} +""" +Maps file paths to the list of dependencies required to build them +which are not tracked by the .cmd dependency mechanism. +Paths are relative to either the source tree or the object tree. +""" + +def get_hardcoded_dependencies(path: PathStr, obj_tree: PathStr, src_tree: PathStr) -> list[PathStr]: + """ + Some files in the kernel build process are not tracked by the .cmd dependency mechanism. + Parsing these dependencies programmatically is too complex for the scope of this project. + Therefore, this function provides manually defined dependencies to be added to the build graph. + + Args: + path: absolute path to a file within the src tree or object tree. + obj_tree: absolute Path to the base directory of the object tree. + src_tree: absolute Path to the `linux` source directory. + + Returns: + list[PathStr]: A list of dependency file paths (relative to the object tree) required to build the file at the given path. + """ + if is_relative_to(path, obj_tree): + path = os.path.relpath(path, obj_tree) + elif is_relative_to(path, src_tree): + path = os.path.relpath(path, src_tree) + + if path not in HARDCODED_DEPENDENCIES: + return [] + + template_variables: dict[str, Callable[[], str | None]] = { + "arch": lambda: _get_arch(path), + } + + dependencies: list[PathStr] = [] + for dependency_template in HARDCODED_DEPENDENCIES[path]: + dependency = _evaluate_template(dependency_template, template_variables) + if dependency is None: + continue + if os.path.exists(os.path.join(obj_tree, dependency)): + dependencies.append(dependency) + elif os.path.exists(dependency_absolute := os.path.join(src_tree, dependency)): + dependencies.append(os.path.relpath(dependency_absolute, obj_tree)) + else: + sbom_logging.error( + "Skip hardcoded dependency '{dependency}' for '{path}' because the dependency lies neither in the src tree nor the object tree.", + dependency=dependency, + path=path, + ) + + return dependencies + + +def _evaluate_template(template: str, variables: dict[str, Callable[[], str | None]]) -> str | None: + for key, value_function in variables.items(): + template_key = "{" + key + "}" + if template_key in template: + value = value_function() + if value is None: + return None + template = template.replace(template_key, value) + return template + + +def _get_arch(path: PathStr): + srcarch = Environment.SRCARCH() + if srcarch is None: + sbom_logging.error( + "Skipped architecture specific hardcoded dependency for '{path}' because the SRCARCH environment variable was not set.", + path=path, + ) + return None + return srcarch diff --git a/scripts/sbom/sbom/cmd_graph/incbin_parser.py b/scripts/sbom/sbom/cmd_graph/incbin_parser.py new file mode 100644 index 000000000000..ca289c2b8888 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/incbin_parser.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +import re + +from sbom.path_utils import PathStr + +INCBIN_PATTERN = re.compile(r'\s*\.incbin\s+"(?P[^"]+)"') +"""Regex pattern for matching `.incbin ""` statements.""" + + +@dataclass +class IncbinStatement: + """A parsed `.incbin ""` directive.""" + + path: PathStr + """path to the file referenced by the `.incbin` directive.""" + + full_statement: str + """Full `.incbin ""` statement as it originally appeared in the file.""" + + +def parse_incbin_statements(absolute_path: PathStr) -> list[IncbinStatement]: + """ + Parses `.incbin` directives from an `.S` assembly file. + + Args: + absolute_path: Absolute path to the `.S` assembly file. + + Returns: + list[IncbinStatement]: Parsed `.incbin` statements. + """ + with open(absolute_path, "rt", encoding="utf-8") as f: + content = f.read() + return [ + IncbinStatement( + path=match.group("path"), + full_statement=match.group(0).strip(), + ) + for match in INCBIN_PATTERN.finditer(content) + ] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py new file mode 100644 index 000000000000..d13876af4dfd --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from sbom.cmd_graph.savedcmd_parser.savedcmd_parser import parse_inputs_from_commands + +__all__ = ["parse_inputs_from_commands"] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py new file mode 100644 index 000000000000..a48040b2c13c --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_parser_registry.py @@ -0,0 +1,516 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import shlex +from typing import Callable, Iterator + +import sbom.sbom_logging as sbom_logging +from sbom.environment import Environment +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.tokenizer import ( + CmdParsingError, + Option, + Positional, + tokenize_single_command, + tokenize_single_command_positionals_only, +) +from sbom.path_utils import PathStr + +CommandParser = Callable[[str], list[PathStr]] +CommandParserRegistryEntry = tuple[re.Pattern[str], CommandParser] + + +def _parse_dd_command(command: str) -> list[PathStr]: + match = re.match(r"dd.*?if=(\S+)", command) + if match: + return [match.group(1)] + return [] + + +def _parse_cat_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["cat", input1, input2, ...] + return [p for p in positionals[1:]] + + +def _parse_compound_command(command: str) -> list[PathStr]: + compound_command_parsers: list[CommandParserRegistryEntry] = [ + (re.compile(r"dd\b"), _parse_dd_command), + (re.compile(r"cat.*?\|"), lambda c: _parse_cat_command(c.split("|")[0])), + (re.compile(r"cat\b[^|>]*$"), _parse_cat_command), + (re.compile(r"echo\b"), _parse_noop), + (re.compile(r"\S+="), _parse_noop), + (re.compile(r"printf\b"), _parse_noop), + (re.compile(r"sed\b"), _parse_sed_command), + ( + re.compile(r"(.*/)scripts/bin2c\s*<"), + lambda c: [input] if (input := c.split("<")[1].split(">")[0].strip()) != "/dev/null" else [], + ), + (re.compile(r"^:$"), _parse_noop), + ] + + match = re.match(r"\s*[\(\{](.*)[\)\}]\s*>", command, re.DOTALL) + if match is None: + raise CmdParsingError("No inner commands found for compound command") + input_files: list[PathStr] = [] + inner_commands = split_commands(match.group(1)) + for inner_command in inner_commands: + if isinstance(inner_command, IfBlock): + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because IfBlock is not supported", + inner_command=inner_command, + ) + continue + + parser = next((parser for pattern, parser in compound_command_parsers if pattern.match(inner_command)), None) + if parser is None: + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because no matching parser was found", + inner_command=inner_command, + ) + continue + try: + input_files += parser(inner_command) + except (CmdParsingError, IndexError) as e: + sbom_logging.error( + "Skip parsing inner command {inner_command} of compound command because of command parsing error: {error_message}", + inner_command=inner_command, + error_message=str(e), + ) + return input_files + + +def _parse_objcopy_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command, flag_options=["-S", "-w"]) + positionals = [part.value for part in command_parts if isinstance(part, Positional)] + # expect positionals to be ['objcopy', input_file] or ['objcopy', input_file, output_file] + return [positionals[1]] + + +def _parse_link_vmlinux_command(command: str) -> list[PathStr]: + """ + For simplicity we do not parse the `scripts/link-vmlinux.sh` script. + Instead the `vmlinux.a` dependency is just hardcoded for now. + """ + return ["vmlinux.a"] + + +def _parse_cp_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["cp", input1, ..., destination] + return positionals[1:-1] + + +def _parse_noop(command: str) -> list[PathStr]: + """ + No-op parser for commands with no input files (e.g., 'rm', 'true'). + Returns an empty list. + """ + return [] + + +def _parse_ar_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['ar', flags, output, input1, input2, ...] + flags = positionals[1] + if "r" not in flags: + # 'r' option indicates that new files are added to the archive. + # If this option is missing we won't find any relevant input files. + return [] + return positionals[3:] + + +def _parse_ar_piped_xargs_command(command: str) -> list[PathStr]: + printf_command, _ = command.split("|", 1) + positionals = tokenize_single_command_positionals_only(printf_command.strip()) + # expect positionals to be ['printf', '{prefix_path}%s ', input1, input2, ...] + prefix_path = positionals[1].removesuffix("%s ") + return [f"{prefix_path}{filename}" for filename in positionals[2:]] + + +def _parse_gcc_or_clang_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # compile mode: expect last positional argument ending in a source file extension to be the input file + for part in reversed(parts): + if not part.startswith("-") and any(part.endswith(suffix) for suffix in [".c", ".S", ".dts"]): + return [part] + + # linking mode: expect all .o files to be the inputs + return [p for p in parts if p.endswith(".o")] + + +def _parse_rustc_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.rs` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".rs"): + return [part] + raise CmdParsingError("Could not find .rs input source file") + + +def _parse_rustdoc_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.rs` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".rs"): + return [part] + raise CmdParsingError("Could not find .rs input source file") + + +def _parse_syscallhdr_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip(), flag_options=["--emit-nr"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscallhdr.sh, input, output] + return [positionals[2]] + + +def _parse_syscalltbl_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip()) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscalltbl.sh, input, output] + return [positionals[2]] + + +def _parse_mkcapflags_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/mkcapflags.sh, output, input1, input2] + return [positionals[3], positionals[4]] + + +def _parse_orc_hash_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/orc_hash.sh, '<', input, '>', output] + return [positionals[3]] + + +def _parse_xen_hypercalls_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["sh", path/to/xen-hypercalls.sh, output, input1, input2, ...] + return positionals[3:] + + +def _parse_gen_initramfs_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/gen_initramfs.sh, input1, input2, ...] + return positionals[2:] + + +def _parse_vdso2c_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['vdso2c', raw_input, stripped_input, output] + return [positionals[1], positionals[2]] + + +def _parse_vdsomunge_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ['vdsomunge', input, output] + return [positionals[1]] + + +def _parse_ld_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command( + command=command.strip(), + flag_options=[ + "-shared", + "--no-undefined", + "--eh-frame-hdr", + "-Bsymbolic", + "-r", + "--no-ld-generated-unwind-info", + "--no-dynamic-linker", + "-pie", + "--no-dynamic-linker--whole-archive", + "--whole-archive", + "--no-whole-archive", + "--start-group", + "--end-group", + ], + ) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["ld", input1, input2, ...] + return positionals[1:] + + +def _parse_sed_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["sed", *, input] + input = command_parts[-1] + if input == "/dev/null": + return [] + return [input] + + +def _parse_awk(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + options = [p for p in command_parts if isinstance(p, Option)] + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + has_script_file = any(p.name == "-f" for p in options) + # With -f option: expect ["awk", input1, input2, ...] + # Without -f option: expect ["awk", inline_program, input1, input2, ...] + return positionals[1:] if has_script_file else positionals[2:] + + +def _parse_nm_piped_command(command: str) -> list[PathStr]: + nm_command, _ = command.split("|", 1) + command_parts = tokenize_single_command( + command=nm_command.strip(), + flag_options=["-p", "--defined-only"], + ) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["nm", input1, input2, ...] + return [p for p in positionals[1:]] + + +def _parse_pnm_to_logo_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["pnmtologo", , input] + return [command_parts[-1]] + + +def _parse_relacheck(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["relacheck", input, log_reference] + return [positionals[1]] + + +def _parse_gen_hyprel_command(command: str) -> list[PathStr]: + gen_hyprel_command, _ = command.split(">", 1) + command_parts = shlex.split(gen_hyprel_command) + # expect command_parts to be ["gen-hyprel", input] + return [command_parts[1]] + + +def _parse_perl_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command.strip()) + # expect positionals to be ["perl", input] + return [positionals[1]] + + +def _parse_strip_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command, flag_options=["--strip-debug"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["strip", input1, input2, ...] + return positionals[1:] + + +def _parse_mkpiggy_command(command: str) -> list[PathStr]: + mkpiggy_command, _ = command.split(">", 1) + positionals = tokenize_single_command_positionals_only(mkpiggy_command) + # expect positionals to be ["mkpiggy", input] + return [positionals[1]] + + +def _parse_relocs_command(command: str) -> list[PathStr]: + if ">" not in command: + # Only consider relocs commands that redirect output to a file. + # If there's no redirection, we assume it produces no output file and therefore has no input we care about. + return [] + relocs_command, _ = command.split(">", 1) + command_parts = shlex.split(relocs_command) + # expect command_parts to be ["relocs", options, input] + return [command_parts[-1]] + + +def _parse_mk_elfconfig_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["mk_elfconfig", "<", input, ">", output] + return [positionals[2]] + + +def _parse_flex_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.l` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".l"): + return [part] + raise CmdParsingError("Could not find .l input source file in command") + + +def _parse_bison_command(command: str) -> list[PathStr]: + parts = shlex.split(command) + # expect last positional argument ending in `.y` to be the input file + for part in reversed(parts): + if not part.startswith("-") and part.endswith(".y"): + return [part] + raise CmdParsingError("Could not find input .y input source file in command") + + +def _parse_tools_build_command(command: str) -> list[PathStr]: + positionals = tokenize_single_command_positionals_only(command) + # expect positionals to be ["tools/build", "input1", "input2", "input3", "output"] + return positionals[1:-1] + + +def _parse_extract_cert_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be [path/to/extract-cert, input, output] + input = command_parts[1] + if not input: + return [] + return [input] + + +def _parse_dtc_command(command: str) -> list[PathStr]: + wno_flags = [command_part for command_part in shlex.split(command) if command_part.startswith("-Wno-")] + command_parts = tokenize_single_command(command, flag_options=wno_flags) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be [path/to/dtc, input] + return [positionals[1]] + + +def _parse_bindgen_command(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + header_file_input_paths = [part for part in command_parts if part.endswith(".h")] + return header_file_input_paths + + +def _parse_gen_header(command: str) -> list[PathStr]: + command_parts = shlex.split(command) + # expect command parts to be ["python3", path/to/gen_headers.py, ..., "--xml", input] + i = next((i for i, token in enumerate(command_parts) if token == "--xml"), None) + if i is None: + raise CmdParsingError(f"Expected --xml input file in gen_headers command but got {command}") + return [command_parts[i + 1]] + +def _parse_mkuboot_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command) + # mkuboot.sh passes all args to mkimage; -d specifies the data/input image file + for part in command_parts: + if isinstance(part, Option) and part.name == "-d" and part.value is not None: + return [part.value] + raise CmdParsingError("Could not find -d (data file) option in mkuboot.sh command") + + +def _parse_syscallnr_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip()) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/syscallnr.sh, input, output] + return [positionals[2]] + + +def _parse_gen_kernel_hwcaps_command(command: str) -> list[PathStr]: + command_parts = tokenize_single_command(command.strip(), flag_options=["-e"]) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + # expect positionals to be ["sh", path/to/gen-kernel-hwcaps.sh, input] + return [positionals[2]] + + +class CommandParserRegistry: + """ + Registry mapping command patterns to their input-file parsers. + """ + + def __init__(self, entries: list[CommandParserRegistryEntry]) -> None: + self._entries = entries + + def __iter__(self) -> Iterator[CommandParserRegistryEntry]: + return iter(self._entries) + + @staticmethod + def create() -> "CommandParserRegistry": + def env_or_default_pattern(env_value: str | None, default_pattern: str) -> str: + if env_value is None or not env_value.strip(): + return default_pattern + return rf"(?:{re.escape(env_value.strip())}|{default_pattern})" + + cc_pattern = env_or_default_pattern(Environment.CC(), r"([^\s]+-)?(gcc|clang)") + ld_pattern = env_or_default_pattern(Environment.LD(), r"([^\s]+-)?ld") + ar_pattern = env_or_default_pattern(Environment.AR(), r"([^\s]+-)?ar") + nm_pattern = env_or_default_pattern(Environment.NM(), r"([^\s]+-)?nm") + objcopy_pattern = env_or_default_pattern(Environment.OBJCOPY(), r"([^\s]+-)?objcopy") + strip_pattern = env_or_default_pattern(Environment.STRIP(), r"([^\s]+-)?strip") + + entries: list[CommandParserRegistryEntry] = [ + # Compound commands + (re.compile(r"\(.*?\)\s*>", re.DOTALL), _parse_compound_command), + (re.compile(r"\{.*?\}\s*>", re.DOTALL), _parse_compound_command), + # Standard Unix utilities and system tools + (re.compile(r"^rm\b"), _parse_noop), + (re.compile(r"^mkdir\b"), _parse_noop), + (re.compile(r"^touch\b"), _parse_noop), + (re.compile(r"^cp\b"), _parse_cp_command), + (re.compile(r"^truncate\b"), _parse_noop), + (re.compile(r"^cat\b.*?[\|>]"), lambda c: _parse_cat_command(c.split("|")[0].split(">")[0])), + (re.compile(r"^echo[^|]*$"), _parse_noop), + (re.compile(r"^sed.*?>"), lambda c: _parse_sed_command(c.split(">")[0])), + (re.compile(r"^sed\b"), _parse_noop), + (re.compile(r"^awk.*?<.*?>"), lambda c: [c.split("<")[1].split(">")[0]]), + (re.compile(r"^awk.*?>"), lambda c: _parse_awk(c.split(">")[0])), + (re.compile(r"^(/bin/)?true\b"), _parse_noop), + (re.compile(r"^(/bin/)?false\b"), _parse_noop), + (re.compile(r"^openssl\s+req.*?-new.*?-keyout"), _parse_noop), + # Compilers and code generators + # (C/LLVM toolchain, Rust, Flex/Bison, Bindgen, Perl, etc.) + ( + re.compile(rf"^{cc_pattern}\b"), + lambda command: _parse_gcc_or_clang_command(re.sub(rf"^{cc_pattern}\b", "gcc", command, count=1)), + ), + ( + re.compile(rf"^{ld_pattern}\b"), + lambda command: _parse_ld_command(re.sub(rf"^{ld_pattern}\b", "ld", command, count=1)), + ), + ( + re.compile(rf"^printf\b.*\| xargs {ar_pattern}\b"), + lambda command: _parse_ar_piped_xargs_command( + re.sub(rf"xargs {ar_pattern}\b", "xargs ar", command, count=1) + ), + ), + ( + re.compile(rf"^{ar_pattern}\b"), + lambda command: _parse_ar_command(re.sub(rf"^{ar_pattern}\b", "ar", command, count=1)), + ), + ( + re.compile(rf"^{nm_pattern}\b.*?\|"), + lambda command: _parse_nm_piped_command(re.sub(rf"^{nm_pattern}\b", "nm", command, count=1)), + ), + ( + re.compile(rf"^{objcopy_pattern}\b"), + lambda command: _parse_objcopy_command(re.sub(rf"^{objcopy_pattern}\b", "objcopy", command, count=1)), + ), + ( + re.compile(rf"^{strip_pattern}\b"), + lambda command: _parse_strip_command(re.sub(rf"^{strip_pattern}\b", "strip", command, count=1)), + ), + (re.compile(r".*?rustc\b"), _parse_rustc_command), + (re.compile(r".*?rustdoc\b"), _parse_rustdoc_command), + (re.compile(r"^flex\b"), _parse_flex_command), + (re.compile(r"^bison\b"), _parse_bison_command), + (re.compile(r"^bindgen\b"), _parse_bindgen_command), + (re.compile(r"^perl\b"), _parse_perl_command), + # Kernel-specific build scripts and tools + (re.compile(r"^(.*/)?link-vmlinux\.sh\b"), _parse_link_vmlinux_command), + (re.compile(r"sh (.*/)?syscallhdr\.sh\b"), _parse_syscallhdr_command), + (re.compile(r"sh (.*/)?syscalltbl\.sh\b"), _parse_syscalltbl_command), + (re.compile(r"sh (.*/)?mkcapflags\.sh\b"), _parse_mkcapflags_command), + (re.compile(r"sh (.*/)?orc_hash\.sh\b"), _parse_orc_hash_command), + (re.compile(r"sh (.*/)?xen-hypercalls\.sh\b"), _parse_xen_hypercalls_command), + (re.compile(r"sh (.*/)?gen_initramfs\.sh\b"), _parse_gen_initramfs_command), + (re.compile(r"sh (.*/)?checkundef\.sh\b"), _parse_noop), + (re.compile(r"(bash|sh) (.*/)?mkuboot\.sh\b"), _parse_mkuboot_command), + (re.compile(r"sh (.*/)?syscallnr\.sh\b"), _parse_syscallnr_command), + (re.compile(r"(/bin/)?sh (.*/)?gen-kernel-hwcaps\.sh\b"), lambda c: _parse_gen_kernel_hwcaps_command(c.split(">")[0])), + (re.compile(r"(.*/)?vdso2c\b"), _parse_vdso2c_command), + (re.compile(r"(.*/)?vdsomunge\b"), _parse_vdsomunge_command), + (re.compile(r"^(.*/)?mkpiggy.*?>"), _parse_mkpiggy_command), + (re.compile(r"^(.*/)?relocs\b"), _parse_relocs_command), + (re.compile(r"^(.*/)?mk_elfconfig.*?<.*?>"), _parse_mk_elfconfig_command), + (re.compile(r"^(.*/)?tools/build\b"), _parse_tools_build_command), + (re.compile(r"^(.*/)?certs/extract-cert"), _parse_extract_cert_command), + (re.compile(r"^(.*/)?scripts/dtc/dtc\b"), _parse_dtc_command), + (re.compile(r"^(.*/)?pnmtologo\b"), _parse_pnm_to_logo_command), + (re.compile(r"^(.*/)?kernel/pi/relacheck"), _parse_relacheck), + (re.compile(r"^(.*/)?gen-hyprel\b"), _parse_gen_hyprel_command), + (re.compile(r"^drivers/gpu/drm/radeon/mkregtable"), lambda c: [c.split(" ")[1]]), + (re.compile(r"(.*/)?genheaders\b"), _parse_noop), + (re.compile(r"^(.*/)?mkcpustr\s+>"), _parse_noop), + (re.compile(r"^(.*/)polgen\b"), _parse_noop), + (re.compile(r"make -f .*/arch/x86/Makefile\.postlink"), _parse_noop), + (re.compile(r"^(.*/)?raid6/mktables\s+>"), _parse_noop), + (re.compile(r"^(.*/)?objtool\b"), _parse_noop), + (re.compile(r"^(.*/)?module/gen_test_kallsyms.sh"), _parse_noop), + (re.compile(r"^(.*/)?gen_header.py"), _parse_gen_header), + (re.compile(r"^(.*/)?scripts/rustdoc_test_gen"), _parse_noop), + ] + return CommandParserRegistry(entries) diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py new file mode 100644 index 000000000000..4749f4bd669e --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +from dataclasses import dataclass + + +# If Block pattern to match a simple, single-level if-then-fi block. Nested If blocks are not supported. +IF_BLOCK_PATTERN = re.compile( + r""" + ^if(.*?);\s* # Match 'if ;' (non-greedy) + then(.*?);\s* # Match 'then ;' (non-greedy) + fi\b # Match 'fi' + """, + re.VERBOSE, +) + + +@dataclass +class IfBlock: + condition: str + then_statement: str + + +def _unwrap_outer_parentheses(s: str) -> str: + s = s.strip() + if not (s.startswith("(") and s.endswith(")")): + return s + + count = 0 + for i, char in enumerate(s): + if char == "(": + count += 1 + elif char == ")": + count -= 1 + # If count is 0 before the end, outer parentheses don't match + if count == 0 and i != len(s) - 1: + return s + + # outer parentheses do match, unwrap once + return _unwrap_outer_parentheses(s[1:-1]) + + +def _find_first_top_level_command_separator( + commands: str, separators: list[str] = [";", "&&"] +) -> tuple[int | None, int | None]: + def is_escaped(index: int) -> bool: + preceding = commands[:index] + return (len(preceding) - len(preceding.rstrip("\\"))) % 2 == 1 + + in_single_quote = False + in_double_quote = False + in_curly_braces = 0 + in_braces = 0 + for i, char in enumerate(commands): + if char == "'" and not in_double_quote and not is_escaped(i): + # Toggle single quote state (unless inside double quotes or escaped) + in_single_quote = not in_single_quote + elif char == '"' and not in_single_quote and not is_escaped(i): + # Toggle double quote state (unless inside single quotes or escaped) + in_double_quote = not in_double_quote + + if in_single_quote or in_double_quote: + continue + + # Toggle braces state + if char == "{": + in_curly_braces += 1 + if char == "}": + in_curly_braces -= 1 + + if char == "(": + in_braces += 1 + if char == ")": + in_braces -= 1 + + if in_curly_braces > 0 or in_braces > 0: + continue + + # return found separator position and separator length + for separator in separators: + if commands[i : i + len(separator)] == separator: + return i, len(separator) + + return None, None + + +def split_commands(commands: str) -> list[str | IfBlock]: + """ + Splits a string of command-line commands into individual parts. + + This function handles: + - Top-level command separators (e.g., `;` and `&&`) to split multiple commands. + - Conditional if-blocks, returning them as `IfBlock` instances. + - Preserves the order of commands and trims whitespace. + + Args: + commands (str): The raw command string. + + Returns: + list[str | IfBlock]: A list of single commands or `IfBlock` objects. + """ + single_commands: list[str | IfBlock] = [] + remaining_commands = _unwrap_outer_parentheses(commands) + while len(remaining_commands) > 0: + remaining_commands = remaining_commands.strip() + + # if block + matched_if = IF_BLOCK_PATTERN.match(remaining_commands) + if matched_if: + condition, then_statement = matched_if.groups() + single_commands.append(IfBlock(condition.strip(), then_statement.strip())) + full_matched = matched_if.group(0) + remaining_commands = remaining_commands.removeprefix(full_matched).lstrip("; \n") + continue + + # command until next separator + separator_position, separator_length = _find_first_top_level_command_separator(remaining_commands) + if separator_position is not None and separator_length is not None: + single_commands.append(remaining_commands[:separator_position].strip()) + remaining_commands = remaining_commands[separator_position + separator_length :].strip() + continue + + # single last command + single_commands.append(remaining_commands) + break + + return single_commands diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py new file mode 100644 index 000000000000..6a7ea4787aa1 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import sbom.sbom_logging as sbom_logging +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.command_parser_registry import CommandParserRegistry +from sbom.cmd_graph.savedcmd_parser.tokenizer import CmdParsingError +from sbom.path_utils import PathStr + +DEFAULT_COMMAND_PARSER_REGISTRY = CommandParserRegistry.create() + + +def parse_inputs_from_commands( + commands: str, + fail_on_unknown_build_command: bool, + registry: CommandParserRegistry | None = None, +) -> list[PathStr]: + """ + Extract input files referenced in a set of command-line commands. + + Args: + commands (str): Command line expression to parse. + fail_on_unknown_build_command (bool): Whether to fail if an unknown build command is encountered. If False, errors are logged as warnings. + registry (CommandParserRegistry | None): Registry of single command parsers. + + Returns: + list[PathStr]: List of input file paths required by the commands. + """ + + def log_error_or_warning(message: str, /, **kwargs: str) -> None: + if fail_on_unknown_build_command: + sbom_logging.error(message, **kwargs) + else: + sbom_logging.warning(message, **kwargs) + + if registry is None: + registry = DEFAULT_COMMAND_PARSER_REGISTRY + + input_files: list[PathStr] = [] + for single_command in split_commands(commands): + if isinstance(single_command, IfBlock): + inputs = parse_inputs_from_commands(single_command.then_statement, fail_on_unknown_build_command, registry) + if inputs: + log_error_or_warning( + "Skipped parsing command {then_statement} because input files in IfBlock 'then' statement are not supported", + then_statement=single_command.then_statement, + ) + continue + + matched_parser = next((parser for pattern, parser in registry if pattern.match(single_command)), None) + if matched_parser is None: + log_error_or_warning( + "Skipped parsing command {single_command} because no matching parser was found", + single_command=single_command, + ) + continue + try: + inputs = matched_parser(single_command) + input_files.extend(inputs) + except (CmdParsingError, IndexError) as e: + log_error_or_warning( + "Skipped parsing command {single_command} because of command parsing error: {error_message}", + single_command=single_command, + error_message=str(e), + ) + + return [input.strip().rstrip("/") for input in input_files] diff --git a/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py new file mode 100644 index 000000000000..1bf081f40be7 --- /dev/null +++ b/scripts/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import shlex +from dataclasses import dataclass +from typing import Union + + +class CmdParsingError(Exception): + pass + + +@dataclass +class Option: + name: str + value: str | None = None + + +@dataclass +class Positional: + value: str + + +_SUBCOMMAND_PATTERN = re.compile(r"\$\$\(([^()]*)\)") +"""Pattern to match $$(...) blocks""" + + +def tokenize_single_command(command: str, flag_options: list[str] | None = None) -> list[Union[Option, Positional]]: + """ + Parse a shell command into a list of Options and Positionals. + - Positional: the command and any positional arguments. + - Options: handles flags and options with values provided as space-separated, or equals-sign + (e.g., '--opt val', '--opt=val', '--flag'). + + Args: + command: Command line string. + flag_options: Options that are flags without values (e.g., '--verbose'). + + Returns: + List of `Option` and `Positional` objects in command order. + """ + + # Wrap all $$(...) blocks in double quotes to prevent shlex from splitting them. + command_with_protected_subcommands = _SUBCOMMAND_PATTERN.sub(lambda m: f'"$$({m.group(1)})"', command) + tokens = shlex.split(command_with_protected_subcommands) + + parsed: list[Option | Positional] = [] + i = 0 + while i < len(tokens): + token = tokens[i] + + # Positional + if not token.startswith("-"): + parsed.append(Positional(token)) + i += 1 + continue + + # Option without value (--flag) + if (token.startswith("-") and i + 1 < len(tokens) and tokens[i + 1].startswith("-")) or ( + flag_options and token in flag_options + ): + parsed.append(Option(name=token)) + i += 1 + continue + + # Option with equals sign (--opt=val) + if "=" in token: + name, value = token.split("=", 1) + parsed.append(Option(name=name, value=value)) + i += 1 + continue + + # Option with space-separated value (--opt val) + if i + 1 < len(tokens) and not tokens[i + 1].startswith("-"): + parsed.append(Option(name=token, value=tokens[i + 1])) + i += 2 + continue + + raise CmdParsingError(f"Unrecognized token: {token} in command {command}") + + return parsed + + +def tokenize_single_command_positionals_only(command: str) -> list[str]: + command_parts = tokenize_single_command(command) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + if len(positionals) != len(command_parts): + raise CmdParsingError( + f"Invalid command format: expected positional arguments only but got options in command {command}." + ) + return positionals diff --git a/scripts/sbom/sbom/config.py b/scripts/sbom/sbom/config.py new file mode 100644 index 000000000000..6811f782943e --- /dev/null +++ b/scripts/sbom/sbom/config.py @@ -0,0 +1,320 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +from enum import Enum +import os +from typing import Any +from sbom.path_utils import PathStr + + +class KernelSpdxDocumentKind(Enum): + SOURCE = "source" + BUILD = "build" + OUTPUT = "output" + + +@dataclass +class KernelSbomConfig: + src_tree: PathStr + """Absolute path to the Linux kernel source directory.""" + + obj_tree: PathStr + """Absolute path to the build output directory.""" + + root_paths: list[PathStr] + """List of paths to root outputs (relative to obj_tree) to base the SBOM on.""" + + generate_spdx: bool + """Whether to generate SPDX SBOM documents. If False, no SPDX files are created.""" + + spdx_file_names: dict[KernelSpdxDocumentKind, str] + """If `generate_spdx` is True, defines the file names for each SPDX SBOM kind + (source, build, output) to store on disk.""" + + generate_used_files: bool + """Whether to generate a flat list of all source files used in the build. + If False, no used-files document is created.""" + + used_files_file_name: str + """If `generate_used_files` is True, specifies the file name for the used-files document.""" + + output_directory: PathStr + """Path to the directory where the generated output documents will be saved.""" + + debug: bool + """Whether to enable debug logging.""" + + fail_on_unknown_build_command: bool + """Whether to fail if an unknown build command is encountered in a .cmd file.""" + + write_output_on_error: bool + """Whether to write output documents even if errors occur.""" + + created: datetime + """Datetime to use for the SPDX created property of the CreationInfo element.""" + + spdxId_prefix: str + """Prefix to use for all SPDX element IDs.""" + + build_type: str + """SPDX buildType property to use for all Build elements.""" + + build_id: str | None + """SPDX buildId property to use for all Build elements.""" + + package_license: str + """License expression applied to all SPDX Packages.""" + + package_version: str | None + """Version string applied to all SPDX Packages.""" + + package_copyright_text: str | None + """Copyright text applied to all SPDX Packages.""" + + prettify_json: bool + """Whether to pretty-print generated SPDX JSON documents.""" + + +def _parse_cli_arguments(parser: argparse.ArgumentParser) -> dict[str, Any]: + """ + Parse command-line arguments using argparse. + + Returns: + Dictionary of parsed arguments. + """ + parser.add_argument( + "--src-tree", + default="../linux", + help="Path to the kernel source tree (default: ../linux)", + ) + parser.add_argument( + "--obj-tree", + default="../linux/kernel_build", + help="Path to the build output directory (default: ../linux/kernel_build)", + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--roots", + nargs="+", + help="Space-separated list of paths relative to obj-tree for which the SBOM will be created.\n" + "Cannot be used together with --roots-file.", + ) + group.add_argument( + "--roots-file", + help="Path to a file containing the root paths (one per line). Cannot be used together with --roots.", + ) + parser.add_argument( + "--generate-spdx", + action="store_true", + default=False, + help=( + "Whether to create sbom-source.spdx.json, sbom-build.spdx.json and " + "sbom-output.spdx.json documents (default: False)" + ), + ) + parser.add_argument( + "--generate-used-files", + action="store_true", + default=False, + help=( + "Whether to create the sbom.used-files.txt file, a flat list of all " + "source files used for the kernel build.\n" + "If src-tree and obj-tree are equal it is not possible to reliably " + "classify source files.\n" + "In this case sbom.used-files.txt will contain all files used for the " + "kernel build including all build artifacts. (default: False)" + ), + ) + parser.add_argument( + "--output-directory", + default=".", + help="Path to the directory where the generated output documents will be stored (default: .)", + ) + parser.add_argument( + "--debug", + action="store_true", + default=False, + help="Enable debug logs (default: False)", + ) + + # Error handling settings + parser.add_argument( + "--do-not-fail-on-unknown-build-command", + action="store_true", + default=False, + help=( + "Whether to fail if an unknown build command is encountered in a .cmd file.\n" + "If set to True, errors are logged as warnings instead. (default: False)" + ), + ) + parser.add_argument( + "--write-output-on-error", + action="store_true", + default=False, + help=( + "Write output documents even if errors occur. The resulting documents " + "may be incomplete.\n" + "A summary of warnings and errors can be found in the 'comment' property " + "of the CreationInfo element. (default: False)" + ), + ) + + # SPDX specific options + spdx_group = parser.add_argument_group("SPDX options", "Options for customizing SPDX document generation") + spdx_group.add_argument( + "--spdxId-prefix", + default="urn:spdx.dev:", + help="The prefix to use for all spdxId properties. (default: urn:spdx.dev:)", + ) + spdx_group.add_argument( + "--build-type", + default="urn:spdx.dev:Kbuild", + help="The SPDX buildType property to use for all Build elements. (default: urn:spdx.dev:Kbuild)", + ) + spdx_group.add_argument( + "--build-id", + default=None, + help="The SPDX buildId property to use for all Build elements.\n" + "If not provided the spdxId of the high level Build element is used as the buildId. (default: None)", + ) + spdx_group.add_argument( + "--package-license", + default="NOASSERTION", + help=( + "The SPDX licenseExpression property to use for the LicenseExpression " + "linked to all SPDX Package elements. (default: NOASSERTION)" + ), + ) + spdx_group.add_argument( + "--package-version", + default=None, + help="The SPDX packageVersion property to use for all SPDX Package elements. (default: None)", + ) + spdx_group.add_argument( + "--package-copyright-text", + default=None, + help=( + "The SPDX copyrightText property to use for all SPDX Package elements.\n" + "If not specified, and if a COPYING file exists in the source tree,\n" + "the package-copyright-text is set to the content of this file. " + "(default: None)" + ), + ) + spdx_group.add_argument( + "--prettify-json", + action="store_true", + default=False, + help="Whether to pretty print the generated spdx.json documents (default: False)", + ) + + args = vars(parser.parse_args()) + return args + + +def get_config() -> KernelSbomConfig: + """ + Parse command-line arguments and construct the configuration object. + + Returns: + KernelSbomConfig: Configuration object with all settings for SBOM generation. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="Generate SPDX SBOM documents for kernel builds", + ) + args = _parse_cli_arguments(parser) + + # Extract and validate cli arguments + src_tree = os.path.realpath(args["src_tree"]) + obj_tree = os.path.realpath(args["obj_tree"]) + root_paths = [] + if args["roots_file"]: + with open(args["roots_file"], "rt", encoding="utf-8") as f: + root_paths = [root.strip() for root in f.readlines()] + if len(root_paths) == 0: + parser.error("--roots-file must contain at least one path") + else: + root_paths = args["roots"] + _validate_path_arguments(parser, src_tree, obj_tree, root_paths) + + generate_spdx = args["generate_spdx"] + generate_used_files = args["generate_used_files"] + output_directory = os.path.realpath(args["output_directory"]) + debug = args["debug"] + + fail_on_unknown_build_command = not args["do_not_fail_on_unknown_build_command"] + write_output_on_error = args["write_output_on_error"] + + created = datetime.fromtimestamp( + max([os.path.getmtime(os.path.join(obj_tree, root_path)) for root_path in root_paths]), + tz=timezone.utc, + ) + spdxId_prefix = args["spdxId_prefix"] + build_type = args["build_type"] + build_id = args["build_id"] + package_license = args["package_license"] + package_version = args["package_version"] if args["package_version"] is not None else None + package_copyright_text: str | None = None + if args["package_copyright_text"] is not None: + package_copyright_text = args["package_copyright_text"] + elif os.path.isfile(copying_path := os.path.join(src_tree, "COPYING")): + with open(copying_path, "r", encoding="utf-8") as f: + package_copyright_text = f.read() + prettify_json = args["prettify_json"] + + # Hardcoded config + spdx_file_names = { + KernelSpdxDocumentKind.SOURCE: "sbom-source.spdx.json", + KernelSpdxDocumentKind.BUILD: "sbom-build.spdx.json", + KernelSpdxDocumentKind.OUTPUT: "sbom-output.spdx.json", + } + used_files_file_name = "sbom.used-files.txt" + + return KernelSbomConfig( + src_tree=src_tree, + obj_tree=obj_tree, + root_paths=root_paths, + generate_spdx=generate_spdx, + spdx_file_names=spdx_file_names, + generate_used_files=generate_used_files, + used_files_file_name=used_files_file_name, + output_directory=output_directory, + debug=debug, + fail_on_unknown_build_command=fail_on_unknown_build_command, + write_output_on_error=write_output_on_error, + created=created, + spdxId_prefix=spdxId_prefix, + build_type=build_type, + build_id=build_id, + package_license=package_license, + package_version=package_version, + package_copyright_text=package_copyright_text, + prettify_json=prettify_json, + ) + + +def _validate_path_arguments( + parser: argparse.ArgumentParser, + src_tree: PathStr, + obj_tree: PathStr, + root_paths: list[PathStr], +) -> None: + """ + Validate that the provided paths exist. + + Args: + parser: The argument parser, used to emit well-formatted error messages. + src_tree: Absolute path to the source tree. + obj_tree: Absolute path to the object tree. + root_paths: List of root paths relative to obj_tree. + """ + if not os.path.exists(src_tree): + parser.error(f"--src-tree {src_tree} does not exist") + if not os.path.exists(obj_tree): + parser.error(f"--obj-tree {obj_tree} does not exist") + for root_path in root_paths: + if not os.path.isfile(root_path_absolute := os.path.join(obj_tree, root_path)): + parser.error(f"path to root artifact {root_path_absolute} is not a file") diff --git a/scripts/sbom/sbom/environment.py b/scripts/sbom/sbom/environment.py new file mode 100644 index 000000000000..4304066fe974 --- /dev/null +++ b/scripts/sbom/sbom/environment.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os + +KERNEL_BUILD_VARIABLES_ALLOWLIST = [ + "AFLAGS_KERNEL", + "AFLAGS_MODULE", + "AR", + "ARCH", + "ARCH_CORE", + "ARCH_DRIVERS", + "ARCH_LIB", + "AWK", + "BASH", + "BINDGEN", + "BITS", + "CC", + "CC_FLAGS_FPU", + "CC_FLAGS_NO_FPU", + "CFLAGS_GCOV", + "CFLAGS_KERNEL", + "CFLAGS_MODULE", + "CHECK", + "CHECKFLAGS", + "CLIPPY_CONF_DIR", + "CONFIG_SHELL", + "CPP", + "CROSS_COMPILE", + "CURDIR", + "GNUMAKEFLAGS", + "HOSTCC", + "HOSTCXX", + "HOSTPKG_CONFIG", + "HOSTRUSTC", + "INSTALLKERNEL", + "INSTALL_DTBS_PATH", + "INSTALL_HDR_PATH", + "INSTALL_PATH", + "KBUILD_AFLAGS", + "KBUILD_AFLAGS_KERNEL", + "KBUILD_AFLAGS_MODULE", + "KBUILD_BUILTIN", + "KBUILD_CFLAGS", + "KBUILD_CFLAGS_KERNEL", + "KBUILD_CFLAGS_MODULE", + "KBUILD_CHECKSRC", + "KBUILD_CLIPPY", + "KBUILD_CPPFLAGS", + "KBUILD_EXTMOD", + "KBUILD_EXTRA_WARN", + "KBUILD_HOSTCFLAGS", + "KBUILD_HOSTCXXFLAGS", + "KBUILD_HOSTLDFLAGS", + "KBUILD_HOSTLDLIBS", + "KBUILD_HOSTRUSTFLAGS", + "KBUILD_IMAGE", + "KBUILD_LDFLAGS", + "KBUILD_LDFLAGS_MODULE", + "KBUILD_LDS", + "KBUILD_MODULES", + "KBUILD_PROCMACROLDFLAGS", + "KBUILD_RUSTFLAGS", + "KBUILD_RUSTFLAGS_KERNEL", + "KBUILD_RUSTFLAGS_MODULE", + "KBUILD_USERCFLAGS", + "KBUILD_USERLDFLAGS", + "KBUILD_VERBOSE", + "KBUILD_VMLINUX_LIBS", + "KBZIP2", + "KCONFIG_CONFIG", + "KERNELDOC", + "KERNELRELEASE", + "KERNELVERSION", + "KGZIP", + "KLZOP", + "LC_COLLATE", + "LC_NUMERIC", + "LD", + "LDFLAGS_MODULE", + "LEX", + "LINUXINCLUDE", + "LZ4", + "LZMA", + "MAKE", + "MAKEFILES", + "MAKEFILE_LIST", + "MAKEFLAGS", + "MAKELEVEL", + "MAKEOVERRIDES", + "MAKE_COMMAND", + "MAKE_HOST", + "MAKE_TERMERR", + "MAKE_TERMOUT", + "MAKE_VERSION", + "MFLAGS", + "MODLIB", + "NM", + "NOSTDINC_FLAGS", + "O", + "OBJCOPY", + "OBJCOPYFLAGS", + "OBJDUMP", + "PAHOLE", + "PATCHLEVEL", + "PERL", + "PYTHON3", + "Q", + "RCS_FIND_IGNORE", + "READELF", + "REALMODE_CFLAGS", + "RESOLVE_BTFIDS", + "RETHUNK_CFLAGS", + "RETHUNK_RUSTFLAGS", + "RETPOLINE_CFLAGS", + "RETPOLINE_RUSTFLAGS", + "RETPOLINE_VDSO_CFLAGS", + "RUSTC", + "RUSTC_BOOTSTRAP", + "RUSTC_OR_CLIPPY", + "RUSTC_OR_CLIPPY_QUIET", + "RUSTDOC", + "RUSTFLAGS_KERNEL", + "RUSTFLAGS_MODULE", + "RUSTFMT", + "SRCARCH", + "STRIP", + "SUBLEVEL", + "SUFFIXES", + "TAR", + "UTS_MACHINE", + "VERSION", + "VPATH", + "XZ", + "YACC", + "ZSTD", + "building_out_of_srctree", + "cross_compiling", + "objtree", + "quiet", + "rust_common_flags", + "srcroot", + "srctree", + "sub_make_done", + "subdir", +] + + +class Environment: + """ + Read-only accessor for kernel build environment variables. + """ + + @classmethod + def KERNEL_BUILD_VARIABLES(cls) -> dict[str, str]: + return { + name: value.strip() + for name in KERNEL_BUILD_VARIABLES_ALLOWLIST + if (value := os.getenv(name)) is not None and value.strip() + } + + @classmethod + def ARCH(cls) -> str | None: + return os.getenv("ARCH") + + @classmethod + def SRCARCH(cls) -> str | None: + return os.getenv("SRCARCH") + + @classmethod + def CC(cls) -> str | None: + return os.getenv("CC") + + @classmethod + def LD(cls) -> str | None: + return os.getenv("LD") + + @classmethod + def AR(cls) -> str | None: + return os.getenv("AR") + + @classmethod + def NM(cls) -> str | None: + return os.getenv("NM") + + @classmethod + def OBJCOPY(cls) -> str | None: + return os.getenv("OBJCOPY") + + @classmethod + def STRIP(cls) -> str | None: + return os.getenv("STRIP") diff --git a/scripts/sbom/sbom/path_utils.py b/scripts/sbom/sbom/path_utils.py new file mode 100644 index 000000000000..29820046dc88 --- /dev/null +++ b/scripts/sbom/sbom/path_utils.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +from functools import lru_cache + +PathStr = str +"""Filesystem path represented as a plain string for better performance than pathlib.Path.""" + + +def is_relative_to(path: PathStr, base: PathStr) -> bool: + return os.path.commonpath([path, base]) == base + +@lru_cache(maxsize=None) +def has_link(path: PathStr) -> bool: + """Returns True if path or any of its ancestor directories is a symlink. Results are cached to avoid duplicate lstat syscalls.""" + if os.path.islink(path): + return True + parent = os.path.dirname(path) + if parent == path: + return False + return has_link(parent) diff --git a/scripts/sbom/sbom/sbom_logging.py b/scripts/sbom/sbom/sbom_logging.py new file mode 100644 index 000000000000..fbc53cc77ef4 --- /dev/null +++ b/scripts/sbom/sbom/sbom_logging.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import logging +import inspect +from typing import Literal + + +MessageTemplate = str + + +class MessageLogger: + """Logger that suppresses repeated messages and stores a summary of all logged messages.""" + + _messages: dict[MessageTemplate, list[str]] + _message_counts: dict[MessageTemplate, int] + _repeated_logs_limit: int + """Maximum number of repeated messages of the same type to log before suppressing further output.""" + + def __init__(self, level: Literal["error", "warning"], repeated_logs_limit: int = 3) -> None: + self._level = level + self._messages = {} + self._message_counts = {} + self._repeated_logs_limit = repeated_logs_limit + + def log(self, template: MessageTemplate, /, **kwargs: str) -> None: + """Log a message based on a template and optional variables. Example: `log("Missing {path}", path=str(p))`.""" + message = template + for key, value in kwargs.items(): + message = message.replace("{" + key + "}", value) + if template not in self._messages: + self._messages[template] = [] + self._message_counts[template] = 0 + self._message_counts[template] += 1 + if self._message_counts[template] <= self._repeated_logs_limit: + if self._level == "error": + logging.error(message) + elif self._level == "warning": + logging.warning(message) + self._messages[template].append(message) + + def get_summary(self) -> str: + if len(self._messages) == 0: + return "" + summary: list[str] = [f"Summarize {self._level}s:"] + for template, messages in self._messages.items(): + for message in messages: + summary.append(message) + n_suppressed_messages = self._message_counts[template] - self._repeated_logs_limit + if n_suppressed_messages > 0: + instances = "instance" if n_suppressed_messages == 1 else "instances" + summary.append(f"... (Found {n_suppressed_messages} more {instances} of this {self._level})") + return "\n".join(summary) + + def has_messages(self) -> bool: + return len(self._message_counts) > 0 + + +_warning_logger: MessageLogger +_error_logger: MessageLogger + + +def warning(msg_template: MessageTemplate, /, **kwargs: str) -> None: + _warning_logger.log(msg_template, **kwargs) + + +def error(msg_template: MessageTemplate, /, **kwargs: str) -> None: + frame = inspect.currentframe() + caller_frame = frame.f_back if frame else None + info = inspect.getframeinfo(caller_frame) if caller_frame else None + if info: + msg_template = f'File "{info.filename}", line {info.lineno}, in {info.function}\n{msg_template}' + _error_logger.log(msg_template, **kwargs) + + +def summarize_warnings() -> str: + return _warning_logger.get_summary() + + +def summarize_errors() -> str: + return _error_logger.get_summary() + + +def has_errors() -> bool: + return _error_logger.has_messages() + + +def init() -> None: + global _warning_logger, _error_logger + _warning_logger = MessageLogger("warning") + _error_logger = MessageLogger("error") + + +init() diff --git a/scripts/sbom/sbom/spdx/__init__.py b/scripts/sbom/sbom/spdx/__init__.py new file mode 100644 index 000000000000..4097b59f8f17 --- /dev/null +++ b/scripts/sbom/sbom/spdx/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .spdxId import SpdxId, SpdxIdGenerator +from .serialization import JsonLdSpdxDocument + +__all__ = ["JsonLdSpdxDocument", "SpdxId", "SpdxIdGenerator"] diff --git a/scripts/sbom/sbom/spdx/build.py b/scripts/sbom/sbom/spdx/build.py new file mode 100644 index 000000000000..a39ec9c09b16 --- /dev/null +++ b/scripts/sbom/sbom/spdx/build.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from sbom.spdx.core import DictionaryEntry, Element, Hash + + +@dataclass(kw_only=True) +class Build(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Build/Classes/Build/""" + + type: str = field(init=False, default="build_Build") + build_buildType: str + build_buildId: str + build_environment: list[DictionaryEntry] = field(default_factory=list) + build_configSourceUri: list[str] = field(default_factory=list) + build_configSourceDigest: list[Hash] = field(default_factory=list) diff --git a/scripts/sbom/sbom/spdx/core.py b/scripts/sbom/sbom/spdx/core.py new file mode 100644 index 000000000000..7eb376a1cd88 --- /dev/null +++ b/scripts/sbom/sbom/spdx/core.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field + +from typing import Any, Literal +from sbom.spdx.spdxId import SpdxId + +SPDX_SPEC_VERSION = "3.0.1" + +ExternalIdentifierType = Literal["email", "gitoid", "urlScheme"] +HashAlgorithm = Literal["sha256", "sha512"] +ProfileIdentifierType = Literal["core", "software", "build", "lite", "simpleLicensing"] +RelationshipType = Literal[ + "contains", + "generates", + "hasDeclaredLicense", + "hasInput", + "hasOutput", + "ancestorOf", + "hasDistributionArtifact", + "dependsOn", +] +RelationshipCompleteness = Literal["complete", "incomplete", "noAssertion"] + + +@dataclass +class SpdxObject: + def to_dict(self) -> dict[str, Any]: + def _to_dict(v: Any): + return v.to_dict() if hasattr(v, "to_dict") else v + + d: dict[str, Any] = {} + for field_name in self.__dataclass_fields__: + value = getattr(self, field_name) + if value is None or value == [] or value == "": + continue + + if isinstance(value, Element): + d[field_name] = value.spdxId + elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], Element): # type: ignore + value: list[Element] = value + d[field_name] = [v.spdxId for v in value] + else: + d[field_name] = [_to_dict(v) for v in value] if isinstance(value, list) else _to_dict(value) # type: ignore + return d + + +@dataclass(kw_only=True) +class IntegrityMethod(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/IntegrityMethod/""" + + +@dataclass(kw_only=True) +class Hash(IntegrityMethod): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Hash/""" + + type: str = field(init=False, default="Hash") + hashValue: str + algorithm: HashAlgorithm + + +@dataclass(kw_only=True) +class Element(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Element/""" + + type: str = field(init=False, default="Element") + spdxId: SpdxId + creationInfo: str = "_:creationinfo" + name: str | None = None + verifiedUsing: list[Hash] = field(default_factory=list) + comment: str | None = None + + +@dataclass(kw_only=True) +class ExternalMap(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/ExternalMap/""" + + type: str = field(init=False, default="ExternalMap") + externalSpdxId: SpdxId + + +@dataclass(kw_only=True) +class NamespaceMap(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/NamespaceMap/""" + + type: str = field(init=False, default="NamespaceMap") + prefix: str + namespace: str + + +@dataclass(kw_only=True) +class ElementCollection(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/ElementCollection/""" + + type: str = field(init=False, default="ElementCollection") + element: list[Element] = field(default_factory=list) + rootElement: list[Element] = field(default_factory=list) + profileConformance: list[ProfileIdentifierType] = field(default_factory=list) + + +@dataclass(kw_only=True) +class SpdxDocument(ElementCollection): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SpdxDocument/""" + + type: str = field(init=False, default="SpdxDocument") + import_: list[ExternalMap] = field(default_factory=list) + namespaceMap: list[NamespaceMap] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return {("import" if k == "import_" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Agent(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Agent/""" + + type: str = field(init=False, default="Agent") + + +@dataclass(kw_only=True) +class SoftwareAgent(Agent): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/SoftwareAgent/""" + + type: str = field(init=False, default="SoftwareAgent") + + +@dataclass(kw_only=True) +class CreationInfo(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/CreationInfo/""" + + type: str = field(init=False, default="CreationInfo") + id: SpdxId = "_:creationinfo" + specVersion: str = SPDX_SPEC_VERSION + createdBy: list[Agent] + created: str + comment: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {("@id" if k == "id" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Relationship(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Relationship/""" + + type: str = field(init=False, default="Relationship") + relationshipType: RelationshipType + from_: Element # underscore because 'from' is a reserved keyword + to: list[Element] + completeness: RelationshipCompleteness | None = None + + def to_dict(self) -> dict[str, Any]: + return {("from" if k == "from_" else k): v for k, v in super().to_dict().items()} + + +@dataclass(kw_only=True) +class Artifact(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Artifact/""" + + type: str = field(init=False, default="Artifact") + + +@dataclass(kw_only=True) +class DictionaryEntry(SpdxObject): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/DictionaryEntry/""" + + type: str = field(init=False, default="DictionaryEntry") + key: str + value: str diff --git a/scripts/sbom/sbom/spdx/serialization.py b/scripts/sbom/sbom/spdx/serialization.py new file mode 100644 index 000000000000..b4df7d368d46 --- /dev/null +++ b/scripts/sbom/sbom/spdx/serialization.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import json +from typing import Any +from sbom.path_utils import PathStr +from sbom.spdx.core import SPDX_SPEC_VERSION, SpdxDocument, SpdxObject + + +class JsonLdSpdxDocument: + """Represents an SPDX document in JSON-LD format for serialization.""" + + graph: list[SpdxObject] + + def __init__(self, graph: list[SpdxObject]) -> None: + """ + Initialize a JSON-LD SPDX document from a graph of SPDX objects. + The graph must contain a single SpdxDocument element. + + Args: + graph: List of SPDX objects representing the complete SPDX document. + """ + self.graph = graph + + @property + def context(self) -> list[str | dict[str, str]]: + spdx_document = next(element for element in self.graph if isinstance(element, SpdxDocument)) + return [ + f"https://spdx.org/rdf/{SPDX_SPEC_VERSION}/spdx-context.jsonld", + {ns.prefix: ns.namespace for ns in spdx_document.namespaceMap}, + ] + + def to_dict(self) -> dict[str, Any]: + """ + Convert the SPDX document to a dictionary representation suitable for JSON serialization. + + Returns: + Dictionary with @context and @graph keys following JSON-LD format. + """ + def _item_to_dict(item: SpdxObject) -> dict: + d = item.to_dict() + if isinstance(item, SpdxDocument): + d.pop("namespaceMap", None) + return d + return { + "@context": self.context, + "@graph": [_item_to_dict(item) for item in self.graph], + } + + def save(self, path: PathStr, prettify: bool) -> None: + """ + Save the SPDX document to a JSON file. + + Args: + path: File path where the document will be saved. + prettify: Whether to pretty-print the JSON with indentation. + """ + with open(path, "w", encoding="utf-8") as f: + if prettify: + json.dump(self.to_dict(), f, indent=2) + else: + json.dump(self.to_dict(), f, separators=(",", ":")) diff --git a/scripts/sbom/sbom/spdx/simplelicensing.py b/scripts/sbom/sbom/spdx/simplelicensing.py new file mode 100644 index 000000000000..750ddd24ad89 --- /dev/null +++ b/scripts/sbom/sbom/spdx/simplelicensing.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from sbom.spdx.core import Element + + +@dataclass(kw_only=True) +class AnyLicenseInfo(Element): + """https://spdx.github.io/spdx-spec/v3.0.1/model/SimpleLicensing/Classes/AnyLicenseInfo/""" + + type: str = field(init=False, default="simplelicensing_AnyLicenseInfo") + + +@dataclass(kw_only=True) +class LicenseExpression(AnyLicenseInfo): + """https://spdx.github.io/spdx-spec/v3.0.1/model/SimpleLicensing/Classes/LicenseExpression/""" + + type: str = field(init=False, default="simplelicensing_LicenseExpression") + simplelicensing_licenseExpression: str diff --git a/scripts/sbom/sbom/spdx/software.py b/scripts/sbom/sbom/spdx/software.py new file mode 100644 index 000000000000..2f46de7c3167 --- /dev/null +++ b/scripts/sbom/sbom/spdx/software.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass, field +from typing import Literal +from sbom.spdx.core import Artifact, ElementCollection, IntegrityMethod + + +SbomType = Literal["source", "build"] +FileKindType = Literal["file", "directory"] +SoftwarePurpose = Literal[ + "source", + "archive", + "library", + "file", + "data", + "configuration", + "executable", + "module", + "application", + "documentation", + "other", +] +ContentIdentifierType = Literal["gitoid", "swhid"] + + +@dataclass(kw_only=True) +class Sbom(ElementCollection): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/Sbom/""" + + type: str = field(init=False, default="software_Sbom") + software_sbomType: list[SbomType] = field(default_factory=list) + + +@dataclass(kw_only=True) +class ContentIdentifier(IntegrityMethod): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/ContentIdentifier/""" + + type: str = field(init=False, default="software_ContentIdentifier") + software_contentIdentifierType: ContentIdentifierType + software_contentIdentifierValue: str + + +@dataclass(kw_only=True) +class SoftwareArtifact(Artifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/SoftwareArtifact/""" + + type: str = field(init=False, default="software_Artifact") + software_primaryPurpose: SoftwarePurpose | None = None + software_copyrightText: str | None = None + software_contentIdentifier: list[ContentIdentifier] = field(default_factory=list) + + +@dataclass(kw_only=True) +class Package(SoftwareArtifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/Package/""" + + type: str = field(init=False, default="software_Package") + name: str # type: ignore + software_packageVersion: str | None = None + + +@dataclass(kw_only=True) +class File(SoftwareArtifact): + """https://spdx.github.io/spdx-spec/v3.0.1/model/Software/Classes/File/""" + + type: str = field(init=False, default="software_File") + name: str # type: ignore + software_fileKind: FileKindType | None = None diff --git a/scripts/sbom/sbom/spdx/spdxId.py b/scripts/sbom/sbom/spdx/spdxId.py new file mode 100644 index 000000000000..589e85c5f706 --- /dev/null +++ b/scripts/sbom/sbom/spdx/spdxId.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from itertools import count +from typing import Iterator + +SpdxId = str + + +class SpdxIdGenerator: + _namespace: str + _prefix: str | None = None + _counter: Iterator[int] + + def __init__(self, namespace: str, prefix: str | None = None) -> None: + """ + Initialize the SPDX ID generator with a namespace. + + Args: + namespace: The full namespace to use for generated IDs. + prefix: Optional. If provided, generated IDs will use this prefix instead of the full namespace. + """ + self._namespace = namespace + self._prefix = prefix + self._counter = count(0) + + def generate(self) -> SpdxId: + return f"{f'{self._prefix}:' if self._prefix else self._namespace}{next(self._counter)}" + + @property + def prefix(self) -> str | None: + return self._prefix + + @property + def namespace(self) -> str: + return self._namespace diff --git a/scripts/sbom/sbom/spdx_graph/__init__.py b/scripts/sbom/sbom/spdx_graph/__init__.py new file mode 100644 index 000000000000..3557b1d51bf9 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from .build_spdx_graphs import build_spdx_graphs +from .spdx_graph_model import SpdxIdGeneratorCollection + +__all__ = ["build_spdx_graphs", "SpdxIdGeneratorCollection"] diff --git a/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py b/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py new file mode 100644 index 000000000000..ee24e9eaf603 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/build_spdx_graphs.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from datetime import datetime +from typing import Protocol + +import logging +from sbom.config import KernelSpdxDocumentKind +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr +from sbom.spdx_graph.kernel_file import KernelFileCollection +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_source_graph import SpdxSourceGraph +from sbom.spdx_graph.spdx_build_graph import SpdxBuildGraph +from sbom.spdx_graph.spdx_output_graph import SpdxOutputGraph + + +class SpdxGraphConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + created: datetime + build_type: str + build_id: str | None + package_license: str + package_version: str | None + package_copyright_text: str | None + + +def build_spdx_graphs( + cmd_graph: CmdGraph, + spdx_id_generators: SpdxIdGeneratorCollection, + config: SpdxGraphConfig, +) -> dict[KernelSpdxDocumentKind, SpdxGraph]: + """ + Builds SPDX graphs (output, source, and build) based on a cmd dependency graph. + If the source and object trees are identical, no dedicated source graph can be created. + In that case the source files are added to the build graph instead. + + Args: + cmd_graph: The dependency graph of a kernel build. + spdx_id_generators: Collection of SPDX ID generators. + config: Configuration options. + + Returns: + Dictionary of SPDX graphs + """ + shared_elements = SharedSpdxElements.create(spdx_id_generators.base, config.created) + kernel_files = KernelFileCollection.create(cmd_graph, config.obj_tree, config.src_tree, spdx_id_generators) + output_graph = SpdxOutputGraph.create( + root_files=list(kernel_files.output.values()), + shared_elements=shared_elements, + spdx_id_generators=spdx_id_generators, + config=config, + ) + spdx_graphs: dict[KernelSpdxDocumentKind, SpdxGraph] = { + KernelSpdxDocumentKind.OUTPUT: output_graph, + } + + if len(kernel_files.source) > 0: + spdx_graphs[KernelSpdxDocumentKind.SOURCE] = SpdxSourceGraph.create( + source_files=list(kernel_files.source.values()), + external_files=list(kernel_files.external.values()), + shared_elements=shared_elements, + spdx_id_generators=spdx_id_generators, + ) + else: + logging.info( + "Skipped creating a dedicated source SBOM because source files cannot be " + "reliably classified when the source and object trees are identical. " + "Added source files to the build SBOM instead." + ) + + build_graph = SpdxBuildGraph.create( + cmd_graph, + kernel_files, + shared_elements, + output_graph.high_level_build_element, + spdx_id_generators, + ) + spdx_graphs[KernelSpdxDocumentKind.BUILD] = build_graph + + return spdx_graphs diff --git a/scripts/sbom/sbom/spdx_graph/kernel_file.py b/scripts/sbom/sbom/spdx_graph/kernel_file.py new file mode 100644 index 000000000000..505f25f66ebb --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/kernel_file.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from enum import Enum +import hashlib +import os +import re +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr, is_relative_to +from sbom.spdx import SpdxId, SpdxIdGenerator +from sbom.spdx.core import Hash +from sbom.spdx.software import ContentIdentifier, File, SoftwarePurpose +import sbom.sbom_logging as sbom_logging +from sbom.spdx_graph.spdx_graph_model import SpdxIdGeneratorCollection + + +class KernelFileLocation(Enum): + """Represents the location of a file relative to the source/object trees.""" + + SOURCE_TREE = "source_tree" + """File is located in the source tree.""" + OBJ_TREE = "obj_tree" + """File is located in the object tree.""" + EXTERNAL = "external" + """File is located outside both source and object trees.""" + BOTH = "both" + """File is located in a folder that is both source and object tree.""" + + +@dataclass +class KernelFile: + """kernel-specific metadata used to generate an SPDX File element.""" + + absolute_path: PathStr + """Absolute path of the file.""" + file_location: KernelFileLocation + """Location of the file relative to the source/object trees.""" + name: str + """Name of the file element. Should be relative to the source tree if + file_location equals SOURCE_TREE and relative to the object tree if + file_location equals OBJ_TREE. If file_location equals EXTERNAL, the + absolute path is used.""" + license_identifier: str | None + """SPDX license ID if file_location equals SOURCE_TREE or BOTH; otherwise None.""" + spdx_id_generator: SpdxIdGenerator + """Generator for the SPDX ID of the file element.""" + + _spdx_file_element: File | None = None + + @classmethod + def create( + cls, + absolute_path: PathStr, + obj_tree: PathStr, + src_tree: PathStr, + spdx_id_generators: SpdxIdGeneratorCollection, + is_output: bool, + ) -> "KernelFile": + is_in_obj_tree = is_relative_to(absolute_path, obj_tree) + is_in_src_tree = is_relative_to(absolute_path, src_tree) + + # file element name should be relative to output or src tree if possible + if not is_in_src_tree and not is_in_obj_tree: + file_element_name = str(absolute_path) + file_location = KernelFileLocation.EXTERNAL + spdx_id_generator = spdx_id_generators.source if src_tree != obj_tree else spdx_id_generators.build + elif is_in_src_tree and src_tree == obj_tree: + file_element_name = os.path.relpath(absolute_path, obj_tree) + file_location = KernelFileLocation.BOTH + spdx_id_generator = spdx_id_generators.output if is_output else spdx_id_generators.build + elif is_in_obj_tree: + file_element_name = os.path.relpath(absolute_path, obj_tree) + file_location = KernelFileLocation.OBJ_TREE + spdx_id_generator = spdx_id_generators.output if is_output else spdx_id_generators.build + else: + file_element_name = os.path.relpath(absolute_path, src_tree) + file_location = KernelFileLocation.SOURCE_TREE + spdx_id_generator = spdx_id_generators.source + + # parse spdx license identifier + license_identifier = ( + _parse_spdx_license_identifier(absolute_path) + if file_location == KernelFileLocation.SOURCE_TREE or file_location == KernelFileLocation.BOTH + else None + ) + + return KernelFile( + absolute_path, + file_location, + file_element_name, + license_identifier, + spdx_id_generator, + ) + + @property + def spdx_file_element(self) -> File: + if self._spdx_file_element is None: + self._spdx_file_element = _build_file_element( + self.absolute_path, + self.name, + self.spdx_id_generator.generate(), + self.file_location, + ) + return self._spdx_file_element + + +@dataclass +class KernelFileCollection: + """Collection of kernel files.""" + + source: dict[PathStr, KernelFile] + build: dict[PathStr, KernelFile] + output: dict[PathStr, KernelFile] + external: dict[PathStr, KernelFile] + + @classmethod + def create( + cls, + cmd_graph: CmdGraph, + obj_tree: PathStr, + src_tree: PathStr, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "KernelFileCollection": + source: dict[PathStr, KernelFile] = {} + build: dict[PathStr, KernelFile] = {} + output: dict[PathStr, KernelFile] = {} + external: dict[PathStr, KernelFile] = {} + root_node_paths = {node.absolute_path for node in cmd_graph.roots} + for node in cmd_graph: + is_root = node.absolute_path in root_node_paths + kernel_file = KernelFile.create( + node.absolute_path, + obj_tree, + src_tree, + spdx_id_generators, + is_root, + ) + if is_root: + output[kernel_file.absolute_path] = kernel_file + elif kernel_file.file_location == KernelFileLocation.SOURCE_TREE: + source[kernel_file.absolute_path] = kernel_file + elif kernel_file.file_location == KernelFileLocation.EXTERNAL: + external[kernel_file.absolute_path] = kernel_file + else: + build[kernel_file.absolute_path] = kernel_file + + return KernelFileCollection(source, build, output, external) + + def to_dict(self) -> dict[PathStr, KernelFile]: + return {**self.source, **self.build, **self.output, **self.external} + + +def _build_file_element(absolute_path: PathStr, name: str, spdx_id: SpdxId, file_location: KernelFileLocation) -> File: + verifiedUsing: list[Hash] = [] + content_identifier: list[ContentIdentifier] = [] + if os.path.isfile(absolute_path): + verifiedUsing = [Hash(algorithm="sha256", hashValue=_sha256(absolute_path))] + content_identifier = [ + ContentIdentifier( + software_contentIdentifierType="gitoid", + software_contentIdentifierValue=_git_blob_oid(absolute_path), + ) + ] + elif file_location == KernelFileLocation.EXTERNAL: + sbom_logging.warning( + "Cannot compute hash for {absolute_path} because file does not exist.", + absolute_path=absolute_path, + ) + else: + sbom_logging.error( + "Cannot compute hash for {absolute_path} because file does not exist.", + absolute_path=absolute_path, + ) + + # primary purpose + primary_purpose = _get_primary_purpose(absolute_path) + + return File( + spdxId=spdx_id, + name=name, + verifiedUsing=verifiedUsing, + software_primaryPurpose=primary_purpose, + software_contentIdentifier=content_identifier, + ) + + +def _sha256(file_path: PathStr, chunk_size: int = 1 << 20) -> str: + """Compute the SHA-256 hex digest of a file, reading it in chunks of chunk_size bytes.""" + h = hashlib.sha256() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +def _git_blob_oid(file_path: str, chunk_size: int = 1 << 20) -> str: + """Compute the Git blob object ID (SHA-1 hex) for a file, like `git hash-object`, reading it in chunks of chunk_size bytes.""" + h = hashlib.sha1() + h.update(f"blob {os.path.getsize(file_path)}\0".encode()) + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +# REUSE-IgnoreStart +SPDX_LICENSE_IDENTIFIER_PATTERN = re.compile( + r"SPDX-License-Identifier:" # literal tag + r"\s*" # optional whitespace after colon + r"(?P.*?)" # license expression (non-greedy, stops before terminator) + r"(?:\s*" # optional whitespace before terminator (not captured) + r"(-->|\*/|$))", # terminator: XML "-->", C-style "*/", or end of line + re.MULTILINE, # match end of each line, not just end of string +) +# REUSE-IgnoreEnd + + +def _parse_spdx_license_identifier(absolute_path: str, max_bytes: int = 512) -> str | None: + """ + Extracts the SPDX-License-Identifier from the beginning of a source file. + + Args: + absolute_path: Path to the source file. + max_bytes: Maximum number of bytes to scan for the license identifier. + + Returns: + The license identifier string (e.g., 'GPL-2.0-only') if found, otherwise None. + """ + try: + with open(absolute_path, "r", encoding="utf-8") as f: + match = SPDX_LICENSE_IDENTIFIER_PATTERN.search(f.read(max_bytes)) + if match: + return match.group("id") + except (UnicodeDecodeError, OSError): + return None + return None + + +def _get_primary_purpose(absolute_path: PathStr) -> SoftwarePurpose | None: + def ends_with(suffixes: list[str]) -> bool: + return any(absolute_path.endswith(suffix) for suffix in suffixes) + + def includes_path_segments(path_segments: list[str]) -> bool: + return any(segment in absolute_path for segment in path_segments) + + # Source code + if ends_with([".c", ".h", ".S", ".s", ".rs", ".pl", "gen_smb1_mapping", "gen_smb2_mapping"]): + return "source" + + # Libraries + if ends_with([".a", ".so", ".so.raw", ".rlib"]): + return "library" + + # Archives + if ends_with([".xz", ".cpio", ".gz", ".tar", ".zip", "piggy_data"]): + return "archive" + + # Applications + if ends_with(["bzImage", "Image", ".efi"]): + return "application" + + # Executables / machine code + if ends_with([".bin", ".elf", "vmlinux", "vmlinux.unstripped", "vmlinuz", "bpfilter_umh"]): + return "executable" + + # Kernel modules + if ends_with([".ko"]): + return "module" + + # Data files + if ends_with( + [ + ".tbl", + ".relocs", + ".rmeta", + ".in", + ".dbg", + ".x509", + ".pbm", + ".ppm", + ".dtb", + ".uc", + ".inc", + ".dts", + ".dtsi", + ".dtbo", + ".xml", + ".ro", + "initramfs_inc_data", + "default_cpio_list", + "x509_certificate_list", + "utf8data.c_shipped", + "blacklist_hash_list", + "x509_revocation_list", + "cpucaps", + "sysreg", + "mach-types", + ] + ) or includes_path_segments(["drivers/gpu/drm/radeon/reg_srcs/"]): + return "data" + + # Configuration files + if ends_with([".pem", ".key", ".conf", ".config", ".cfg", ".bconf"]): + return "configuration" + + # Documentation + if ends_with([".md"]): + return "documentation" + + # Other / miscellaneous + if ends_with([".o", ".tmp"]): + return "other" + + sbom_logging.warning("Could not infer primary purpose for {absolute_path}", absolute_path=absolute_path) diff --git a/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py b/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py new file mode 100644 index 000000000000..115e8778a467 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/shared_spdx_elements.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from datetime import datetime, timezone +from sbom.spdx.core import CreationInfo, SoftwareAgent +from sbom.spdx.spdxId import SpdxIdGenerator + + +@dataclass(frozen=True) +class SharedSpdxElements: + agent: SoftwareAgent + creation_info: CreationInfo + + @classmethod + def create(cls, spdx_id_generator: SpdxIdGenerator, created: datetime) -> "SharedSpdxElements": + """ + Creates shared SPDX elements used across multiple documents. + + Args: + spdx_id_generator: Generator for creating SPDX IDs. + created: SPDX 'created' property used for the creation info. + + Returns: + SharedSpdxElements with agent and creation info. + """ + agent = SoftwareAgent( + spdxId=spdx_id_generator.generate(), + name="KernelSbom", + ) + creation_info = CreationInfo(createdBy=[agent], created=created.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")) + return SharedSpdxElements(agent=agent, creation_info=creation_info) diff --git a/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py new file mode 100644 index 000000000000..4d738bc3b3e2 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_build_graph.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from typing import Mapping +from sbom.cmd_graph import CmdGraph +from sbom.path_utils import PathStr +from sbom.spdx import SpdxIdGenerator +from sbom.spdx.build import Build +from sbom.spdx.core import ExternalMap, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.software import File, Sbom +from sbom.spdx_graph.kernel_file import KernelFileCollection +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection +from sbom.spdx_graph.spdx_source_graph import source_file_license_elements + + +@dataclass +class SpdxBuildGraph(SpdxGraph): + """SPDX graph representing build dependencies connecting source files and + distributable output files""" + + @classmethod + def create( + cls, + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "SpdxBuildGraph": + if len(kernel_files.source) > 0: + return _create_spdx_build_graph( + cmd_graph, + kernel_files, + shared_elements, + high_level_build_element, + spdx_id_generators, + ) + else: + return _create_spdx_build_graph_with_mixed_sources( + cmd_graph, + kernel_files, + shared_elements, + high_level_build_element, + spdx_id_generators, + ) + + +def _create_spdx_build_graph( + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, +) -> SpdxBuildGraph: + """ + Creates an SPDX build graph where source and output files are referenced + from external documents. + + Args: + cmd_graph: The dependency graph of a kernel build. + kernel_files: Collection of categorized kernel files involved in the build. + shared_elements: SPDX elements shared across multiple documents. + high_level_build_element: The high-level Build element referenced by the build graph. + spdx_id_generators: Collection of generators for SPDX element IDs. + + Returns: + SpdxBuildGraph: The SPDX build graph connecting source files and distributable output files. + """ + # SpdxDocument + build_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.build.generate(), + profileConformance=["core", "software", "build"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [ + spdx_id_generators.build, + spdx_id_generators.source, + spdx_id_generators.output, + spdx_id_generators.base, + ] + if generator.prefix is not None + ], + ) + + # Sbom + build_sbom = Sbom( + spdxId=spdx_id_generators.build.generate(), + software_sbomType=["build"], + ) + + # Src and object tree elements + obj_tree_element = File( + spdxId=spdx_id_generators.build.generate(), + name="$(obj_tree)", + software_fileKind="directory", + ) + obj_tree_contains_relationship = Relationship( + spdxId=spdx_id_generators.build.generate(), + relationshipType="contains", + from_=obj_tree_element, + to=[], + ) + + # File elements + build_file_elements = [file.spdx_file_element for file in kernel_files.build.values()] + file_relationships = _file_relationships( + cmd_graph=cmd_graph, + file_elements={key: file.spdx_file_element for key, file in kernel_files.to_dict().items()}, + high_level_build_element=high_level_build_element, + spdx_id_generator=spdx_id_generators.build, + ) + + # Update relationships + build_spdx_document.rootElement = [build_sbom] + + build_spdx_document.import_ = [ + *( + ExternalMap(externalSpdxId=file.spdx_file_element.spdxId) + for file in (*kernel_files.source.values(), *kernel_files.external.values()) + ), + ExternalMap(externalSpdxId=high_level_build_element.spdxId), + *(ExternalMap(externalSpdxId=file.spdx_file_element.spdxId) for file in kernel_files.output.values()), + ] + + build_sbom.rootElement = [obj_tree_element] + build_sbom.element = [ + obj_tree_element, + obj_tree_contains_relationship, + *build_file_elements, + *file_relationships, + ] + + obj_tree_contains_relationship.to = [ + *build_file_elements, + *(file.spdx_file_element for file in kernel_files.output.values()), + ] + + # create Spdx graphs + build_graph = SpdxBuildGraph( + build_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + build_sbom, + ) + return build_graph + + +def _create_spdx_build_graph_with_mixed_sources( + cmd_graph: CmdGraph, + kernel_files: KernelFileCollection, + shared_elements: SharedSpdxElements, + high_level_build_element: Build, + spdx_id_generators: SpdxIdGeneratorCollection, +) -> SpdxBuildGraph: + """ + Creates an SPDX build graph where only output files are referenced from + an external document. Source files are included directly in the build graph. + + Args: + cmd_graph: The dependency graph of a kernel build. + kernel_files: Collection of categorized kernel files involved in the build. + shared_elements: SPDX elements shared across multiple documents. + high_level_build_element: The high-level Build element referenced by the build graph. + spdx_id_generators: Collection of generators for SPDX element IDs. + + Returns: + SpdxBuildGraph: The SPDX build graph connecting source files and distributable output files. + """ + # SpdxDocument + build_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.build.generate(), + profileConformance=["core", "software", "build"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [ + spdx_id_generators.build, + spdx_id_generators.output, + spdx_id_generators.base, + ] + if generator.prefix is not None + ], + ) + + # Sbom + build_sbom = Sbom( + spdxId=spdx_id_generators.build.generate(), + software_sbomType=["build"], + ) + + # File elements + build_file_elements = [file.spdx_file_element for file in kernel_files.build.values()] + external_file_elements = [file.spdx_file_element for file in kernel_files.external.values()] + file_relationships = _file_relationships( + cmd_graph=cmd_graph, + file_elements={key: file.spdx_file_element for key, file in kernel_files.to_dict().items()}, + high_level_build_element=high_level_build_element, + spdx_id_generator=spdx_id_generators.build, + ) + + # Source file license elements + source_file_license_identifiers, source_file_license_relationships = source_file_license_elements( + list(kernel_files.build.values()), spdx_id_generators.build + ) + + # Update relationships + build_spdx_document.rootElement = [build_sbom] + root_file_elements = [file.spdx_file_element for file in kernel_files.output.values()] + build_spdx_document.import_ = [ + ExternalMap(externalSpdxId=high_level_build_element.spdxId), + *(ExternalMap(externalSpdxId=file.spdxId) for file in root_file_elements), + ] + + build_sbom.rootElement = [*root_file_elements] + build_sbom.element = [ + *build_file_elements, + *external_file_elements, + *source_file_license_identifiers, + *source_file_license_relationships, + *file_relationships, + ] + + build_graph = SpdxBuildGraph( + build_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + build_sbom, + ) + return build_graph + + +def _file_relationships( + cmd_graph: CmdGraph, + file_elements: Mapping[PathStr, File], + high_level_build_element: Build, + spdx_id_generator: SpdxIdGenerator, +) -> list[Build | Relationship]: + """ + Construct SPDX Build and Relationship elements representing dependency + relationships in the cmd graph. + + Args: + cmd_graph: The dependency graph of a kernel build. + file_elements: Mapping of filesystem paths (PathStr) to their + corresponding SPDX File elements. + high_level_build_element: The SPDX Build element representing the overall build process/root. + spdx_id_generator: Generator for unique SPDX IDs. + + Returns: + list[Build | Relationship]: List of SPDX Build and Relationship elements + """ + high_level_build_ancestorOf_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="ancestorOf", + from_=high_level_build_element, + completeness="complete", + to=[], + ) + + # Create a relationship between each node (output file) + # and its children (input files) + build_and_relationship_elements: list[Build | Relationship] = [high_level_build_ancestorOf_relationship] + for node in cmd_graph: + # .cmd file dependencies + if node.cmd_file is not None: + build_element = Build( + spdxId=spdx_id_generator.generate(), + build_buildType=high_level_build_element.build_buildType, + build_buildId=high_level_build_element.build_buildId, + comment=node.cmd_file.savedcmd, + ) + build_and_relationship_elements.append(build_element) + + if node.cmd_file_dependencies: + hasInput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasInput", + from_=build_element, + to=[file_elements[dep.absolute_path] for dep in node.cmd_file_dependencies], + ) + build_and_relationship_elements.append(hasInput_relationship) + + hasOutput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasOutput", + from_=build_element, + to=[file_elements[node.absolute_path]], + ) + build_and_relationship_elements.append(hasOutput_relationship) + + high_level_build_ancestorOf_relationship.to.append(build_element) + + # incbin dependencies + if len(node.incbin_dependencies) > 0: + incbin_dependsOn_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="dependsOn", + comment="\n".join([incbin_dependency.full_statement for incbin_dependency in node.incbin_dependencies]), + from_=file_elements[node.absolute_path], + to=[ + file_elements[incbin_dependency.node.absolute_path] + for incbin_dependency in node.incbin_dependencies + ], + ) + build_and_relationship_elements.append(incbin_dependsOn_relationship) + + # hardcoded dependencies + if len(node.hardcoded_dependencies) > 0: + hardcoded_dependency_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="dependsOn", + from_=file_elements[node.absolute_path], + to=[file_elements[n.absolute_path] for n in node.hardcoded_dependencies], + ) + build_and_relationship_elements.append(hardcoded_dependency_relationship) + + return build_and_relationship_elements diff --git a/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py b/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py new file mode 100644 index 000000000000..682194d4362a --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_graph_model.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from sbom.spdx.core import CreationInfo, SoftwareAgent, SpdxDocument, SpdxObject +from sbom.spdx.software import Sbom +from sbom.spdx.spdxId import SpdxIdGenerator + + +@dataclass +class SpdxGraph: + """Represents the complete graph of a single SPDX document.""" + + spdx_document: SpdxDocument + agent: SoftwareAgent + creation_info: CreationInfo + sbom: Sbom + + def to_list(self) -> list[SpdxObject]: + return [ + self.spdx_document, + self.agent, + self.creation_info, + self.sbom, + *self.sbom.element, + ] + + +@dataclass +class SpdxIdGeneratorCollection: + """Holds SPDX ID generators for different document types to ensure globally unique SPDX IDs.""" + + base: SpdxIdGenerator + source: SpdxIdGenerator + build: SpdxIdGenerator + output: SpdxIdGenerator diff --git a/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py new file mode 100644 index 000000000000..ff9b2c31fb04 --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_output_graph.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +import os +from typing import Protocol +from sbom.environment import Environment +from sbom.path_utils import PathStr +from sbom.spdx.build import Build +from sbom.spdx.core import DictionaryEntry, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.simplelicensing import LicenseExpression +from sbom.spdx.software import File, Package, Sbom +from sbom.spdx.spdxId import SpdxIdGenerator +from sbom.spdx_graph.kernel_file import KernelFile +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection + + +class SpdxOutputGraphConfig(Protocol): + obj_tree: PathStr + src_tree: PathStr + build_type: str + build_id: str | None + package_license: str + package_version: str | None + package_copyright_text: str | None + + +@dataclass +class SpdxOutputGraph(SpdxGraph): + """SPDX graph representing distributable output files""" + + high_level_build_element: Build + + @classmethod + def create( + cls, + root_files: list[KernelFile], + shared_elements: SharedSpdxElements, + spdx_id_generators: SpdxIdGeneratorCollection, + config: SpdxOutputGraphConfig, + ) -> "SpdxOutputGraph": + """ + Args: + root_files: List of distributable output files which act as roots + of the dependency graph. + shared_elements: Shared SPDX elements used across multiple documents. + spdx_id_generators: Collection of SPDX ID generators. + config: Configuration options. + + Returns: + SpdxOutputGraph: The SPDX output graph. + """ + # SpdxDocument + spdx_document = SpdxDocument( + spdxId=spdx_id_generators.output.generate(), + profileConformance=["core", "software", "build", "simpleLicensing"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [spdx_id_generators.output, spdx_id_generators.base] + if generator.prefix is not None + ], + ) + + # Sbom + sbom = Sbom( + spdxId=spdx_id_generators.output.generate(), + software_sbomType=["build"], + ) + + # High-level Build elements + config_source_element = KernelFile.create( + absolute_path=os.path.join(config.obj_tree, ".config"), + obj_tree=config.obj_tree, + src_tree=config.src_tree, + spdx_id_generators=spdx_id_generators, + is_output=True, + ).spdx_file_element + high_level_build_element, high_level_build_element_hasOutput_relationship = _high_level_build_elements( + config.build_type, + config.build_id, + config_source_element, + spdx_id_generators.output, + ) + + # Root file elements + root_file_elements: list[File] = [file.spdx_file_element for file in root_files] + + # Package elements + package_elements = [ + Package( + spdxId=spdx_id_generators.output.generate(), + name=_get_package_name(file.name), + software_packageVersion=config.package_version, + software_copyrightText=config.package_copyright_text, + comment=f"Architecture={arch}" if (arch := Environment.ARCH() or Environment.SRCARCH()) else None, + software_primaryPurpose=file.software_primaryPurpose, + ) + for file in root_file_elements + ] + package_hasDistributionArtifact_file_relationships = [ + Relationship( + spdxId=spdx_id_generators.output.generate(), + relationshipType="hasDistributionArtifact", + from_=package, + to=[file], + ) + for package, file in zip(package_elements, root_file_elements) + ] + package_license_expression = LicenseExpression( + spdxId=spdx_id_generators.output.generate(), + simplelicensing_licenseExpression=config.package_license, + ) + package_hasDeclaredLicense_relationships = [ + Relationship( + spdxId=spdx_id_generators.output.generate(), + relationshipType="hasDeclaredLicense", + from_=package, + to=[package_license_expression], + ) + for package in package_elements + ] + + # Update relationships + spdx_document.rootElement = [sbom] + + sbom.rootElement = [*package_elements] + sbom.element = [ + config_source_element, + high_level_build_element, + high_level_build_element_hasOutput_relationship, + *root_file_elements, + *package_elements, + *package_hasDistributionArtifact_file_relationships, + package_license_expression, + *package_hasDeclaredLicense_relationships, + ] + + high_level_build_element_hasOutput_relationship.to = [*root_file_elements] + + output_graph = SpdxOutputGraph( + spdx_document, + shared_elements.agent, + shared_elements.creation_info, + sbom, + high_level_build_element, + ) + return output_graph + + +def _get_package_name(filename: str) -> str: + """ + Generates a SPDX package name from a filename. + Kernel images (bzImage, Image) get a descriptive name, others use the basename of the file. + """ + KERNEL_FILENAMES = ["bzImage", "Image"] + basename = os.path.basename(filename) + return f"Linux Kernel ({basename})" if basename in KERNEL_FILENAMES else basename + + +def _high_level_build_elements( + build_type: str, + build_id: str | None, + config_source_element: File, + spdx_id_generator: SpdxIdGenerator, +) -> tuple[Build, Relationship]: + build_spdxId = spdx_id_generator.generate() + high_level_build_element = Build( + spdxId=build_spdxId, + build_buildType=build_type, + build_buildId=build_id if build_id is not None else build_spdxId, + build_environment=[ + DictionaryEntry(key=key, value=value) + for key, value in Environment.KERNEL_BUILD_VARIABLES().items() + if value + ], + build_configSourceUri=[config_source_element.spdxId], + build_configSourceDigest=config_source_element.verifiedUsing, + ) + + high_level_build_element_hasOutput_relationship = Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasOutput", + from_=high_level_build_element, + to=[], + ) + return high_level_build_element, high_level_build_element_hasOutput_relationship diff --git a/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py b/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py new file mode 100644 index 000000000000..90880212dedd --- /dev/null +++ b/scripts/sbom/sbom/spdx_graph/spdx_source_graph.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from dataclasses import dataclass +from sbom.spdx import SpdxIdGenerator +from sbom.spdx.core import Element, NamespaceMap, Relationship, SpdxDocument +from sbom.spdx.simplelicensing import LicenseExpression +from sbom.spdx.software import File, Sbom +from sbom.spdx_graph.kernel_file import KernelFile +from sbom.spdx_graph.shared_spdx_elements import SharedSpdxElements +from sbom.spdx_graph.spdx_graph_model import SpdxGraph, SpdxIdGeneratorCollection + + +@dataclass +class SpdxSourceGraph(SpdxGraph): + """SPDX graph representing source files""" + + @classmethod + def create( + cls, + source_files: list[KernelFile], + external_files: list[KernelFile], + shared_elements: SharedSpdxElements, + spdx_id_generators: SpdxIdGeneratorCollection, + ) -> "SpdxSourceGraph": + """ + Args: + source_files: List of files within the kernel source tree. + external_files: Files outside both source and object trees. + shared_elements: Shared SPDX elements used across multiple documents. + spdx_id_generators: Collection of SPDX ID generators. + + Returns: + SpdxSourceGraph: The SPDX source graph. + """ + # SpdxDocument + source_spdx_document = SpdxDocument( + spdxId=spdx_id_generators.source.generate(), + profileConformance=["core", "software", "simpleLicensing"], + namespaceMap=[ + NamespaceMap(prefix=generator.prefix, namespace=generator.namespace) + for generator in [spdx_id_generators.source, spdx_id_generators.base] + if generator.prefix is not None + ], + ) + + # Sbom + source_sbom = Sbom( + spdxId=spdx_id_generators.source.generate(), + software_sbomType=["source"], + ) + + # Src Tree Elements + src_tree_element = File( + spdxId=spdx_id_generators.source.generate(), + name="$(src_tree)", + software_fileKind="directory", + ) + src_tree_contains_relationship = Relationship( + spdxId=spdx_id_generators.source.generate(), + relationshipType="contains", + from_=src_tree_element, + to=[], + ) + + # Source file elements + source_file_elements: list[Element] = [file.spdx_file_element for file in source_files] + external_file_elements: list[Element] = [file.spdx_file_element for file in external_files] + + # Source file license elements + source_file_license_identifiers, source_file_license_relationships = source_file_license_elements( + source_files, spdx_id_generators.source + ) + + # Update relationships + source_spdx_document.rootElement = [source_sbom] + source_sbom.rootElement = [src_tree_element] + source_sbom.element = [ + src_tree_element, + src_tree_contains_relationship, + *source_file_elements, + *external_file_elements, + *source_file_license_identifiers, + *source_file_license_relationships, + ] + src_tree_contains_relationship.to = source_file_elements + + source_graph = SpdxSourceGraph( + source_spdx_document, + shared_elements.agent, + shared_elements.creation_info, + source_sbom, + ) + return source_graph + + +def source_file_license_elements( + source_files: list[KernelFile], spdx_id_generator: SpdxIdGenerator +) -> tuple[list[LicenseExpression], list[Relationship]]: + """ + Creates SPDX license expressions and links them to the given source files + via hasDeclaredLicense relationships. + + Args: + source_files: List of files within the kernel source tree. + spdx_id_generator: Generator for unique SPDX IDs. + + Returns: + Tuple of (license expressions, hasDeclaredLicense relationships). + """ + license_expressions: dict[str, LicenseExpression] = {} + for file in source_files: + if file.license_identifier is None or file.license_identifier in license_expressions: + continue + license_expressions[file.license_identifier] = LicenseExpression( + spdxId=spdx_id_generator.generate(), + simplelicensing_licenseExpression=file.license_identifier, + ) + + source_file_license_relationships = [ + Relationship( + spdxId=spdx_id_generator.generate(), + relationshipType="hasDeclaredLicense", + from_=file.spdx_file_element, + to=[license_expressions[file.license_identifier]], + ) + for file in source_files + if file.license_identifier is not None + ] + return ([*license_expressions.values()], source_file_license_relationships) diff --git a/scripts/sbom/tests/__init__.py b/scripts/sbom/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/sbom/tests/cmd_graph/__init__.py b/scripts/sbom/tests/cmd_graph/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py new file mode 100644 index 000000000000..a061a748e1bf --- /dev/null +++ b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py @@ -0,0 +1,443 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import os +import unittest +from unittest.mock import patch + +from sbom.cmd_graph.savedcmd_parser import parse_inputs_from_commands +from sbom.cmd_graph.savedcmd_parser.command_parser_registry import CommandParserRegistry +import sbom.sbom_logging as sbom_logging + + +class TestSavedCmdParser(unittest.TestCase): + def _assert_parsing(self, cmd: str, expected: str, registry: CommandParserRegistry | None = None) -> None: + sbom_logging.init() + parsed = parse_inputs_from_commands(cmd, fail_on_unknown_build_command=False, registry=registry) + target = [] if expected == "" else expected.split(" ") + self.assertEqual(parsed, target) + errors = sbom_logging._error_logger._message_counts # type: ignore + self.assertEqual(errors, {}) + + # Compound command tests + def test_dd_cat(self): + cmd = "(dd if=arch/x86/boot/setup.bin bs=4k conv=sync status=none; cat arch/x86/boot/vmlinux.bin) >arch/x86/boot/bzImage" + expected = "arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin" + self._assert_parsing(cmd, expected) + + def test_manual_file_creation(self): + cmd = """{ symbase=__dtbo_overlay_bad_unresolved; echo '$(pound)include '; echo '.section .rodata,"a"'; echo '.balign STRUCT_ALIGNMENT'; echo ".global $${symbase}_begin"; echo "$${symbase}_begin:"; echo '.incbin "drivers/of/unittest-data/overlay_bad_unresolved.dtbo" '; echo ".global $${symbase}_end"; echo "$${symbase}_end:"; echo '.balign STRUCT_ALIGNMENT'; } > drivers/of/unittest-data/overlay_bad_unresolved.dtbo.S""" + expected = "" + self._assert_parsing(cmd, expected) + + def test_cat_xz_wrap(self): + cmd = "{ cat arch/x86/boot/compressed/vmlinux.bin | sh ../scripts/xz_wrap.sh; printf \\130\\064\\024\\000; } > arch/x86/boot/compressed/vmlinux.bin.xz" + expected = "arch/x86/boot/compressed/vmlinux.bin" + self._assert_parsing(cmd, expected) + + def test_printf_sed(self): + cmd = r"""{ printf 'static char tomoyo_builtin_profile[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_exception_policy[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- ../security/tomoyo/policy/exception_policy.conf.default; printf '\t"";\n'; printf 'static char tomoyo_builtin_domain_policy[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_manager[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; printf 'static char tomoyo_builtin_stat[] __initdata =\n'; sed -e 's/\\/\\\\/g' -e 's/\"/\\"/g' -e 's/\(.*\)/\t"\1\\n"/' -- /dev/null; printf '\t"";\n'; } > security/tomoyo/builtin-policy.h""" + expected = "../security/tomoyo/policy/exception_policy.conf.default" + self._assert_parsing(cmd, expected) + + def test_bin2c_echo(self): + cmd = """(echo "static char tomoyo_builtin_profile[] __initdata ="; ./scripts/bin2c security/tomoyo/builtin-policy.h""" + expected = "../security/tomoyo/policy/exception_policy.conf.default" + self._assert_parsing(cmd, expected) + + def test_cat_colon(self): + cmd = "{ cat init/modules.order; cat usr/modules.order; cat arch/x86/modules.order; cat arch/x86/boot/startup/modules.order; cat kernel/modules.order; cat certs/modules.order; cat mm/modules.order; cat fs/modules.order; cat ipc/modules.order; cat security/modules.order; cat crypto/modules.order; cat block/modules.order; cat io_uring/modules.order; cat lib/modules.order; cat arch/x86/lib/modules.order; cat drivers/modules.order; cat sound/modules.order; cat samples/modules.order; cat net/modules.order; cat virt/modules.order; cat arch/x86/pci/modules.order; cat arch/x86/power/modules.order; cat arch/x86/video/modules.order; :; } > modules.order" + expected = "init/modules.order usr/modules.order arch/x86/modules.order arch/x86/boot/startup/modules.order kernel/modules.order certs/modules.order mm/modules.order fs/modules.order ipc/modules.order security/modules.order crypto/modules.order block/modules.order io_uring/modules.order lib/modules.order arch/x86/lib/modules.order drivers/modules.order sound/modules.order samples/modules.order net/modules.order virt/modules.order arch/x86/pci/modules.order arch/x86/power/modules.order arch/x86/video/modules.order" + self._assert_parsing(cmd, expected) + + def test_cat_zstd(self): + cmd = "{ cat arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs | zstd -22 --ultra; printf \\340\\362\\066\\003; } > arch/x86/boot/compressed/vmlinux.bin.zst" + expected = "arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs" + self._assert_parsing(cmd, expected) + + # cat command tests + def test_cat_redirect(self): + cmd = "cat ../fs/unicode/utf8data.c_shipped > fs/unicode/utf8data.c" + expected = "../fs/unicode/utf8data.c_shipped" + self._assert_parsing(cmd, expected) + + def test_cat_piped(self): + cmd = "cat arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs | gzip -n -f -9 > arch/x86/boot/compressed/vmlinux.bin.gz" + expected = "arch/x86/boot/compressed/vmlinux.bin arch/x86/boot/compressed/vmlinux.relocs" + self._assert_parsing(cmd, expected) + + # sed command tests + def test_sed(self): + cmd = "sed -n 's/.*define *BLIST_\\([A-Z0-9_]*\\) *.*/BLIST_FLAG_NAME(\\1),/p' ../include/scsi/scsi_devinfo.h > drivers/scsi/scsi_devinfo_tbl.c" + expected = "../include/scsi/scsi_devinfo.h" + self._assert_parsing(cmd, expected) + + # awk command tests + def test_awk(self): + cmd = "awk -f ../arch/arm64/tools/gen-cpucaps.awk ../arch/arm64/tools/cpucaps > arch/arm64/include/generated/asm/cpucap-defs.h" + expected = "../arch/arm64/tools/cpucaps" + self._assert_parsing(cmd, expected) + + def test_awk_with_input_redirection(self): + cmd = "awk -v N=1 -f ../lib/raid6/unroll.awk < ../lib/raid6/int.uc > lib/raid6/int1.c" + expected = "../lib/raid6/int.uc" + self._assert_parsing(cmd, expected) + + # openssl command tests + def test_openssl(self): + cmd = "openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 -config certs/x509.genkey -outform PEM -out certs/signing_key.pem -keyout certs/signing_key.pem 2>&1" + expected = "" + self._assert_parsing(cmd, expected) + + # gcc/clang command tests + def test_gcc(self): + cmd = ( + "gcc -Wp,-MMD,arch/x86/pci/.i386.o.d -nostdinc -I../arch/x86/include -I./arch/x86/include/generated -I../include -I./include -I../arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I../include/uapi -I./include/generated/uapi -include ../include/linux/compiler-version.h -include ../include/linux/kconfig.h -include ../include/linux/compiler_types.h -D__KERNEL__ -fmacro-prefix-map=../= -Werror -std=gnu11 -fshort-wchar -funsigned-char -fno-common -fno-PIE -fno-strict-aliasing -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx -fcf-protection=branch -fno-jump-tables -m64 -falign-jumps=1 -falign-loops=1 -mno-80387 -mno-fp-ret-in-387 -mpreferred-stack-boundary=3 -mskip-rax-setup -march=x86-64 -mtune=generic -mno-red-zone -mcmodel=kernel -mstack-protector-guard-reg=gs -mstack-protector-guard-symbol=__ref_stack_chk_guard -Wno-sign-compare -fno-asynchronous-unwind-tables -mindirect-branch=thunk-extern -mindirect-branch-register -mindirect-branch-cs-prefix -mfunction-return=thunk-extern -fno-jump-tables -fpatchable-function-entry=16,16 -fno-delete-null-pointer-checks -O2 -fno-allow-store-data-races -fstack-protector-strong -fomit-frame-pointer -fno-stack-clash-protection -falign-functions=16 -fno-strict-overflow -fno-stack-check -fconserve-stack -fno-builtin-wcslen -Wall -Wextra -Wundef -Werror=implicit-function-declaration -Werror=implicit-int -Werror=return-type -Werror=strict-prototypes -Wno-format-security -Wno-trigraphs -Wno-frame-address -Wno-address-of-packed-member -Wmissing-declarations -Wmissing-prototypes -Wframe-larger-than=2048 -Wno-main -Wvla-larger-than=1 -Wno-pointer-sign -Wcast-function-type -Wno-array-bounds -Wno-stringop-overflow -Wno-alloc-size-larger-than -Wimplicit-fallthrough=5 -Werror=date-time -Werror=incompatible-pointer-types -Werror=designated-init -Wenum-conversion -Wunused -Wno-unused-but-set-variable -Wno-unused-const-variable -Wno-packed-not-aligned -Wno-format-overflow -Wno-format-truncation -Wno-stringop-truncation -Wno-override-init -Wno-missing-field-initializers -Wno-type-limits -Wno-shift-negative-value -Wno-maybe-uninitialized -Wno-sign-compare -Wno-unused-parameter -I../arch/x86/pci -Iarch/x86/pci -DKBUILD_MODFILE=" + "arch/x86/pci/i386" + " -DKBUILD_BASENAME=" + "i386" + " -DKBUILD_MODNAME=" + "i386" + " -D__KBUILD_MODNAME=kmod_i386 -c -o arch/x86/pci/i386.o ../arch/x86/pci/i386.c " + ) + expected = "../arch/x86/pci/i386.c" + self._assert_parsing(cmd, expected) + + def test_gcc_linking(self): + cmd = "gcc -o arch/x86/tools/relocs arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + expected = "arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + self._assert_parsing(cmd, expected) + + def test_gcc_without_compile_flag(self): + cmd = "gcc -Wp,-MMD,arch/x86/boot/compressed/.mkpiggy.d -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -I ../scripts/include -I../tools/include -I arch/x86/boot/compressed -o arch/x86/boot/compressed/mkpiggy ../arch/x86/boot/compressed/mkpiggy.c" + expected = "../arch/x86/boot/compressed/mkpiggy.c" + self._assert_parsing(cmd, expected) + + def test_gcc_with_env_override(self): + with patch.dict(os.environ, {"CC": "ccache gcc"}): + registry = CommandParserRegistry.create() + cmd = "gcc -o arch/x86/tools/relocs arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + expected = "arch/x86/tools/relocs_32.o arch/x86/tools/relocs_64.o arch/x86/tools/relocs_common.o" + self._assert_parsing(cmd, expected, registry) + self._assert_parsing(f"ccache {cmd}", expected, registry) + + def test_gcc_dts_preprocessing(self): + cmd = "gcc -E -Wp,-MMD,drivers/of/.empty_root.dtb.d.pre.tmp -nostdinc -I ../scripts/dtc/include-prefixes -undef -D__DTS__ -x assembler-with-cpp -o drivers/of/.empty_root.dtb.dts.tmp ../drivers/of/empty_root.dts" + expected = "../drivers/of/empty_root.dts" + self._assert_parsing(cmd, expected) + + def test_clang(self): + cmd = """clang -Wp,-MMD,arch/x86/entry/.entry_64_compat.o.d -nostdinc -I../arch/x86/include -I./arch/x86/include/generated -I../include -I./include -I../arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I../include/uapi -I./include/generated/uapi -include ../include/linux/compiler-version.h -include ../include/linux/kconfig.h -D__KERNEL__ --target=x86_64-linux-gnu -fintegrated-as -Werror=unknown-warning-option -Werror=ignored-optimization-argument -Werror=option-ignored -Werror=unused-command-line-argument -fmacro-prefix-map=../= -Werror -D__ASSEMBLY__ -fno-PIE -m64 -I../arch/x86/entry -Iarch/x86/entry -DKBUILD_MODFILE='"arch/x86/entry/entry_64_compat"' -DKBUILD_MODNAME='"entry_64_compat"' -D__KBUILD_MODNAME=kmod_entry_64_compat -c -o arch/x86/entry/entry_64_compat.o ../arch/x86/entry/entry_64_compat.S""" + expected = "../arch/x86/entry/entry_64_compat.S" + self._assert_parsing(cmd, expected) + + # ld command tests + def test_ld(self): + cmd = r'ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --no-undefined --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o; if readelf -rW arch/x86/entry/vdso/vdso64.so.dbg | grep -v _NONE | grep -q " R_\w*_"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi' + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o" + self._assert_parsing(cmd, expected) + + def test_ld_with_env_override(self): + with patch.dict(os.environ, {"LD": "some-tool ld"}): + registry = CommandParserRegistry.create() + cmd = r'ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --no-undefined --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o; if readelf -rW arch/x86/entry/vdso/vdso64.so.dbg | grep -v _NONE | grep -q " R_\w*_"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi' + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vgetrandom.o arch/x86/entry/vdso/vgetrandom-chacha.o" + self._assert_parsing(cmd, expected, registry) + self._assert_parsing(f"some-tool {cmd}", expected, registry) + + def test_ld_whole_archive(self): + cmd = "ld -m elf_x86_64 -z noexecstack -r -o vmlinux.o --whole-archive vmlinux.a --no-whole-archive --start-group --end-group" + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + def test_ld_with_at_symbol(self): + cmd = "ld.lld -m elf_x86_64 -z noexecstack -r -o fs/efivarfs/efivarfs.o @fs/efivarfs/efivarfs.mod ; ./tools/objtool/objtool --hacks=jump_label --hacks=noinstr --hacks=skylake --ibt --orc --retpoline --rethunk --static-call --uaccess --prefix=16 --link --module fs/efivarfs/efivarfs.o" + expected = "@fs/efivarfs/efivarfs.mod" + self._assert_parsing(cmd, expected) + + def test_ld_if_objdump(self): + cmd = """ld -o arch/x86/entry/vdso/vdso64.so.dbg -shared --hash-style=both --build-id=sha1 --eh-frame-hdr -Bsymbolic -z noexecstack -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined -z max-page-size=4096 -T arch/x86/entry/vdso/vdso.lds arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vsgx.o && sh ./arch/x86/entry/vdso/checkundef.sh 'nm' 'arch/x86/entry/vdso/vdso64.so.dbg'; if objdump -R arch/x86/entry/vdso/vdso64.so.dbg | grep -E -h "R_X86_64_JUMP_SLOT|R_X86_64_GLOB_DAT|R_X86_64_RELATIVE| R_386_GLOB_DAT|R_386_JMP_SLOT|R_386_RELATIVE"; then (echo >&2 "arch/x86/entry/vdso/vdso64.so.dbg: dynamic relocations are not supported"; rm -f arch/x86/entry/vdso/vdso64.so.dbg; /bin/false); fi""" + expected = "arch/x86/entry/vdso/vdso-note.o arch/x86/entry/vdso/vclock_gettime.o arch/x86/entry/vdso/vgetcpu.o arch/x86/entry/vdso/vsgx.o" + self._assert_parsing(cmd, expected) + + # printf | xargs ar command tests + def test_ar_printf(self): + cmd = 'rm -f built-in.a; printf "./%s " init/built-in.a usr/built-in.a arch/x86/built-in.a arch/x86/boot/startup/built-in.a kernel/built-in.a certs/built-in.a mm/built-in.a fs/built-in.a ipc/built-in.a security/built-in.a crypto/built-in.a block/built-in.a io_uring/built-in.a lib/built-in.a arch/x86/lib/built-in.a drivers/built-in.a sound/built-in.a net/built-in.a virt/built-in.a arch/x86/pci/built-in.a arch/x86/power/built-in.a arch/x86/video/built-in.a | xargs ar cDPrST built-in.a' + expected = "./init/built-in.a ./usr/built-in.a ./arch/x86/built-in.a ./arch/x86/boot/startup/built-in.a ./kernel/built-in.a ./certs/built-in.a ./mm/built-in.a ./fs/built-in.a ./ipc/built-in.a ./security/built-in.a ./crypto/built-in.a ./block/built-in.a ./io_uring/built-in.a ./lib/built-in.a ./arch/x86/lib/built-in.a ./drivers/built-in.a ./sound/built-in.a ./net/built-in.a ./virt/built-in.a ./arch/x86/pci/built-in.a ./arch/x86/power/built-in.a ./arch/x86/video/built-in.a" + self._assert_parsing(cmd, expected) + + def test_ar_printf_nested(self): + cmd = 'rm -f arch/x86/pci/built-in.a; printf "arch/x86/pci/%s " i386.o init.o mmconfig_64.o direct.o mmconfig-shared.o fixup.o acpi.o legacy.o irq.o common.o early.o bus_numa.o amd_bus.o | xargs ar cDPrST arch/x86/pci/built-in.a' + expected = "arch/x86/pci/i386.o arch/x86/pci/init.o arch/x86/pci/mmconfig_64.o arch/x86/pci/direct.o arch/x86/pci/mmconfig-shared.o arch/x86/pci/fixup.o arch/x86/pci/acpi.o arch/x86/pci/legacy.o arch/x86/pci/irq.o arch/x86/pci/common.o arch/x86/pci/early.o arch/x86/pci/bus_numa.o arch/x86/pci/amd_bus.o" + self._assert_parsing(cmd, expected) + + # ar command tests + def test_ar_reordering(self): + cmd = "rm -f vmlinux.a; ar cDPrST vmlinux.a built-in.a lib/lib.a arch/x86/lib/lib.a; ar mPiT $$(ar t vmlinux.a | sed -n 1p) vmlinux.a $$(ar t vmlinux.a | grep -F -f ../scripts/head-object-list.txt)" + expected = "built-in.a lib/lib.a arch/x86/lib/lib.a" + self._assert_parsing(cmd, expected) + + def test_ar_default(self): + cmd = "rm -f lib/lib.a; ar cDPrsT lib/lib.a lib/argv_split.o lib/bug.o lib/buildid.o lib/clz_tab.o lib/cmdline.o lib/cpumask.o lib/ctype.o lib/dec_and_lock.o lib/decompress.o lib/decompress_bunzip2.o lib/decompress_inflate.o lib/decompress_unlz4.o lib/decompress_unlzma.o lib/decompress_unlzo.o lib/decompress_unxz.o lib/decompress_unzstd.o lib/dump_stack.o lib/earlycpio.o lib/extable.o lib/flex_proportions.o lib/idr.o lib/iomem_copy.o lib/irq_regs.o lib/is_single_threaded.o lib/klist.o lib/kobject.o lib/kobject_uevent.o lib/logic_pio.o lib/maple_tree.o lib/memcat_p.o lib/nmi_backtrace.o lib/objpool.o lib/plist.o lib/radix-tree.o lib/ratelimit.o lib/rbtree.o lib/seq_buf.o lib/siphash.o lib/string.o lib/sys_info.o lib/timerqueue.o lib/union_find.o lib/vsprintf.o lib/win_minmax.o lib/xarray.o" + expected = "lib/argv_split.o lib/bug.o lib/buildid.o lib/clz_tab.o lib/cmdline.o lib/cpumask.o lib/ctype.o lib/dec_and_lock.o lib/decompress.o lib/decompress_bunzip2.o lib/decompress_inflate.o lib/decompress_unlz4.o lib/decompress_unlzma.o lib/decompress_unlzo.o lib/decompress_unxz.o lib/decompress_unzstd.o lib/dump_stack.o lib/earlycpio.o lib/extable.o lib/flex_proportions.o lib/idr.o lib/iomem_copy.o lib/irq_regs.o lib/is_single_threaded.o lib/klist.o lib/kobject.o lib/kobject_uevent.o lib/logic_pio.o lib/maple_tree.o lib/memcat_p.o lib/nmi_backtrace.o lib/objpool.o lib/plist.o lib/radix-tree.o lib/ratelimit.o lib/rbtree.o lib/seq_buf.o lib/siphash.o lib/string.o lib/sys_info.o lib/timerqueue.o lib/union_find.o lib/vsprintf.o lib/win_minmax.o lib/xarray.o" + self._assert_parsing(cmd, expected) + + def test_ar_llvm(self): + cmd = "llvm-ar mPiT $$(llvm-ar t vmlinux.a | sed -n 1p) vmlinux.a $$(llvm-ar t vmlinux.a | grep -F -f ../scripts/head-object-list.txt)" + expected = "" + self._assert_parsing(cmd, expected) + + # nm command tests + def test_nm(self): + cmd = """llvm-nm -p --defined-only rust/core.o | awk '$$2~/(T|R|D|B)/ && $$3!~/__(pfx|cfi|odr_asan)/ { printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3 }' > rust/exports_core_generated.h""" + expected = "rust/core.o" + self._assert_parsing(cmd, expected) + + def test_nm_vmlinux(self): + cmd = r"nm vmlinux | sed -n -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|__bss_start\|_end\)$/#define VO_\2 _AC(0x\1,UL)/p' > arch/x86/boot/voffset.h" + expected = "vmlinux" + self._assert_parsing(cmd, expected) + + # objcopy command tests + def test_objcopy(self): + cmd = "objcopy --remove-section='.rel*' --remove-section=!'.rel*.dyn' vmlinux.unstripped vmlinux" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + def test_objcopy_llvm(self): + cmd = "llvm-objcopy --remove-section='.rel*' --remove-section=!'.rel*.dyn' vmlinux.unstripped vmlinux" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + # strip command tests + def test_strip(self): + cmd = "strip --strip-debug -o drivers/firmware/efi/libstub/mem.stub.o drivers/firmware/efi/libstub/mem.o" + expected = "drivers/firmware/efi/libstub/mem.o" + self._assert_parsing(cmd, expected) + + # cp command tests + def test_cp_truncate(self): + cmd = "cp arch/arm64/boot/Image arch/arm64/boot/vmlinux.bin; truncate -s $$(hexdump -s16 -n4 -e '\"%u\"' arch/arm64/boot/Image) arch/arm64/boot/vmlinux.bin" + expected = "arch/arm64/boot/Image" + self._assert_parsing(cmd, expected) + + # rustc command tests + def test_rustc(self): + cmd = """OBJTREE=/workspace/linux/kernel_build rustc -Zbinary_dep_depinfo=y -Astable_features -Dnon_ascii_idents -Dunsafe_op_in_unsafe_fn -Wmissing_docs -Wrust_2018_idioms -Wclippy::all -Wclippy::as_ptr_cast_mut -Wclippy::as_underscore -Wclippy::cast_lossless -Wclippy::ignored_unit_patterns -Wclippy::mut_mut -Wclippy::needless_bitwise_bool -Aclippy::needless_lifetimes -Wclippy::no_mangle_with_rust_abi -Wclippy::ptr_as_ptr -Wclippy::ptr_cast_constness -Wclippy::ref_as_ptr -Wclippy::undocumented_unsafe_blocks -Wclippy::unnecessary_safety_comment -Wclippy::unnecessary_safety_doc -Wrustdoc::missing_crate_level_docs -Wrustdoc::unescaped_backticks -Cpanic=abort -Cembed-bitcode=n -Clto=n -Cforce-unwind-tables=n -Ccodegen-units=1 -Csymbol-mangling-version=v0 -Crelocation-model=static -Zfunction-sections=n -Wclippy::float_arithmetic --target=./scripts/target.json -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2 -Zcf-protection=branch -Zno-jump-tables -Ctarget-cpu=x86-64 -Ztune-cpu=generic -Cno-redzone=y -Ccode-model=kernel -Zfunction-return=thunk-extern -Zpatchable-function-entry=16,16 -Copt-level=2 -Cdebug-assertions=n -Coverflow-checks=y -Dwarnings @./include/generated/rustc_cfg --edition=2021 --cfg no_fp_fmt_parse --emit=dep-info=rust/.core.o.d --emit=obj=rust/core.o --emit=metadata=rust/libcore.rmeta --crate-type rlib -L./rust --crate-name core /usr/lib/rust-1.84/lib/rustlib/src/rust/library/core/src/lib.rs --sysroot=/dev/null ;llvm-objcopy --redefine-sym __addsf3=__rust__addsf3 --redefine-sym __eqsf2=__rust__eqsf2 --redefine-sym __extendsfdf2=__rust__extendsfdf2 --redefine-sym __gesf2=__rust__gesf2 --redefine-sym __lesf2=__rust__lesf2 --redefine-sym __ltsf2=__rust__ltsf2 --redefine-sym __mulsf3=__rust__mulsf3 --redefine-sym __nesf2=__rust__nesf2 --redefine-sym __truncdfsf2=__rust__truncdfsf2 --redefine-sym __unordsf2=__rust__unordsf2 --redefine-sym __adddf3=__rust__adddf3 --redefine-sym __eqdf2=__rust__eqdf2 --redefine-sym __ledf2=__rust__ledf2 --redefine-sym __ltdf2=__rust__ltdf2 --redefine-sym __muldf3=__rust__muldf3 --redefine-sym __unorddf2=__rust__unorddf2 --redefine-sym __muloti4=__rust__muloti4 --redefine-sym __multi3=__rust__multi3 --redefine-sym __udivmodti4=__rust__udivmodti4 --redefine-sym __udivti3=__rust__udivti3 --redefine-sym __umodti3=__rust__umodti3 rust/core.o""" + expected = "/usr/lib/rust-1.84/lib/rustlib/src/rust/library/core/src/lib.rs rust/core.o" + self._assert_parsing(cmd, expected) + + # rustdoc command tests + def test_rustdoc(self): + cmd = """OBJTREE=/workspace/linux/kernel_build rustdoc --test --edition=2021 -Zbinary_dep_depinfo=y -Astable_features -Dnon_ascii_idents -Dunsafe_op_in_unsafe_fn -Wmissing_docs -Wrust_2018_idioms -Wunreachable_pub -Wclippy::all -Wclippy::as_ptr_cast_mut -Wclippy::as_underscore -Wclippy::cast_lossless -Wclippy::ignored_unit_patterns -Wclippy::mut_mut -Wclippy::needless_bitwise_bool -Aclippy::needless_lifetimes -Wclippy::no_mangle_with_rust_abi -Wclippy::ptr_as_ptr -Wclippy::ptr_cast_constness -Wclippy::ref_as_ptr -Wclippy::undocumented_unsafe_blocks -Wclippy::unnecessary_safety_comment -Wclippy::unnecessary_safety_doc -Wrustdoc::missing_crate_level_docs -Wrustdoc::unescaped_backticks -Cpanic=abort -Cembed-bitcode=n -Clto=n -Cforce-unwind-tables=n -Ccodegen-units=1 -Csymbol-mangling-version=v0 -Crelocation-model=static -Zfunction-sections=n -Wclippy::float_arithmetic --target=aarch64-unknown-none -Ctarget-feature="-neon" -Cforce-unwind-tables=n -Zbranch-protection=pac-ret -Copt-level=2 -Cdebug-assertions=y -Coverflow-checks=y -Dwarnings -Cforce-frame-pointers=y -Zsanitizer=kernel-address -Zsanitizer-recover=kernel-address -Cllvm-args=-asan-mapping-offset=0xdfff800000000000 -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=3 -Cllvm-args=-sanitizer-coverage-trace-pc -Cllvm-args=-sanitizer-coverage-trace-compares @./include/generated/rustc_cfg -L./rust --extern ffi --extern pin_init --extern kernel --extern build_error --extern macros --extern bindings --extern uapi --no-run --crate-name kernel -Zunstable-options --sysroot=/dev/null --test-builder ./scripts/rustdoc_test_builder ../rust/kernel/lib.rs >/dev/null""" + expected = "../rust/kernel/lib.rs" + self._assert_parsing(cmd, expected) + + def test_rustdoc_test_gen(self): + cmd = "./scripts/rustdoc_test_gen" + expected = "" + self._assert_parsing(cmd, expected) + + # flex command tests + def test_flex(self): + cmd = "flex -oscripts/kconfig/lexer.lex.c -L ../scripts/kconfig/lexer.l" + expected = "../scripts/kconfig/lexer.l" + self._assert_parsing(cmd, expected) + + # bison command tests + def test_bison(self): + cmd = "bison -o scripts/kconfig/parser.tab.c --defines=scripts/kconfig/parser.tab.h -t -l ../scripts/kconfig/parser.y" + expected = "../scripts/kconfig/parser.y" + self._assert_parsing(cmd, expected) + + # bindgen command tests + def test_bindgen(self): + cmd = ( + "bindgen ../rust/bindings/bindings_helper.h " + "--blocklist-type __kernel_s?size_t --blocklist-type __kernel_ptrdiff_t " + "--opaque-type xregs_state --opaque-type desc_struct --no-doc-comments " + "--rust-target 1.68 --use-core --with-derive-default -o rust/bindings/bindings_generated.rs " + "-- -Wp,-MMD,rust/bindings/.bindings_generated.rs.d -nostdinc -I../arch/x86/include " + "-include ../include/linux/compiler-version.h -D__KERNEL__ -fintegrated-as -fno-builtin -DMODULE; " + "sed -Ei 's/pub const RUST_CONST_HELPER_([a-zA-Z0-9_]*)/pub const \\1/g' rust/bindings/bindings_generated.rs" + ) + expected = "../rust/bindings/bindings_helper.h ../include/linux/compiler-version.h" + self._assert_parsing(cmd, expected) + + # perl command tests + def test_perl(self): + cmd = "perl ../lib/crypto/x86/poly1305-x86_64-cryptogams.pl > lib/crypto/x86/poly1305-x86_64-cryptogams.S" + expected = "../lib/crypto/x86/poly1305-x86_64-cryptogams.pl" + self._assert_parsing(cmd, expected) + + # link-vmlinux.sh command tests + def test_link_vmlinux(self): + cmd = '../scripts/link-vmlinux.sh "ld" "-m elf_x86_64 -z noexecstack" "-z max-page-size=0x200000 --build-id=sha1 --orphan-handling=error --emit-relocs --discard-none" "vmlinux.unstripped"; true' + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + def test_link_vmlinux_postlink(self): + cmd = '../scripts/link-vmlinux.sh "ld" "-m elf_x86_64 -z noexecstack --no-warn-rwx-segments" "--emit-relocs --discard-none -z max-page-size=0x200000 --build-id=sha1 -X --orphan-handling=error"; make -f ../arch/x86/Makefile.postlink vmlinux' + expected = "vmlinux.a" + self._assert_parsing(cmd, expected) + + # syscallhdr.sh command tests + def test_syscallhdr(self): + cmd = "sh ../scripts/syscallhdr.sh --abis common,64 --emit-nr ../arch/x86/entry/syscalls/syscall_64.tbl arch/x86/include/generated/uapi/asm/unistd_64.h" + expected = "../arch/x86/entry/syscalls/syscall_64.tbl" + self._assert_parsing(cmd, expected) + + # syscalltbl.sh command tests + def test_syscalltbl(self): + cmd = "sh ../scripts/syscalltbl.sh --abis common,64 ../arch/x86/entry/syscalls/syscall_64.tbl arch/x86/include/generated/asm/syscalls_64.h" + expected = "../arch/x86/entry/syscalls/syscall_64.tbl" + self._assert_parsing(cmd, expected) + + # mkcapflags.sh command tests + def test_mkcapflags(self): + cmd = "sh ../arch/x86/kernel/cpu/mkcapflags.sh arch/x86/kernel/cpu/capflags.c ../arch/x86/kernel/cpu/../../include/asm/cpufeatures.h ../arch/x86/kernel/cpu/../../include/asm/vmxfeatures.h ../arch/x86/kernel/cpu/mkcapflags.sh FORCE" + expected = "../arch/x86/kernel/cpu/../../include/asm/cpufeatures.h ../arch/x86/kernel/cpu/../../include/asm/vmxfeatures.h" + self._assert_parsing(cmd, expected) + + # orc_hash.sh command tests + def test_orc_hash(self): + cmd = "mkdir -p arch/x86/include/generated/asm/; sh ../scripts/orc_hash.sh < ../arch/x86/include/asm/orc_types.h > arch/x86/include/generated/asm/orc_hash.h" + expected = "../arch/x86/include/asm/orc_types.h" + self._assert_parsing(cmd, expected) + + # xen-hypercalls.sh command tests + def test_xen_hypercalls(self): + cmd = "sh '../scripts/xen-hypercalls.sh' arch/x86/include/generated/asm/xen-hypercalls.h ../include/xen/interface/xen-mca.h ../include/xen/interface/xen.h ../include/xen/interface/xenpmu.h" + expected = "../include/xen/interface/xen-mca.h ../include/xen/interface/xen.h ../include/xen/interface/xenpmu.h" + self._assert_parsing(cmd, expected) + + # gen_initramfs.sh command tests + def test_gen_initramfs(self): + cmd = "sh ../usr/gen_initramfs.sh -o usr/initramfs_data.cpio -l usr/.initramfs_data.cpio.d ../usr/default_cpio_list" + expected = "../usr/default_cpio_list" + self._assert_parsing(cmd, expected) + + # mkuboot.sh command tests + def test_mkuboot(self): + cmd = "bash ../scripts/mkuboot.sh -A arm -O linux -C none -T kernel -a 0x8000 -e 0x8000 -n 'Linux-6.15.0' -d arch/arm/boot/zImage arch/arm/boot/uImage" + expected = "arch/arm/boot/zImage" + self._assert_parsing(cmd, expected) + + # syscallnr.sh command tests + def test_syscallnr(self): + cmd = "sh ../arch/arm/tools/syscallnr.sh ../arch/arm/tools/syscall.tbl arch/arm/include/generated/asm/unistd-nr.h" + expected = "../arch/arm/tools/syscall.tbl" + self._assert_parsing(cmd, expected) + + # gen-kernel-hwcaps.sh command tests + def test_gen_kernel_hwcaps(self): + cmd = "/bin/sh -e ../arch/arm64/tools/gen-kernel-hwcaps.sh ../arch/arm64/include/uapi/asm/hwcap.h > arch/arm64/include/generated/asm/kernel-hwcap.h" + expected = "../arch/arm64/include/uapi/asm/hwcap.h" + self._assert_parsing(cmd, expected) + + # vdso2c command tests + def test_vdso2c(self): + cmd = "arch/x86/entry/vdso/vdso2c arch/x86/entry/vdso/vdso64.so.dbg arch/x86/entry/vdso/vdso64.so arch/x86/entry/vdso/vdso-image-64.c" + expected = "arch/x86/entry/vdso/vdso64.so.dbg arch/x86/entry/vdso/vdso64.so" + self._assert_parsing(cmd, expected) + + # vdsomunge command tests + def test_vdsomunge(self): + cmd = "arch/arm64/kernel/vdso32/../../../arm/vdso/vdsomunge arch/arm64/kernel/vdso32/vdso.so.raw arch/arm64/kernel/vdso32/vdso32.so.dbg" + expected = "arch/arm64/kernel/vdso32/vdso.so.raw" + self._assert_parsing(cmd, expected) + + # mkpiggy command tests + def test_mkpiggy(self): + cmd = "arch/x86/boot/compressed/mkpiggy arch/x86/boot/compressed/vmlinux.bin.gz > arch/x86/boot/compressed/piggy.S" + expected = "arch/x86/boot/compressed/vmlinux.bin.gz" + self._assert_parsing(cmd, expected) + + # relocs command tests + def test_relocs(self): + cmd = "arch/x86/tools/relocs vmlinux.unstripped > arch/x86/boot/compressed/vmlinux.relocs;arch/x86/tools/relocs --abs-relocs vmlinux.unstripped" + expected = "vmlinux.unstripped" + self._assert_parsing(cmd, expected) + + def test_relocs_with_realmode(self): + cmd = ( + "arch/x86/tools/relocs --realmode arch/x86/realmode/rm/realmode.elf > arch/x86/realmode/rm/realmode.relocs" + ) + expected = "arch/x86/realmode/rm/realmode.elf" + self._assert_parsing(cmd, expected) + + # mk_elfconfig command tests + def test_mk_elfconfig(self): + cmd = "scripts/mod/mk_elfconfig < scripts/mod/empty.o > scripts/mod/elfconfig.h" + expected = "scripts/mod/empty.o" + self._assert_parsing(cmd, expected) + + # tools/build command tests + def test_build(self): + cmd = "arch/x86/boot/tools/build arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin arch/x86/boot/zoffset.h arch/x86/boot/bzImage" + expected = "arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin arch/x86/boot/zoffset.h" + self._assert_parsing(cmd, expected) + + # extract-cert command tests + def test_extract_cert(self): + cmd = 'certs/extract-cert "" certs/signing_key.x509' + expected = "" + self._assert_parsing(cmd, expected) + + # dtc command tests + def test_dtc_cat(self): + cmd = "./scripts/dtc/dtc -o drivers/of/empty_root.dtb -b 0 -i../drivers/of/ -i../scripts/dtc/include-prefixes -Wno-unique_unit_address -Wno-unit_address_vs_reg -Wno-avoid_unnecessary_addr_size -Wno-alias_paths -Wno-graph_child_address -Wno-simple_bus_reg -d drivers/of/.empty_root.dtb.d.dtc.tmp drivers/of/.empty_root.dtb.dts.tmp ; cat drivers/of/.empty_root.dtb.d.pre.tmp drivers/of/.empty_root.dtb.d.dtc.tmp > drivers/of/.empty_root.dtb.d" + expected = "drivers/of/.empty_root.dtb.dts.tmp drivers/of/.empty_root.dtb.d.pre.tmp drivers/of/.empty_root.dtb.d.dtc.tmp" + self._assert_parsing(cmd, expected) + + # pnmtologo command tests + def test_pnmtologo(self): + cmd = "drivers/video/logo/pnmtologo -t clut224 -n logo_linux_clut224 -o drivers/video/logo/logo_linux_clut224.c ../drivers/video/logo/logo_linux_clut224.ppm" + expected = "../drivers/video/logo/logo_linux_clut224.ppm" + self._assert_parsing(cmd, expected) + + # relacheck command tests + def test_relacheck(self): + cmd = "arch/arm64/kernel/pi/relacheck arch/arm64/kernel/pi/idreg-override.pi.o arch/arm64/kernel/pi/idreg-override.o" + expected = "arch/arm64/kernel/pi/idreg-override.pi.o" + self._assert_parsing(cmd, expected) + + # gen-hyprel command tests + def test_gen_hyprel(self): + cmd = "arch/arm64/kvm/hyp/nvhe/gen-hyprel arch/arm64/kvm/hyp/nvhe/kvm_nvhe.tmp.o > arch/arm64/kvm/hyp/nvhe/hyp-reloc.S" + expected = "arch/arm64/kvm/hyp/nvhe/kvm_nvhe.tmp.o" + self._assert_parsing(cmd, expected) + + # mkregtable command tests + def test_mkregtable(self): + cmd = "drivers/gpu/drm/radeon/mkregtable ../drivers/gpu/drm/radeon/reg_srcs/r100 > drivers/gpu/drm/radeon/r100_reg_safe.h" + expected = "../drivers/gpu/drm/radeon/reg_srcs/r100" + self._assert_parsing(cmd, expected) + + # genheaders command tests + def test_genheaders(self): + cmd = "security/selinux/genheaders security/selinux/flask.h security/selinux/av_permissions.h" + expected = "" + self._assert_parsing(cmd, expected) + + # mkcpustr command tests + def test_mkcpustr(self): + cmd = "arch/x86/boot/mkcpustr > arch/x86/boot/cpustr.h" + expected = "" + self._assert_parsing(cmd, expected) + + # polgen command tests + def test_polgen(self): + cmd = "scripts/ipe/polgen/polgen security/ipe/boot_policy.c" + expected = "" + self._assert_parsing(cmd, expected) + + # gen_header.py command tests + def test_gen_header(self): + cmd = "mkdir -p drivers/gpu/drm/msm/generated && python3 ../drivers/gpu/drm/msm/registers/gen_header.py --no-validate --rnn ../drivers/gpu/drm/msm/registers --xml ../drivers/gpu/drm/msm/registers/adreno/a2xx.xml c-defines > drivers/gpu/drm/msm/generated/a2xx.xml.h" + expected = "../drivers/gpu/drm/msm/registers/adreno/a2xx.xml" + self._assert_parsing(cmd, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/sbom/tests/spdx_graph/__init__.py b/scripts/sbom/tests/spdx_graph/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/sbom/tests/spdx_graph/test_kernel_file.py b/scripts/sbom/tests/spdx_graph/test_kernel_file.py new file mode 100644 index 000000000000..35a63a768ba2 --- /dev/null +++ b/scripts/sbom/tests/spdx_graph/test_kernel_file.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import unittest +from pathlib import Path +import tempfile +from sbom.spdx_graph.kernel_file import _parse_spdx_license_identifier # type: ignore + + +class TestKernelFile(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.src_tree = Path(self.tmpdir.name) + + def tearDown(self): + self.tmpdir.cleanup() + + def test_parse_spdx_license_identifier(self): + # REUSE-IgnoreStart + test_cases: list[tuple[str, str | None]] = [ + ("/* SPDX-License-Identifier: MIT*/", "MIT"), + ("// SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("# SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("#!/bin/bash\n# SPDX-License-Identifier: GPL-2.0-only", "GPL-2.0-only"), + ("/* SPDX-License-Identifier: GPL-2.0-or-later OR MIT */", "GPL-2.0-or-later OR MIT"), + ("/* SPDX-License-Identifier: Apache-2.0 */\n extra text", "Apache-2.0"), + ("", "GPL-2.0"), + ("int main() { return 0; }", None), + ] + # REUSE-IgnoreEnd + + for i, (file_content, expected_identifier) in enumerate(test_cases): + file_path = self.src_tree / f"file_{i}.c" + file_path.write_text(file_content) + self.assertEqual(_parse_spdx_license_identifier(str(file_path)), expected_identifier) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index e8ed11c680c6..d8dc2a1b7c31 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -891,17 +891,22 @@ static int do_file(char const *const fname, void *addr) table_sort_t custom_sort = NULL; switch (elf_map_machine(ehdr)) { - case EM_AARCH64: #ifdef MCOUNT_SORT_ENABLED + case EM_AARCH64: + /* arm64 also needs RELA-based weak-function fixups. */ sort_reloc = true; rela_type = 0x403; - /* arm64 uses patchable function entry placing before function */ + /* fallthrough */ + case EM_RISCV: + /* arm64 and RISC-V place patchable entries before the function. */ before_func = 8; +#else + case EM_AARCH64: + case EM_RISCV: #endif /* fallthrough */ case EM_386: case EM_LOONGARCH: - case EM_RISCV: case EM_S390: case EM_X86_64: custom_sort = sort_relative_table_with_data; diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 2f2e81dbda03..3372873cd7bb 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -176,6 +176,7 @@ assgined||assigned assiged||assigned assigment||assignment assigments||assignments +assinged||assigned assistent||assistant assocaited||associated assocated||associated diff --git a/security/apparmor/Kconfig b/security/apparmor/Kconfig index 1e3bd44643da..1c5608068ba3 100644 --- a/security/apparmor/Kconfig +++ b/security/apparmor/Kconfig @@ -2,6 +2,7 @@ config SECURITY_APPARMOR bool "AppArmor support" depends on SECURITY && NET + depends on !DEBUG_AID_FOR_SYZBOT select AUDIT select SECURITY_PATH select SECURITYFS diff --git a/security/apparmor/af_unix.c b/security/apparmor/af_unix.c index fdb4a9f212c3..9ca9d1b890ba 100644 --- a/security/apparmor/af_unix.c +++ b/security/apparmor/af_unix.c @@ -20,6 +20,7 @@ #include "include/apparmor.h" #include "include/file.h" #include "include/label.h" +#include "include/net.h" #include "include/path.h" #include "include/policy.h" #include "include/cred.h" @@ -615,7 +616,7 @@ static int unix_peer_perm(const struct cred *subj_cred, peer_label, &ad)); } -/** +/* * * Requires: lock held on both @sk and @peer_sk * called by unix_stream_connect, unix_may_send @@ -674,9 +675,11 @@ static void update_sk_ctx(struct sock *sk, struct aa_label *label, old = rcu_dereference_protected(ctx->peer, lockdep_is_held(&unix_sk(sk)->lock)); if (old == plabel) { - rcu_assign_pointer(ctx->peer_lastupdate, plabel); + rcu_assign_pointer(ctx->peer_lastupdate, + aa_get_label(plabel)); } else if (aa_label_is_subset(plabel, old)) { - rcu_assign_pointer(ctx->peer_lastupdate, plabel); + rcu_assign_pointer(ctx->peer_lastupdate, + aa_get_label(plabel)); rcu_assign_pointer(ctx->peer, aa_get_label(plabel)); aa_put_label(old); } /* else race or a subset - don't update */ @@ -748,42 +751,47 @@ int aa_unix_file_perm(const struct cred *subj_cred, struct aa_label *label, if (!peer_sk) goto out; - peer_addr = aa_sunaddr(unix_sk(peer_sk), &peer_addrlen); - - struct path peer_path; - - peer_path = unix_sk(peer_sk)->path; - if (!is_sk_fs && is_unix_fs(peer_sk)) { - last_error(error, - unix_fs_perm(op, request, subj_cred, label, - is_unix_fs(peer_sk) ? &peer_path : NULL)); - } else if (!is_sk_fs) { - struct aa_label *plabel; - struct aa_sk_ctx *pctx = aa_sock(peer_sk); - - rcu_read_lock(); - plabel = aa_get_label_rcu(&pctx->label); - rcu_read_unlock(); - /* no fs check of aa_unix_peer_perm because conditions above - * ensure they will never be done - */ - last_error(error, - xcheck(unix_peer_perm(subj_cred, label, op, + if (!is_sk_fs) { + bool is_peer_fs = is_unix_fs(peer_sk); + + peer_addr = aa_sunaddr(unix_sk(peer_sk), &peer_addrlen); + if (is_peer_fs) { + struct path peer_path; + + unix_state_lock(peer_sk); + peer_path = unix_sk(peer_sk)->path; + if (peer_path.dentry) + path_get(&peer_path); + unix_state_unlock(peer_sk); + + last_error(error, + unix_fs_perm(op, request, subj_cred, label, + &peer_path)); + if (peer_path.dentry) + path_put(&peer_path); + } else { + struct aa_sk_ctx *pctx = aa_sock(peer_sk); + + rcu_read_lock(); + plabel = aa_get_newest_label(pctx->label); + rcu_read_unlock(); + /* no fs check of aa_unix_peer_perm because conditions + * above ensure they will never be done + */ + last_error(error, + xcheck(unix_peer_perm(subj_cred, label, op, MAY_READ | MAY_WRITE, sock->sk, is_sk_fs ? &path : NULL, peer_addr, peer_addrlen, - is_unix_fs(peer_sk) ? - &peer_path : NULL, - plabel), - unix_peer_perm(file->f_cred, plabel, op, + NULL, plabel), + unix_peer_perm(file->f_cred, plabel, op, MAY_READ | MAY_WRITE, peer_sk, - is_unix_fs(peer_sk) ? - &peer_path : NULL, - addr, addrlen, + NULL, addr, addrlen, is_sk_fs ? &path : NULL, label))); - if (!error && !__aa_subj_label_is_cached(plabel, label)) - update_peer_ctx(peer_sk, pctx, label); + if (!error && !__aa_subj_label_is_cached(plabel, label)) + update_peer_ctx(peer_sk, pctx, label); + } } sock_put(peer_sk); diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index ededaf46f3ca..2ae9ab94a5a9 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include @@ -32,6 +34,7 @@ #include "include/crypto.h" #include "include/ipc.h" #include "include/label.h" +#include "include/net.h" #include "include/lib.h" #include "include/policy.h" #include "include/policy_ns.h" @@ -71,10 +74,10 @@ struct rawdata_f_data { struct aa_loaddata *loaddata; + DECLARE_FLEX_ARRAY(char, data); }; #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY -#define RAWDATA_F_DATA_BUF(p) (char *)(p + 1) static void rawdata_f_data_free(struct rawdata_f_data *private) { @@ -174,6 +177,7 @@ static struct aa_proxy *get_proxy_common_ref(struct aa_common_ref *ref) return NULL; } +#ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) { if (ref) @@ -181,6 +185,7 @@ static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) count)); return NULL; } +#endif static void aa_put_common_ref(struct aa_common_ref *ref) { @@ -478,6 +483,78 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, return data; } +static int decompress_zstd(char *src, size_t slen, char *dst, size_t dlen); +/** + * aa_get_data_from_compressed - common routine for getting compressed policy + * from user and get both compressed and uncompressed version. + * @userbuf: user buffer to copy data from (NOT NULL) + * @buffer_size: size of user buffer + * @pos: position write is at in the file (NOT NULL) + * @compressed_data Ptr on compressed data. *compressed_data is allocated there + * + * Returns: kernel buffer containing copy of user buffer data or an + * ERR_PTR on failure. + */ + +static struct aa_loaddata *aa_get_data_from_compressed(const char __user *userbuf, + size_t buffer_size, + loff_t *pos, + char **compressed_data) +{ + struct aa_loaddata *data; + zstd_frame_header header; + int error; + + if (!userbuf || !pos) + return ERR_PTR(-EINVAL); + if (*pos) + return ERR_PTR(-ESPIPE); + + *compressed_data = kvmalloc(buffer_size, GFP_KERNEL); + if (!*compressed_data) + return ERR_PTR(-ENOMEM); + error = copy_from_user(*compressed_data, userbuf, buffer_size); + if (error) + goto fail; + + error = zstd_get_frame_header(&header, *compressed_data, buffer_size); + if (error || header.frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN || + header.frameContentSize == ZSTD_CONTENTSIZE_ERROR) { + error = -EINVAL; + goto fail; + } + + data = aa_loaddata_alloc(header.frameContentSize); + if (IS_ERR(data)) { + error = PTR_ERR(data); + goto fail; + } + + // We then decompress the data + error = decompress_zstd(*compressed_data, buffer_size, data->data, + header.frameContentSize); + if (error) + goto fail_decompress; + + data->size = header.frameContentSize; + return data; + +fail_decompress: + aa_put_i_loaddata(data); +fail: + kvfree(*compressed_data); + return ERR_PTR(error); + +} + +struct aa_user_hdr { + uint8_t version; + uint8_t compress_level; + uint8_t padding[6]; /* force 8-byte alignment */ +} __packed __aligned(8); + +#define aa_hdr_magic "\x04\x08\x00\x76\x65\x72\x73\x69\x6f\x6e\x00\x02" +#define aa_hdr_magic_size 12 static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, loff_t *pos, struct aa_ns *ns, @@ -486,6 +563,10 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, struct aa_loaddata *data; struct aa_label *label; ssize_t error; + char *compressed_data = NULL; + __le32 magic_le; + bool is_compressed; + u8 aahdr[aa_hdr_magic_size]; label = begin_current_label_crit_section(); @@ -496,10 +577,52 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, if (error) goto end_section; - data = aa_simple_write_to_buffer(buf, size, size, pos); - error = PTR_ERR(data); + /* If the policy is userspace compressed we start by decompressing it + * to make the required checks (computing hash, verifying profile, ...) + * + * Getting a userspace-compressed version then decompressing it in the + * kernel actually makes sense since zstd decompression is ~3.5x faster + * than compression. This also allow to increase the compression level. + */ + + if (size >= sizeof(__le32) && + !copy_from_user(&magic_le, buf, sizeof(magic_le)) && + le32_to_cpu(magic_le) == ZSTD_MAGICNUMBER) { + is_compressed = true; + } else if (size >= sizeof(struct aa_user_hdr) + sizeof(__le32) && + !copy_from_user(&magic_le, + buf + sizeof(struct aa_user_hdr), + sizeof(magic_le)) && + le32_to_cpu(magic_le) == ZSTD_MAGICNUMBER) { + is_compressed = true; + /* skip the userspace header if present */ + buf += sizeof(struct aa_user_hdr); + size -= sizeof(struct aa_user_hdr); + } else if (size >= sizeof(struct aa_user_hdr) + + aa_hdr_magic_size && + !copy_from_user(&aahdr, + buf + sizeof(struct aa_user_hdr), + aa_hdr_magic_size) && + memcmp(&aahdr, aa_hdr_magic, aa_hdr_magic_size) == 0) { + /* uncompressed blob with user hdr */ + buf += sizeof(struct aa_user_hdr); + size -= sizeof(struct aa_user_hdr); + is_compressed = false; + } else { + is_compressed = false; + } + if (is_compressed) { + + data = aa_get_data_from_compressed(buf, size, pos, &compressed_data); + error = PTR_ERR(data); + } else { + data = aa_simple_write_to_buffer(buf, size, size, pos); + error = PTR_ERR(data); + } + if (!IS_ERR(data)) { - error = aa_replace_profiles(ns, label, mask, data); + error = aa_replace_profiles(ns, label, mask, data, + compressed_data, size); /* put pcount, which will put count and free if no * profiles referencing it. */ @@ -546,6 +669,7 @@ static const struct file_operations aa_fs_profile_replace = { .llseek = default_llseek, }; + /* .remove file hook fn to remove loaded policy */ static ssize_t profile_remove(struct file *f, const char __user *buf, size_t size, loff_t *pos) @@ -904,7 +1028,7 @@ static void multi_transaction_kref(struct kref *kref) struct multi_transaction *t; t = container_of(kref, struct multi_transaction, count); - free_page((unsigned long) t); + kfree(t); } static struct multi_transaction * @@ -947,7 +1071,7 @@ static struct multi_transaction *multi_transaction_new(struct file *file, if (size > MULTI_TRANSACTION_LIMIT - 1) return ERR_PTR(-EFBIG); - t = (struct multi_transaction *)get_zeroed_page(GFP_KERNEL); + t = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!t) return ERR_PTR(-ENOMEM); kref_init(&t->count); @@ -1434,7 +1558,7 @@ static ssize_t rawdata_read(struct file *file, char __user *buf, size_t size, struct rawdata_f_data *private = file->private_data; return simple_read_from_buffer(buf, size, ppos, - RAWDATA_F_DATA_BUF(private), + private->data, private->loaddata->size); } @@ -1467,8 +1591,7 @@ static int rawdata_open(struct inode *inode, struct file *file) private->loaddata = loaddata; error = decompress_zstd(loaddata->data, loaddata->compressed_size, - RAWDATA_F_DATA_BUF(private), - loaddata->size); + private->data, loaddata->size); if (error) goto fail_decompress; @@ -1756,6 +1879,80 @@ static const struct inode_operations rawdata_link_abi_iops = { static const struct inode_operations rawdata_link_data_iops = { .get_link = rawdata_get_link_data, }; + +/* + * Requires: @profile->ns->lock held + */ +void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile) +{ + aafs_remove(profile->dents[AAFS_PROF_RAW_HASH]); + profile->dents[AAFS_PROF_RAW_HASH] = NULL; + aafs_remove(profile->dents[AAFS_PROF_RAW_ABI]); + profile->dents[AAFS_PROF_RAW_ABI] = NULL; + aafs_remove(profile->dents[AAFS_PROF_RAW_DATA]); + profile->dents[AAFS_PROF_RAW_DATA] = NULL; +} + +static inline int create_symlink_dent(struct aa_profile *profile, + const char *name, + enum aafs_prof_type type, + const struct inode_operations *iops) +{ + struct dentry *dent = NULL; + struct dentry *dir = prof_dir(profile); + + if (profile->dents[type]) + return 0; + + dent = aafs_create(name, S_IFLNK | 0444, dir, + &profile->label.proxy->count, NULL, NULL, iops); + if (IS_ERR(dent)) + return PTR_ERR(dent); + + profile->dents[type] = dent; + return 0; +} + +/* + * Requires: @profile->ns->lock held + */ +int __aa_create_rawdata_symlink_dents(struct aa_profile *profile) +{ + int error; + + if (!profile || + (profile->dents[AAFS_PROF_RAW_HASH] && + profile->dents[AAFS_PROF_RAW_ABI] && + profile->dents[AAFS_PROF_RAW_DATA])) + return 0; + + if (!profile->rawdata) + return 0; + + if (aa_g_hash_policy) { + error = create_symlink_dent(profile, "raw_sha256", + AAFS_PROF_RAW_HASH, + &rawdata_link_sha256_iops); + if (error) + return error; + } + + error = create_symlink_dent(profile, "raw_abi", + AAFS_PROF_RAW_ABI, + &rawdata_link_abi_iops); + if (error) + return error; + + + error = create_symlink_dent(profile, "raw_data", + AAFS_PROF_RAW_DATA, + &rawdata_link_data_iops); + if (error) + return error; + + return 0; +} + #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ /* @@ -1831,31 +2028,9 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent) profile->dents[AAFS_PROF_HASH] = dent; } -#ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY - if (profile->rawdata) { - if (aa_g_hash_policy) { - dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, - NULL, &rawdata_link_sha256_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_HASH] = dent; - } - dent = aafs_create("raw_abi", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, NULL, - &rawdata_link_abi_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_ABI] = dent; - - dent = aafs_create("raw_data", S_IFLNK | 0444, dir, - &profile->label.proxy->count, NULL, NULL, - &rawdata_link_data_iops); - if (IS_ERR(dent)) - goto fail; - profile->dents[AAFS_PROF_RAW_DATA] = dent; - } -#endif /*CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ + error = __aa_create_rawdata_symlink_dents(profile); + if (error) + goto fail2; list_for_each_entry(child, &profile->base.profiles, base.list) { error = __aafs_profile_mkdir(child, prof_child_dir(profile)); @@ -1922,7 +2097,7 @@ static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, mutex_unlock(&parent->lock); aa_put_ns(parent); - return ERR_PTR(error); + return error ? ERR_PTR(error) : NULL; } static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) @@ -2422,12 +2597,15 @@ static struct aa_sfs_entry aa_sfs_entry_versions[] = { static struct aa_sfs_entry aa_sfs_entry_policy[] = { AA_SFS_DIR("versions", aa_sfs_entry_versions), AA_SFS_FILE_BOOLEAN("set_load", 1), + AA_SFS_FILE_BOOLEAN("diff-encode", 1), /* number of out of band transitions supported */ AA_SFS_FILE_U64("outofband", MAX_OOB_SUPPORTED), AA_SFS_FILE_U64("permstable32_version", 3), AA_SFS_FILE_STRING("permstable32", PERMS32STR), AA_SFS_FILE_U64("state32", 1), AA_SFS_DIR("unconfined_restrictions", aa_sfs_entry_unconfined), + AA_SFS_FILE_BOOLEAN("compressed_load", 1), + AA_SFS_FILE_BOOLEAN("extended_policy_header", 1), { } }; diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index f02bf770f638..4154964cdac5 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "include/domain.h" #include "include/file.h" #include "include/ipc.h" +#include "include/lib.h" #include "include/match.h" #include "include/path.h" #include "include/policy.h" @@ -135,7 +137,7 @@ static int label_compound_match(struct aa_profile *profile, struct label_it i; struct path_cond cond = { }; - /* find first subcomponent that is in view and going to be interated with */ + /* find first subcomponent that is in view and going to be interacted with */ label_for_each(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; @@ -863,6 +865,15 @@ static int profile_onexec(const struct cred *subj_cred, } /* ensure none ns domain transitions are correctly applied with onexec */ +static struct aa_label *label_merge_wrap(struct aa_label *a, struct aa_label *b, + gfp_t gfp) +{ + struct aa_label *label = aa_label_merge(a, b, gfp); + + if (!label) + return ERR_PTR(-ENOMEM); + return label; +} static struct aa_label *handle_onexec(const struct cred *subj_cred, struct aa_label *label, @@ -890,12 +901,13 @@ static struct aa_label *handle_onexec(const struct cred *subj_cred, return ERR_PTR(error); new = fn_label_build_in_scope(label, profile, GFP_KERNEL, - stack ? aa_label_merge(&profile->label, onexec, - GFP_KERNEL) + stack ? label_merge_wrap(&profile->label, onexec, + GFP_KERNEL) : aa_get_newest_label(onexec), profile_transition(subj_cred, profile, bprm, buffer, cond, unsafe)); - if (new) + AA_BUG(!new); + if (!IS_ERR(new)) return new; /* TODO: get rid of GLOBAL_ROOT_UID */ @@ -904,7 +916,8 @@ static struct aa_label *handle_onexec(const struct cred *subj_cred, OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, bprm->filename, NULL, onexec, GLOBAL_ROOT_UID, - "failed to build target label", -ENOMEM)); + "failed to build target label", + PTR_ERR(new))); return ERR_PTR(error); } @@ -967,14 +980,10 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) profile_transition(subj_cred, profile, bprm, buffer, &cond, &unsafe)); - AA_BUG(!new); if (IS_ERR(new)) { error = PTR_ERR(new); goto done; - } else if (!new) { - error = -ENOMEM; - goto done; } /* Policy has specified a domain transitions. If no_new_privs and @@ -1109,6 +1118,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, int count, int flags) { struct aa_profile *profile, *root, *hat = NULL; + struct aa_ns *ns, *new_ns; struct aa_label *new; struct label_it it; bool sibling = false; @@ -1119,6 +1129,32 @@ static struct aa_label *change_hat(const struct cred *subj_cred, AA_BUG(!hats); AA_BUG(count < 1); + /* + * Acquire the newest label and then hold the lock until we choose a + * hat, so that profile replacement doesn't atomically truncate the + * list of potential hats. Because we are getting the namespaces from + * the profiles and label, we can rely on the namespaces being live + * and avoid incrementing their refcounts while grabbing the lock. + */ + label = aa_get_label(label); + ns = labels_ns(label); + +retry: + mutex_lock_nested(&ns->lock, ns->level); + if (label_is_stale(label)) { + new = aa_get_newest_label(label); + new_ns = labels_ns(new); + if (new_ns != ns) { + aa_put_label(new); + mutex_unlock(&ns->lock); + ns = new_ns; + label = new; + goto retry; + } + aa_put_label(label); + label = new; + } + if (PROFILE_IS_HAT(labels_profile(label))) sibling = true; @@ -1127,7 +1163,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, name = hats[i]; label_for_each_in_scope(it, labels_ns(label), label, profile) { if (sibling && PROFILE_IS_HAT(profile)) { - root = aa_get_profile_rcu(&profile->parent); + root = aa_get_profile(profile->parent); } else if (!sibling && !PROFILE_IS_HAT(profile)) { root = aa_get_profile(profile); } else { /* conflicting change type */ @@ -1187,6 +1223,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, GLOBAL_ROOT_UID, info, error); } } + mutex_unlock(&ns->lock); return ERR_PTR(error); build: @@ -1194,11 +1231,9 @@ static struct aa_label *change_hat(const struct cred *subj_cred, build_change_hat(subj_cred, profile, name, sibling), aa_get_label(&profile->label)); - if (!new) { - info = "label build failed"; - error = -ENOMEM; - goto fail; - } /* else if (IS_ERR) build_change_hat has logged error so return new */ + mutex_unlock(&ns->lock); + AA_BUG(!new); + /* return new label or error ptr */ return new; } @@ -1527,6 +1562,9 @@ int aa_change_profile(const char *fqname, int flags) new = fn_label_build_in_scope(label, profile, GFP_KERNEL, aa_get_label(target), aa_get_label(&profile->label)); + AA_BUG(!new); + if (IS_ERR(new)) + goto build_fail; /* * no new privs prevents domain transitions that would * reduce restrictions. @@ -1545,27 +1583,29 @@ int aa_change_profile(const char *fqname, int flags) /* only transition profiles in the current ns */ if (stack) new = aa_label_merge(label, target, GFP_KERNEL); - if (IS_ERR_OR_NULL(new)) { - info = "failed to build target label"; - if (!new) - error = -ENOMEM; - else - error = PTR_ERR(new); - new = NULL; - perms.allow = 0; - goto audit; - } + if (IS_ERR_OR_NULL(new)) + goto build_fail; error = aa_replace_current_label(new); } else { - if (new) { - aa_put_label(new); - new = NULL; - } + /* new will be recomputed so at exec time. So discard */ + aa_put_label(new); + new = NULL; /* full transition will be built in exec path */ aa_set_current_onexec(target, stack); } + goto audit; + +build_fail: + info = "failed to build target label"; + if (!new) + error = -ENOMEM; + else + error = PTR_ERR(new); + new = NULL; + perms.allow = 0; + audit: error = fn_for_each_in_scope(label, profile, aa_audit_file(subj_cred, diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 694e157149e8..c9d55fe1086f 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -157,10 +157,10 @@ static int path_name(const char *op, const struct cred *subj_cred, /* don't reaudit files closed during inheritance */ if (unlikely(path->dentry == aa_null.dentry)) - error = -EACCES; - else - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + return -EACCES; + + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -250,7 +250,7 @@ static int profile_path_perm(const char *op, const struct cred *subj_cred, struct path_cond *cond, int flags, struct aa_perms *perms) { - const char *name; + const char *name = NULL; int error; if (profile_unconfined(profile)) @@ -328,7 +328,7 @@ static int profile_path_link(const struct cred *subj_cred, struct path_cond *cond) { struct aa_ruleset *rules = profile->label.rules[0]; - const char *lname, *tname = NULL; + const char *lname = NULL, *tname = NULL; struct aa_perms lperms = {}, perms; const char *info = NULL; u32 request = AA_MAY_LINK; diff --git a/security/apparmor/include/apparmorfs.h b/security/apparmor/include/apparmorfs.h index dd580594dfb7..c6c8fcde728f 100644 --- a/security/apparmor/include/apparmorfs.h +++ b/security/apparmor/include/apparmorfs.h @@ -11,6 +11,9 @@ #ifndef __AA_APPARMORFS_H #define __AA_APPARMORFS_H +#include +#include + extern struct path aa_null; enum aa_sfs_type { @@ -120,6 +123,8 @@ struct aa_loaddata; #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY void __aa_fs_remove_rawdata(struct aa_loaddata *rawdata); int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata); +void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile); +int __aa_create_rawdata_symlink_dents(struct aa_profile *profile); #else static inline void __aa_fs_remove_rawdata(struct aa_loaddata *rawdata) { @@ -131,6 +136,16 @@ static inline int __aa_fs_create_rawdata(struct aa_ns *ns, { return 0; } + +static inline void __aa_remove_rawdata_symlink_dents(struct aa_profile *profile) +{ + /* empty stub */ +} + +static inline int __aa_create_rawdata_symlink_dents(struct aa_profile *profile) +{ + return 0; +} #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ #endif /* __AA_APPARMORFS_H */ diff --git a/security/apparmor/include/capability.h b/security/apparmor/include/capability.h index 1ddcec2d1160..3fb8b8f2182a 100644 --- a/security/apparmor/include/capability.h +++ b/security/apparmor/include/capability.h @@ -11,6 +11,7 @@ #ifndef __AA_CAPABILITY_H #define __AA_CAPABILITY_H +#include #include #include "apparmorfs.h" diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index 335f21930702..b5a722a47fd2 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -423,6 +423,38 @@ static inline struct aa_label *aa_get_newest_label(struct aa_label *l) return aa_get_label(l); } +/** + * aa_get_newest_label_condref - find the newest version of @l + * @l: the label to check for newer versions of + * @needput: returns whether the reference needs put + * + * Returns: refcounted newest version of @l taking into account + * replacement, renames and removals + * return @l. + */ +static inline struct aa_label *aa_get_newest_label_condref(struct aa_label *l, + bool *needput) +{ + if (l && unlikely(label_is_stale(l))) { + struct aa_label *tmp; + + AA_BUG(!l->proxy); + AA_BUG(!l->proxy->label); + /* BUG: only way this can happen is @l ref count and its + * replacement count have gone to 0 and are on their way + * to destruction. ie. we have a refcounting error + */ + tmp = aa_get_label_rcu(&l->proxy->label); + AA_BUG(!tmp); + + *needput = true; + return tmp; + } + + *needput = false; + return l; +} + static inline void aa_put_label(struct aa_label *l) { if (l) diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 8c6ce8484552..e3c8cb044a90 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -281,15 +281,15 @@ void aa_policy_destroy(struct aa_policy *policy); * @FN: fn to call for each profile transition. @P is set to the profile * * Returns: new label on success + * NULL if all callbacks decline to specify a transition * ERR_PTR if build @FN fails - * NULL if label_build fails due to low memory conditions * - * @FN must return a label or ERR_PTR on failure. NULL is not allowed + * @FN must return a label or ERR_PTR on failure. */ #define fn_label_build(L, P, GFP, FN) \ ({ \ __label__ __do_cleanup, __done; \ - struct aa_label *__new_; \ + struct aa_label *__new_ = NULL; \ \ if ((L)->size > 1) { \ /* TODO: add cache of transitions already done */ \ @@ -298,17 +298,21 @@ void aa_policy_destroy(struct aa_policy *policy); DEFINE_VEC(label, __lvec); \ DEFINE_VEC(profile, __pvec); \ if (vec_setup(label, __lvec, (L)->size, (GFP))) { \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ goto __done; \ } \ __j = 0; \ label_for_each(__i, (L), (P)) { \ __new_ = (FN); \ - AA_BUG(!__new_); \ + if (!__new_) \ + continue; \ if (IS_ERR(__new_)) \ goto __do_cleanup; \ __lvec[__j++] = __new_; \ } \ + if (__j == 0) \ + /* no components adding to build */ \ + goto __do_cleanup; \ for (__j = __count = 0; __j < (L)->size; __j++) \ __count += __lvec[__j]->size; \ if (!vec_setup(profile, __pvec, __count, (GFP))) { \ @@ -320,14 +324,13 @@ void aa_policy_destroy(struct aa_policy *policy); if (__count > 1) { \ __new_ = aa_vec_find_or_create_label(__pvec,\ __count, (GFP)); \ - /* only fails if out of Mem */ \ if (!__new_) \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ } else \ __new_ = aa_get_label(&__pvec[0]->label); \ vec_cleanup(profile, __pvec, __count); \ } else \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ __do_cleanup: \ vec_cleanup(label, __lvec, (L)->size); \ } else { \ @@ -335,7 +338,7 @@ __do_cleanup: \ __new_ = (FN); \ } \ __done: \ - if (!__new_) \ + if (PTR_ERR(__new_)) \ AA_DEBUG(DEBUG_LABEL, "label build failed\n"); \ (__new_); \ }) diff --git a/security/apparmor/include/path.h b/security/apparmor/include/path.h index 8bb915d48dc7..250812a77e11 100644 --- a/security/apparmor/include/path.h +++ b/security/apparmor/include/path.h @@ -11,6 +11,9 @@ #ifndef __AA_PATH_H #define __AA_PATH_H +#include +#include + enum path_flags { PATH_IS_DIR = 0x1, /* path is a directory */ PATH_SOCK_COND = 0x2, diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 3895f8774a3f..240b2eba7687 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -26,7 +26,6 @@ #include "file.h" #include "lib.h" #include "label.h" -#include "net.h" #include "perms.h" #include "resource.h" @@ -305,7 +304,8 @@ struct aa_profile *aa_fqlookupn_profile(struct aa_label *base, const char *fqname, size_t n); ssize_t aa_replace_profiles(struct aa_ns *view, struct aa_label *label, - u32 mask, struct aa_loaddata *udata); + u32 mask, struct aa_loaddata *udata, + char *compressed_profile, size_t compressed_size); ssize_t aa_remove_profiles(struct aa_ns *view, struct aa_label *label, char *name, size_t size); void __aa_profile_list_release(struct list_head *head); diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h index e5a95dc4da1f..c01f6885dbe3 100644 --- a/security/apparmor/include/policy_unpack.h +++ b/security/apparmor/include/policy_unpack.h @@ -16,6 +16,7 @@ #include #include +#include "lib.h" struct aa_load_ent { struct list_head list; @@ -128,10 +129,11 @@ struct aa_loaddata { char *data; }; -int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns); +int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns, + char *compressed_data, size_t compressed_size); /** - * aa_get_loaddata - get a reference count from a counted data reference + * aa_get_i_loaddata - get a reference count from a counted data reference * @data: reference to get a count on * * Returns: pointer to reference @@ -163,6 +165,25 @@ aa_get_profile_loaddata(struct aa_loaddata *data) return data; } +/** + * aa_get_profile_loaddata_not0 - get a profile reference count if not zero + * @data: reference to get a count on + * + * Like aa_get_profile_loaddata(), but safe to call on an entry that may + * be on a list (e.g. ns->rawdata_list) where the last pcount has already + * dropped and the deferred cleanup has not yet run. + * + * Returns: pointer to reference, or %NULL if @data is NULL or its + * profile refcount has already reached zero. + */ +static inline struct aa_loaddata * +aa_get_profile_loaddata_not0(struct aa_loaddata *data) +{ + if (data && kref_get_unless_zero(&data->pcount)) + return data; + return NULL; +} + void __aa_loaddata_update(struct aa_loaddata *data, long revision); bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r); void aa_loaddata_kref(struct kref *kref); diff --git a/security/apparmor/include/procattr.h b/security/apparmor/include/procattr.h index 03dbfdb2f2c0..56acd1bdb634 100644 --- a/security/apparmor/include/procattr.h +++ b/security/apparmor/include/procattr.h @@ -11,6 +11,8 @@ #ifndef __AA_PROCATTR_H #define __AA_PROCATTR_H +#include "label.h" + int aa_getprocattr(struct aa_label *label, char **string, bool newline); int aa_setprocattr_changehat(char *args, size_t size, int flags); diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index b1aaaf60fa8b..017d8b06b8f2 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -10,6 +10,11 @@ #ifndef __AA_TASK_H #define __AA_TASK_H +#include + +#include "audit.h" +#include "label.h" + static inline struct aa_task_ctx *task_ctx(struct task_struct *task) { return task->security + apparmor_blob_sizes.lbs_task; diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 3a721fdf1833..3fd384d8c41a 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -83,7 +83,7 @@ void __aa_proxy_redirect(struct aa_label *orig, struct aa_label *new) tmp = rcu_dereference_protected(orig->proxy->label, &labels_ns(orig)->lock); rcu_assign_pointer(orig->proxy->label, aa_get_label(new)); - orig->flags |= FLAG_STALE; + __label_make_stale(orig); aa_put_label(tmp); } @@ -458,7 +458,7 @@ struct aa_label *aa_label_alloc(int size, struct aa_proxy *proxy, gfp_t gfp) return new; fail: - kfree(new); + aa_label_free(new); return NULL; } @@ -1176,22 +1176,21 @@ static struct aa_label *__label_find_merge(struct aa_labelset *ls, struct aa_label *aa_label_find_merge(struct aa_label *a, struct aa_label *b) { struct aa_labelset *ls; - struct aa_label *label, *ar = NULL, *br = NULL; + struct aa_label *label; unsigned long flags; + bool a_needput, b_needput; AA_BUG(!a); AA_BUG(!b); - if (label_is_stale(a)) - a = ar = aa_get_newest_label(a); - if (label_is_stale(b)) - b = br = aa_get_newest_label(b); + a = aa_get_newest_label_condref(a, &a_needput); + b = aa_get_newest_label_condref(b, &b_needput); ls = labelset_of_merge(a, b); read_lock_irqsave(&ls->lock, flags); label = __label_find_merge(ls, a, b); read_unlock_irqrestore(&ls->lock, flags); - aa_put_label(ar); - aa_put_label(br); + aa_put_label_condref(a, a_needput); + aa_put_label_condref(b, b_needput); return label; } @@ -1228,9 +1227,10 @@ struct aa_label *aa_label_merge(struct aa_label *a, struct aa_label *b, if (!label) { struct aa_label *new; + bool a_needput, b_needput; - a = aa_get_newest_label(a); - b = aa_get_newest_label(b); + a = aa_get_newest_label_condref(a, &a_needput); + b = aa_get_newest_label_condref(b, &b_needput); /* could use label_merge_len(a, b), but requires double * comparison for small savings @@ -1242,8 +1242,8 @@ struct aa_label *aa_label_merge(struct aa_label *a, struct aa_label *b, label = label_merge_insert(new, a, b); label_free_or_put_new(label, new); out: - aa_put_label(a); - aa_put_label(b); + aa_put_label_condref(a, a_needput); + aa_put_label_condref(b, b_needput); } return label; diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 3491e9f60194..88d12e89d115 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -1422,7 +1422,21 @@ static int aa_sock_msg_perm(const char *op, u32 request, struct socket *sock, static int apparmor_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return aa_sock_msg_perm(OP_SENDMSG, AA_MAY_SEND, sock, msg, size); + int error = aa_sock_msg_perm(OP_SENDMSG, AA_MAY_SEND, sock, msg, size); + + if (error) + return error; + + /* TCP fast open carries connect() semantics in sendmsg(); mediate + * the implicit connect so it cannot bypass the connect permission. + */ + if ((msg->msg_flags & MSG_FASTOPEN) && msg->msg_name && + (sk_is_tcp(sock->sk) || + (sk_is_inet(sock->sk) && sock->sk->sk_type == SOCK_STREAM && + sock->sk->sk_protocol == IPPROTO_MPTCP))) + error = aa_sk_perm(OP_CONNECT, AA_MAY_CONNECT, sock->sk); + + return error; } static int apparmor_socket_recvmsg(struct socket *sock, @@ -1493,7 +1507,7 @@ static int apparmor_socket_shutdown(struct socket *sock, int how) * * Note: can not sleep may be called with locks held * - * dont want protocol specific in __skb_recv_datagram() + * don't want protocol specific in __skb_recv_datagram() * to deny an incoming connection socket_sock_rcv_skb() */ static int apparmor_socket_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) @@ -2129,7 +2143,7 @@ static int param_set_mode(const char *val, const struct kernel_param *kp) */ static void cache_hold_inc(unsigned int *hold) { - if (*hold > MAX_HOLD_COUNT) + if (*hold < MAX_HOLD_COUNT) (*hold)++; } diff --git a/security/apparmor/match.c b/security/apparmor/match.c index 3a2c6cf02b3c..d43ff34d705c 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -27,13 +27,13 @@ * @blob: data to unpack (NOT NULL) * @bsize: size of blob * - * Returns: pointer to table else NULL on failure + * Returns: pointer to table else ERR_PTR on failure * * NOTE: must be freed by kvfree (not kfree) */ static struct table_header *unpack_table(char *blob, size_t bsize) { - struct table_header *table = NULL; + struct table_header *table = ERR_PTR(-EPROTO); struct table_header th; size_t tsize; @@ -74,20 +74,21 @@ static struct table_header *unpack_table(char *blob, size_t bsize) else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, u32, __be32, get_unaligned_be32); - else - goto fail; + else { + kvfree(table); + table = ERR_PTR(-EPROTO); + goto out; + } /* if table was vmalloced make sure the page tables are synced * before it is used, as it goes live to all cpus. */ if (is_vmalloc_addr(table)) vm_unmap_aliases(); - } + } else + table = ERR_PTR(-ENOMEM); out: return table; -fail: - kvfree(table); - return NULL; } /** @@ -359,8 +360,11 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) while (size > 0) { table = unpack_table(data, size); - if (!table) + if (IS_ERR(table)) { + error = PTR_ERR(table); + table = NULL; goto fail; + } switch (table->td_id) { case YYTD_ID_ACCEPT: diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index 523570aa1a5a..2f5d918832c1 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -735,17 +735,11 @@ int aa_pivotroot(const struct cred *subj_cred, struct aa_label *label, build_pivotroot(subj_cred, profile, new_path, new_buffer, old_path, old_buffer)); - if (!target) { - info = "label build failed"; - error = -ENOMEM; - goto fail; - } else if (!IS_ERR(target)) { + AA_BUG(!target); + if (!IS_ERR(target)) { error = aa_replace_current_label(target); - if (error) { - /* TODO: audit target */ - aa_put_label(target); - goto out; - } + if (error) + goto fail; aa_put_label(target); } else /* already audited error */ @@ -763,7 +757,8 @@ int aa_pivotroot(const struct cred *subj_cred, struct aa_label *label, NULL /*new_name */, NULL /* old_name */, NULL, NULL, - 0, NULL, AA_MAY_PIVOTROOT, &nullperms, info, + 0, target->hname, AA_MAY_PIVOTROOT, &nullperms, info, error)); + aa_put_label(target); goto out; } diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 44c04102062f..cf590dd08540 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -22,12 +22,14 @@ struct aa_sfs_entry aa_sfs_entry_network[] = { AA_SFS_FILE_STRING("af_mask", AA_SFS_AF_MASK), + AA_SFS_FILE_BOOLEAN("tcp-fast-open", 1), { } }; struct aa_sfs_entry aa_sfs_entry_networkv9[] = { AA_SFS_FILE_STRING("af_mask", AA_SFS_AF_MASK), AA_SFS_FILE_BOOLEAN("af_unix", 1), + AA_SFS_FILE_BOOLEAN("tcp-fast-open", 1), { } }; @@ -354,6 +356,7 @@ static int apparmor_secmark_init(struct aa_secmark *secmark) return PTR_ERR(label); secmark->secid = label->secid; + aa_put_label(label); return 0; } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index b6a5eb4021db..f6f1b72d7c3d 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -81,6 +81,7 @@ #include "include/file.h" #include "include/ipc.h" #include "include/match.h" +#include "include/net.h" #include "include/path.h" #include "include/policy.h" #include "include/policy_ns.h" @@ -232,6 +233,13 @@ static void __remove_profile(struct aa_profile *profile) aa_label_remove(&profile->label); __aafs_profile_rmdir(profile); __list_remove_profile(profile); + /* rawdata is only ever referenced by fs lookup, that is no + * longer possible here, so put the reference to it. This will + * enable the rawdata to be freed if for some reason the profile + * is pinned and going to live for a while. + */ + aa_put_profile_loaddata(profile->rawdata); + profile->rawdata = NULL; } /** @@ -1151,6 +1159,8 @@ static struct aa_profile *update_to_newest_parent(struct aa_profile *new) * @label: label that is attempting to load/replace policy * @mask: permission mask * @udata: serialized data stream (NOT NULL) + * @compressed_profile: The userspace-provided compressed profile. May be NULL + * @compressed_size: If compressed_data is not NULL, the compressed data size * * unpack and replace a profile on the profile list and uses of that profile * by any task creds via invalidating the old version of the profile, which @@ -1160,7 +1170,8 @@ static struct aa_profile *update_to_newest_parent(struct aa_profile *new) * Returns: size of data consumed else error code on failure. */ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, - u32 mask, struct aa_loaddata *udata) + u32 mask, struct aa_loaddata *udata, + char *compressed_profile, size_t compressed_size) { const char *ns_name = NULL, *info = NULL; struct aa_ns *ns = NULL; @@ -1173,7 +1184,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, op = mask & AA_MAY_REPLACE_POLICY ? OP_PROF_REPL : OP_PROF_LOAD; aa_get_profile_loaddata(udata); /* released below */ - error = aa_unpack(udata, &lh, &ns_name); + error = aa_unpack(udata, &lh, &ns_name, compressed_profile, compressed_size); if (error) goto out; @@ -1223,8 +1234,12 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (aa_rawdata_eq(rawdata_ent, udata)) { struct aa_loaddata *tmp; - tmp = aa_get_profile_loaddata(rawdata_ent); - /* check we didn't fail the race */ + /* + * Entries remain on rawdata_list with + * pcount == 0 until do_ploaddata_rmfs() + * runs; only take a live profile ref. + */ + tmp = aa_get_profile_loaddata_not0(rawdata_ent); if (tmp) { aa_put_profile_loaddata(udata); udata = tmp; @@ -1342,6 +1357,16 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, goto skip; } + if (!aa_g_export_binary) { + if (ent->old && ent->old->rawdata && + ent->old->dents[AAFS_LOADDATA_DIR]) { + /* remove rawdata symlinks because the symlink + * target will be removed + */ + __aa_remove_rawdata_symlink_dents(ent->old); + } + } + /* * TODO: finer dedup based on profile range in data. Load set * can differ but profile may remain unchanged @@ -1352,6 +1377,11 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (ent->old) { share_name(ent->old, ent->new); __replace_profile(ent->old, ent->new); + if (aa_g_export_binary) { + /* recreate rawdata symlinks */ + if (!ent->old->rawdata) + __aa_create_rawdata_symlink_dents(ent->new); + } } else { struct list_head *lh; @@ -1372,12 +1402,15 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, out: aa_put_ns(ns); + + ssize_t udata_sz = udata->size; + aa_put_profile_loaddata(udata); kfree(ns_name); if (error) return error; - return udata->size; + return udata_sz; fail_lock: mutex_unlock(&ns->lock); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 9f45d5513d2c..5969d78f16af 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -25,6 +25,7 @@ #include "include/crypto.h" #include "include/file.h" #include "include/match.h" +#include "include/net.h" #include "include/path.h" #include "include/policy.h" #include "include/policy_unpack.h" @@ -1045,7 +1046,7 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, } /* accept2 is in some cases being allocated, even with perms */ - if (pdb->perms && !pdb->dfa->tables[YYTD_ID_ACCEPT2]) { + if (pdb->dfa && pdb->perms && !pdb->dfa->tables[YYTD_ID_ACCEPT2]) { /* add dfa flags table missing in v2 */ u32 noents = pdb->dfa->tables[YYTD_ID_ACCEPT]->td_lolen; u16 tdflags = pdb->dfa->tables[YYTD_ID_ACCEPT]->td_flags; @@ -1054,7 +1055,8 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, pdb->dfa->tables[YYTD_ID_ACCEPT2] = kvzalloc(tsize, GFP_KERNEL); if (!pdb->dfa->tables[YYTD_ID_ACCEPT2]) { *info = "failed to alloc dfa flags table"; - goto out; + error = -ENOMEM; + goto fail; } pdb->dfa->tables[YYTD_ID_ACCEPT2]->td_lolen = noents; pdb->dfa->tables[YYTD_ID_ACCEPT2]->td_flags = tdflags; @@ -1079,7 +1081,6 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, * - move free of unneeded trans table here, has to be done * after perm mapping. */ -out: *policy = pdb; return 0; @@ -1717,6 +1718,8 @@ static int compress_loaddata(struct aa_loaddata *data) * @udata: user data copied to kmem (NOT NULL) * @lh: list to place unpacked profiles in a aa_repl_ws * @ns: Returns namespace profile is in if specified else NULL (NOT NULL) + * @compressed_data: The userspace-provided compressed data. May be NULL + * @compressed_size: If compressed_data is not NULL, the compressed data size * * Unpack user data and return refcounted allocated profile(s) stored in * @lh in order of discovery, with the list chain stored in base.list @@ -1725,12 +1728,12 @@ static int compress_loaddata(struct aa_loaddata *data) * Returns: profile(s) on @lh else error pointer if fails to unpack */ int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, - const char **ns) + const char **ns, char *compressed_data, size_t compressed_size) { struct aa_load_ent *tmp, *ent; struct aa_profile *profile = NULL; char *ns_name = NULL; - int error; + int error = 0; struct aa_ext e = { .start = udata->data, .end = udata->data + udata->size, @@ -1783,10 +1786,23 @@ int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, } if (aa_g_export_binary) { - error = compress_loaddata(udata); + /* Do we have userspace-compressed data? */ + if (compressed_data) { + kvfree(udata->data); + udata->data = compressed_data; + udata->compressed_size = compressed_size; + compressed_data = NULL; /* consumed */ + + } else + error = compress_loaddata(udata); + if (error) goto fail; + } else if (compressed_data) { + kvfree(compressed_data); + compressed_data = NULL; } + return 0; fail_profile: @@ -1794,6 +1810,8 @@ int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, aa_put_profile(profile); fail: + if (compressed_data) + kvfree(compressed_data); list_for_each_entry_safe(ent, tmp, lh, list) { list_del_init(&ent->list); aa_load_ent_free(ent); diff --git a/security/apparmor/procattr.c b/security/apparmor/procattr.c index ce40f15d4952..c07b6e8fd9c9 100644 --- a/security/apparmor/procattr.c +++ b/security/apparmor/procattr.c @@ -54,6 +54,8 @@ int aa_getprocattr(struct aa_label *label, char **string, bool newline) FLAG_SHOW_MODE | FLAG_VIEW_SUBNS | FLAG_HIDDEN_UNCONFINED); if (len < 0) { + kfree(*string); + *string = NULL; aa_put_ns(current_ns); return len; } diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 0db0e81b4600..b9fb3738124e 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -314,7 +314,7 @@ static const char *get_current_exe_path(char *buffer, int buffer_size) path_get(&p); if (aa_path_name(&p, FLAG_VIEW_SUBNS, buffer, &path_str, NULL, NULL)) - return ERR_PTR(-ENOMEM); + path_str = ERR_PTR(-ENOMEM); fput(exe_file); path_put(&p); diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 40efde233f3a..7b98f5d1e2be 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c @@ -7,6 +7,8 @@ #include #include +bool bpf_lsm_initialized __ro_after_init; + static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = { #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ LSM_HOOK_INIT(NAME, bpf_lsm_##NAME), @@ -24,6 +26,7 @@ static int __init bpf_lsm_init(void) { security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks), &bpf_lsmid); + bpf_lsm_initialized = true; pr_info("LSM support for eBPF active\n"); return 0; } diff --git a/security/landlock/net.c b/security/landlock/net.c index cbff59ec3aba..46c17116fcf4 100644 --- a/security/landlock/net.c +++ b/security/landlock/net.c @@ -351,6 +351,14 @@ static int hook_socket_sendmsg(struct socket *const sock, access_mask_t access_request; int ret = 0; + if ((msg->msg_flags & MSG_FASTOPEN) && address && sk_is_tcp(sock->sk)) { + ret = current_check_access_socket( + sock, address, addrlen, LANDLOCK_ACCESS_NET_CONNECT_TCP, + true); + if (ret != 0) + return ret; + } + if (sk_is_udp(sock->sk)) access_request = LANDLOCK_ACCESS_NET_CONNECT_SEND_UDP; else diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 61f3c253d5c9..0437adf17428 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -35,8 +35,8 @@ struct landlock_layer { */ struct { /** - * @quiet: Suppresses denial logs for the object covered by this - * rule in this domain. For filesystem rules, this inherits + * @flags.quiet: Suppresses denial logs for the object covered by + * this rule in this domain. For filesystem rules, this inherits * down the file hierarchy. */ u8 quiet : 1; diff --git a/security/security.c b/security/security.c index 71aea8fdf014..2ee276ab15c5 100644 --- a/security/security.c +++ b/security/security.c @@ -3301,15 +3301,14 @@ int security_task_kill(struct task_struct *p, struct kernel_siginfo *info, } /** - * security_task_prctl() - Check if a prctl op is allowed + * security_task_prctl() - Handle an LSM specific prctl() call * @option: operation * @arg2: argument * @arg3: argument * @arg4: argument * @arg5: argument * - * Check permission before performing a process control operation on the - * current process. + * Handle lsm specific prctl() operations. * * Return: Return -ENOSYS if no-one wanted to handle this op, any other value * to cause prctl() to return immediately with that value. diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 813e82bcfc27..a9401d6c2e5f 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -497,7 +497,7 @@ static struct avc_node *avc_alloc_node(void) node = kmem_cache_zalloc(avc_node_cachep, GFP_NOWAIT); if (!node) - goto out; + return NULL; INIT_HLIST_NODE(&node->list); avc_cache_stats_incr(allocations); @@ -506,7 +506,6 @@ static struct avc_node *avc_alloc_node(void) selinux_avc.avc_cache_threshold) avc_reclaim_node(); -out: return node; } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 1a713d96206f..c70f65d86ab1 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1336,11 +1336,11 @@ static int selinux_genfs_get_sid(struct dentry *dentry, struct super_block *sb = dentry->d_sb; char *buffer, *path; - buffer = (char *)__get_free_page(GFP_KERNEL); + buffer = kmalloc(PATH_MAX, GFP_KERNEL); if (!buffer) return -ENOMEM; - path = dentry_path_raw(dentry, buffer, PAGE_SIZE); + path = dentry_path_raw(dentry, buffer, PATH_MAX); if (IS_ERR(path)) rc = PTR_ERR(path); else { @@ -1361,7 +1361,7 @@ static int selinux_genfs_get_sid(struct dentry *dentry, rc = 0; } } - free_page((unsigned long)buffer); + kfree(buffer); return rc; } @@ -4994,9 +4994,8 @@ static int selinux_socket_socketpair(struct socket *socka, Need to determine whether we should perform a name_bind permission check between the socket and the port number. */ -static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) +static int __selinux_socket_bind(struct sock *sk, struct sockaddr *address, int addrlen) { - struct sock *sk = sock->sk; struct sk_security_struct *sksec = selinux_sock(sk); u16 family; int err; @@ -5126,13 +5125,17 @@ static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, in return -EAFNOSUPPORT; } +static int selinux_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) +{ + return __selinux_socket_bind(sock->sk, address, addrlen); +} + /* This supports connect(2) and SCTP connect services such as sctp_connectx(3) * and sctp_sendmsg(3) as described in Documentation/security/SCTP.rst */ -static int selinux_socket_connect_helper(struct socket *sock, +static int selinux_socket_connect_helper(struct sock *sk, struct sockaddr *address, int addrlen) { - struct sock *sk = sock->sk; struct sk_security_struct *sksec = selinux_sock(sk); int err; @@ -5221,7 +5224,7 @@ static int selinux_socket_connect(struct socket *sock, int err; struct sock *sk = sock->sk; - err = selinux_socket_connect_helper(sock, address, addrlen); + err = selinux_socket_connect_helper(sk, address, addrlen); if (err) return err; @@ -5262,7 +5265,24 @@ static int selinux_socket_accept(struct socket *sock, struct socket *newsock) static int selinux_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return sock_has_perm(sock->sk, SOCKET__WRITE); + int rc; + struct sockaddr *const addr = msg->msg_name; + const int addrlen = msg->msg_namelen; + + rc = sock_has_perm(sock->sk, SOCKET__WRITE); + if (rc) + return rc; + + if (addr && (msg->msg_flags & MSG_FASTOPEN) && + (sk_is_tcp(sock->sk) || + (sk_is_inet(sock->sk) && sock->sk->sk_type == SOCK_STREAM && + sock->sk->sk_protocol == IPPROTO_MPTCP))) { + rc = selinux_socket_connect(sock, addr, addrlen); + if (rc) + return rc; + } + + return 0; } static int selinux_socket_recvmsg(struct socket *sock, struct msghdr *msg, @@ -5706,13 +5726,11 @@ static int selinux_sctp_bind_connect(struct sock *sk, int optname, int len, err = 0, walk_size = 0; void *addr_buf; struct sockaddr *addr; - struct socket *sock; if (!selinux_policycap_extsockclass()) return 0; /* Process one or more addresses that may be IPv4 or IPv6 */ - sock = sk->sk_socket; addr_buf = address; while (walk_size < addrlen) { @@ -5741,14 +5759,14 @@ static int selinux_sctp_bind_connect(struct sock *sk, int optname, case SCTP_PRIMARY_ADDR: case SCTP_SET_PEER_PRIMARY_ADDR: case SCTP_SOCKOPT_BINDX_ADD: - err = selinux_socket_bind(sock, addr, len); + err = __selinux_socket_bind(sk, addr, len); break; /* Connect checks */ case SCTP_SOCKOPT_CONNECTX: case SCTP_PARAM_SET_PRIMARY: case SCTP_PARAM_ADD_IP: case SCTP_SENDMSG_CONNECT: - err = selinux_socket_connect_helper(sock, addr, len); + err = selinux_socket_connect_helper(sk, addr, len); if (err) return err; diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c index 5aaaf69410bb..c7d91476971c 100644 --- a/security/selinux/selinuxfs.c +++ b/security/selinux/selinuxfs.c @@ -1984,17 +1984,15 @@ int __init init_sel_fs(void) return err; err = register_filesystem(&sel_fs_type); - if (err) { - sysfs_remove_mount_point(fs_kobj, "selinux"); - return err; - } + if (err) + goto err_remove_mount_point; selinux_null.mnt = kern_mount(&sel_fs_type); if (IS_ERR(selinux_null.mnt)) { pr_err("selinuxfs: could not mount!\n"); err = PTR_ERR(selinux_null.mnt); selinux_null.mnt = NULL; - return err; + goto err_unregister_fs; } selinux_null.dentry = try_lookup_noperm(&null_name, @@ -2003,7 +2001,7 @@ int __init init_sel_fs(void) pr_err("selinuxfs: could not lookup null!\n"); err = PTR_ERR(selinux_null.dentry); selinux_null.dentry = NULL; - return err; + goto err_unmount; } /* @@ -2012,5 +2010,14 @@ int __init init_sel_fs(void) */ (void) selinux_kernel_status_page(); + return 0; + +err_unmount: + kern_unmount(selinux_null.mnt); + selinux_null.mnt = NULL; +err_unregister_fs: + unregister_filesystem(&sel_fs_type); +err_remove_mount_point: + sysfs_remove_mount_point(fs_kobj, "selinux"); return err; } diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index ead504a639e3..ba1c9bd9e29f 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -604,10 +604,15 @@ static int type_index(void *key, void *datum, void *datap) typdatum = datum; p = datap; + if (!typdatum->value || typdatum->value > p->p_types.nprim || + typdatum->bounds > p->p_types.nprim) { + pr_err("SELinux: type %s had value %u bounds %u nprim %u\n", + (char *)key, typdatum->value, typdatum->bounds, + p->p_types.nprim); + return -EINVAL; + } + if (typdatum->primary) { - if (!typdatum->value || typdatum->value > p->p_types.nprim || - typdatum->bounds > p->p_types.nprim) - return -EINVAL; p->sym_val_to_name[SYM_TYPES][typdatum->value - 1] = key; p->type_val_to_struct[typdatum->value - 1] = typdatum; } diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 2d828548f3db..c03d3acc0a7c 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1355,8 +1355,8 @@ const char *security_get_initial_sid_context(u32 sid) } static int security_sid_to_context_core(u32 sid, char **scontext, - u32 *scontext_len, int force, - int only_invalid) + u32 *scontext_len, bool force, + bool only_invalid) { struct selinux_policy *policy; struct policydb *policydb; @@ -1439,14 +1439,14 @@ static int security_sid_to_context_core(u32 sid, char **scontext, int security_sid_to_context(u32 sid, char **scontext, u32 *scontext_len) { return security_sid_to_context_core(sid, scontext, - scontext_len, 0, 0); + scontext_len, false, false); } int security_sid_to_context_force(u32 sid, char **scontext, u32 *scontext_len) { return security_sid_to_context_core(sid, scontext, - scontext_len, 1, 0); + scontext_len, true, false); } /** @@ -1466,7 +1466,7 @@ int security_sid_to_context_inval(u32 sid, char **scontext, u32 *scontext_len) { return security_sid_to_context_core(sid, scontext, - scontext_len, 1, 1); + scontext_len, true, true); } /* @@ -1552,7 +1552,7 @@ static int string_to_context_struct(struct policydb *pol, static int security_context_to_sid_core(const char *scontext, u32 scontext_len, u32 *sid, u32 def_sid, gfp_t gfp_flags, - int force) + bool force) { struct selinux_policy *policy; struct policydb *policydb; @@ -1641,7 +1641,7 @@ int security_context_to_sid(const char *scontext, u32 scontext_len, u32 *sid, gfp_t gfp) { return security_context_to_sid_core(scontext, scontext_len, - sid, SECSID_NULL, gfp, 0); + sid, SECSID_NULL, gfp, false); } int security_context_str_to_sid(const char *scontext, u32 *sid, gfp_t gfp) @@ -1673,14 +1673,14 @@ int security_context_to_sid_default(const char *scontext, u32 scontext_len, u32 *sid, u32 def_sid, gfp_t gfp_flags) { return security_context_to_sid_core(scontext, scontext_len, - sid, def_sid, gfp_flags, 1); + sid, def_sid, gfp_flags, true); } int security_context_to_sid_force(const char *scontext, u32 scontext_len, u32 *sid) { return security_context_to_sid_core(scontext, scontext_len, - sid, SECSID_NULL, GFP_KERNEL, 1); + sid, SECSID_NULL, GFP_KERNEL, true); } static int compute_sid_handle_invalid_context( diff --git a/security/smack/smack.h b/security/smack/smack.h index 9b9eb262fe33..2a60a045b8cf 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -261,6 +261,7 @@ struct smack_audit_data { char *subject; char *object; char *request; + struct task_struct *subj_tsk; int result; }; @@ -317,8 +318,9 @@ int smack_populate_secattr(struct smack_known *skp); * Shared data. */ extern int smack_enabled __initdata; -extern int smack_cipso_direct; -extern int smack_cipso_mapped; +extern u8 smack_cipso_auto_level[2]; +#define smack_cipso_direct (+smack_cipso_auto_level[0]) +#define smack_cipso_mapped (+smack_cipso_auto_level[1]) extern struct smack_known *smack_net_ambient; extern struct smack_known *smack_syslog_label; #ifdef CONFIG_SECURITY_SMACK_BRINGUP diff --git a/security/smack/smack_access.c b/security/smack/smack_access.c index 350b88d582b3..fb85356266e5 100644 --- a/security/smack/smack_access.c +++ b/security/smack/smack_access.c @@ -331,6 +331,15 @@ static void smack_log_callback(struct audit_buffer *ab, void *a) audit_log_format(ab, " labels_differ"); else audit_log_format(ab, " requested=%s", sad->request); + + if (sad->subj_tsk) { + char comm[TASK_COMM_LEN]; + + audit_log_format(ab, " subj_pid=%d subj_comm=", + task_tgid_nr(sad->subj_tsk)); + audit_log_untrustedstring(ab, + get_task_comm(comm, sad->subj_tsk)); + } } /** diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index ff115068c5c0..df2693f11754 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -130,12 +130,13 @@ static int smk_bu_note(char *note, struct smack_known *sskp, #define smk_bu_note(note, sskp, oskp, mode, RC) (RC) #endif -#ifdef CONFIG_SECURITY_SMACK_BRINGUP -static int smk_bu_current(char *note, struct smack_known *oskp, - int mode, int rc) +static int +smk_bu_tsk_to_obj(struct task_struct *tsk, const struct task_smack *tsp, + char *note, struct smack_known *oskp, int mode, int rc) { - struct task_smack *tsp = smack_cred(current_cred()); +#ifdef CONFIG_SECURITY_SMACK_BRINGUP char acc[SMK_NUM_ACCESS_TYPE + 1]; + char comm[TASK_COMM_LEN]; if (rc <= 0) return rc; @@ -143,14 +144,22 @@ static int smk_bu_current(char *note, struct smack_known *oskp, rc = 0; smk_bu_mode(mode, acc); + pr_info("Smack %s: (%s %s %s) %s %s\n", smk_bu_mess[rc], - tsp->smk_task->smk_known, oskp->smk_known, - acc, current->comm, note); + smk_of_task(tsp)->smk_known, oskp->smk_known, + acc, get_task_comm(comm, tsk), note); return 0; -} #else -#define smk_bu_current(note, oskp, mode, RC) (RC) + return rc; #endif +} + +static int smk_bu_current(char *note, struct smack_known *oskp, + int mode, int rc) +{ + return smk_bu_tsk_to_obj(current, smack_cred(current_cred()), + note, oskp, mode, rc); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP static int smk_bu_task(struct task_struct *otp, int mode, int rc) @@ -1312,7 +1321,7 @@ static int smack_inode_getattr(const struct path *path) */ static int smack_inode_xattr_skipcap(const char *name) { - if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX))) + if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0) return 0; if (strcmp(name, XATTR_NAME_SMACK) == 0 || @@ -3348,14 +3357,20 @@ static int smack_sem_semop(struct kern_ipc_perm *isp, struct sembuf *sops, } /** - * smk_curacc_msq : helper to check if current has access on msq - * @isp : the msq + * smk_tskacc_msq : helper to check if tsk has access on msq + * @tsk: the task that requests access + * @isp : the sysv msg queue permissions * @access : access requested * - * return 0 if current has access, error otherwise + * return 0 if tsk has access, error otherwise */ -static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +static int +smk_tskacc_msq(struct task_struct *tsk, struct kern_ipc_perm *isp, int access) { + const bool tsk_is_current = (tsk == current); + const struct cred * const tsk_cred = + (tsk_is_current ? current_cred() : get_task_cred(tsk)); + struct task_smack * const tsp = smack_cred(tsk_cred); struct smack_known *msp = smack_of_ipc(isp); struct smk_audit_info ad; int rc; @@ -3363,12 +3378,28 @@ static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) #ifdef CONFIG_AUDIT smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_IPC); ad.a.u.ipc_id = isp->id; + if (!tsk_is_current) + ad.sad.subj_tsk = tsk; #endif - rc = smk_curacc(msp, access, &ad); - rc = smk_bu_current("msq", msp, access, rc); + rc = smk_tskacc(tsp, msp, access, &ad); + rc = smk_bu_tsk_to_obj(tsk, tsp, "msq", msp, access, rc); + if (!tsk_is_current) + put_cred(tsk_cred); return rc; } +/** + * smk_curacc_msq : helper to check if current has access on msq + * @isp : the sysv msg queue permissions + * @access : access requested + * + * return 0 if current has access, error otherwise + */ +static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +{ + return smk_tskacc_msq(current, isp, access); +} + /** * smack_msg_queue_associate - Smack access check for msg_queue * @isp: the object @@ -3436,21 +3467,21 @@ static int smack_msg_queue_msgsnd(struct kern_ipc_perm *isp, struct msg_msg *msg } /** - * smack_msg_queue_msgrcv - Smack access check for msg_queue + * smack_msg_queue_msgrcv - check it target has r/w access to msg_queue * @isp: the object * @msg: unused - * @target: unused + * @target: the task that msgrcv() from the queue * @type: unused * @mode: unused * - * Returns 0 if current has read and write access, error code otherwise + * Returns 0 if target has read and write access, error code otherwise */ static int smack_msg_queue_msgrcv(struct kern_ipc_perm *isp, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return smk_curacc_msq(isp, MAY_READWRITE); + return smk_tskacc_msq(target, isp, MAY_READWRITE); } /** diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index 6e62dcb36f74..a72bc7fabea9 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -83,18 +83,27 @@ static DEFINE_MUTEX(smk_net6addr_lock); struct smack_known *smack_net_ambient; /* - * This is the level in a CIPSO header that indicates a + * Sensitivity levels for automatically created CIPSO labels. + * See smack_access.c`smack_populate_secattr() + * + * [0] "direct" labeling, label length < SMK_CIPSOLEN(24): * smack label is contained directly in the category set. * It can be reset via smackfs/direct - */ -int smack_cipso_direct = SMACK_CIPSO_DIRECT_DEFAULT; - -/* - * This is the level in a CIPSO header that indicates a + * + * [1] "mapped" labeling, label length >= SMK_CIPSOLEN(24): * secid is contained directly in the category set. * It can be reset via smackfs/mapped */ -int smack_cipso_mapped = SMACK_CIPSO_MAPPED_DEFAULT; +u8 smack_cipso_auto_level[2] = { + SMACK_CIPSO_DIRECT_DEFAULT, + SMACK_CIPSO_MAPPED_DEFAULT, +}; + +static int +smk_cipso_auto_level_idx(const struct file *file) +{ + return (file_inode(file)->i_ino != SMK_DIRECT); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP /* @@ -115,7 +124,7 @@ struct smack_known *smack_syslog_label; /* * Ptrace current rule * SMACK_PTRACE_DEFAULT regular smack ptrace rules (/proc based) - * SMACK_PTRACE_EXACT labels must match, but can be overriden with + * SMACK_PTRACE_EXACT labels must match, but can be overridden with * CAP_SYS_PTRACE * SMACK_PTRACE_DRACONIAN labels must match, CAP_SYS_PTRACE has no effect */ @@ -1598,24 +1607,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - unsigned long u; + int ret; + u32 u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; + ret = kstrtou32_from_user(buf, count, 10, &u); + if (unlikely(ret)) + return ret; - temp[count] = '\0'; - - if (kstrtoul(temp, 10, &u)) - return -EINVAL; - - if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + if (u == CIPSO_V4_DOI_UNKNOWN) return -EINVAL; return smk_cipso_doi(u, GFP_KERNEL) ? : count; @@ -1628,158 +1630,80 @@ static const struct file_operations smk_doi_ops = { }; /** - * smk_read_direct - read() for /smack/direct - * @filp: file pointer, not actually used + * smk_read_cipso_auto_level - read() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to put the result * @count: maximum to send along * @ppos: where to start * * Returns number of bytes read or error code, as appropriate */ -static ssize_t smk_read_direct(struct file *filp, char __user *buf, +static ssize_t smk_read_cipso_auto_level(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - ssize_t rc; + char temp[sizeof "255"]; + int n; if (*ppos != 0) return 0; - sprintf(temp, "%d", smack_cipso_direct); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; + n = sprintf(temp, "%u", (unsigned int)smack_cipso_auto_level[ + smk_cipso_auto_level_idx(filp)]); + return simple_read_from_buffer(buf, count, ppos, temp, n); } /** - * smk_write_direct - write() for /smack/direct - * @file: file pointer, not actually used + * smk_write_cipso_auto_level - write() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to get the data from * @count: bytes sent * @ppos: where to start * * Returns number of bytes written or error code, as appropriate */ -static ssize_t smk_write_direct(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t +smk_write_cipso_auto_level(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) { - struct smack_known *skp; - char temp[80]; - int i; + int ret, idx; + u8 i, old_lvl; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; - /* - * Don't do anything if the value hasn't actually changed. - * If it is changing reset the level on entries that were - * set up to be direct when they were created. + * draft-ietf-cipso-ipsecurity-01 (CIPSO 2.2), 3.4.2.4: + * "Sensitivity Level is 1 octet in length. Its value is from 0 to 255" */ - if (smack_cipso_direct != i) { - mutex_lock(&smack_known_lock); - list_for_each_entry_rcu(skp, &smack_known_list, list) - if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_direct) - skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_direct = i; - mutex_unlock(&smack_known_lock); - } - - return count; -} - -static const struct file_operations smk_direct_ops = { - .read = smk_read_direct, - .write = smk_write_direct, - .llseek = default_llseek, -}; - -/** - * smk_read_mapped - read() for /smack/mapped - * @filp: file pointer, not actually used - * @buf: where to put the result - * @count: maximum to send along - * @ppos: where to start - * - * Returns number of bytes read or error code, as appropriate - */ -static ssize_t smk_read_mapped(struct file *filp, char __user *buf, - size_t count, loff_t *ppos) -{ - char temp[80]; - ssize_t rc; - - if (*ppos != 0) - return 0; - - sprintf(temp, "%d", smack_cipso_mapped); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; -} - -/** - * smk_write_mapped - write() for /smack/mapped - * @file: file pointer, not actually used - * @buf: where to get the data from - * @count: bytes sent - * @ppos: where to start - * - * Returns number of bytes written or error code, as appropriate - */ -static ssize_t smk_write_mapped(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct smack_known *skp; - char temp[80]; - int i; - - if (!smack_privileged(CAP_MAC_ADMIN)) - return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; + ret = kstrtou8_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; /* * Don't do anything if the value hasn't actually changed. * If it is changing reset the level on entries that were - * set up to be mapped when they were created. + * set up to be "auto" level when they were created. */ - if (smack_cipso_mapped != i) { + idx = smk_cipso_auto_level_idx(filp); + old_lvl = smack_cipso_auto_level[idx]; + + if (old_lvl != i) { + struct smack_known *skp; mutex_lock(&smack_known_lock); list_for_each_entry_rcu(skp, &smack_known_list, list) if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_mapped) + old_lvl) skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_mapped = i; + smack_cipso_auto_level[idx] = i; mutex_unlock(&smack_known_lock); } return count; } -static const struct file_operations smk_mapped_ops = { - .read = smk_read_mapped, - .write = smk_write_mapped, +static const struct file_operations +smk_cipso_auto_level_ops = { + .read = smk_read_cipso_auto_level, + .write = smk_write_cipso_auto_level, .llseek = default_llseek, }; @@ -2179,22 +2103,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf, static ssize_t smk_write_logging(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < 0 || i > 3) return -EINVAL; log_policy = i; @@ -2838,22 +2755,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf, static ssize_t smk_write_ptrace(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (*ppos != 0 || count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX) return -EINVAL; smack_ptrace_rule = i; @@ -2888,7 +2798,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_DOI] = { "doi", &smk_doi_ops, S_IRUGO|S_IWUSR}, [SMK_DIRECT] = { - "direct", &smk_direct_ops, S_IRUGO|S_IWUSR}, + "direct", &smk_cipso_auto_level_ops, 0644}, [SMK_AMBIENT] = { "ambient", &smk_ambient_ops, S_IRUGO|S_IWUSR}, [SMK_NET4ADDR] = { @@ -2904,7 +2814,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_ACCESSES] = { "access", &smk_access_ops, S_IRUGO|S_IWUGO}, [SMK_MAPPED] = { - "mapped", &smk_mapped_ops, S_IRUGO|S_IWUSR}, + "mapped", &smk_cipso_auto_level_ops, 0644}, [SMK_LOAD2] = { "load2", &smk_load2_ops, S_IRUGO|S_IWUSR}, [SMK_LOAD_SELF2] = { diff --git a/sound/atmel/ac97c.c b/sound/atmel/ac97c.c index df0a049192de..e394205f469b 100644 --- a/sound/atmel/ac97c.c +++ b/sound/atmel/ac97c.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index fd63d219bf86..ea699491f0c3 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -1083,15 +1083,18 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ file descriptors are allocated before fd_install() */ if (!task->input || !task->input->file || !task->output || !task->output->file) { retval = -EINVAL; - goto cleanup; + goto free_driver_task; } fd_i = get_unused_fd_flags(O_WRONLY|O_CLOEXEC); - if (fd_i < 0) - goto cleanup; + if (fd_i < 0) { + retval = fd_i; + goto free_driver_task; + } fd_o = get_unused_fd_flags(O_RDONLY|O_CLOEXEC); if (fd_o < 0) { + retval = fd_o; put_unused_fd(fd_i); - goto cleanup; + goto free_driver_task; } /* keep dmabuf reference until freed with task free ioctl */ get_dma_buf(task->input); @@ -1103,6 +1106,8 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_ list_add_tail(&task->list, &stream->runtime->tasks); stream->runtime->total_tasks++; return 0; +free_driver_task: + stream->ops->task_free(stream, task); cleanup: snd_compr_task_free(task); return retval; diff --git a/sound/core/control.c b/sound/core/control.c index 7a8dc506221e..037e455bb11b 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -55,9 +55,7 @@ static int snd_ctl_open(struct inode *inode, struct file *file) struct snd_ctl_file *ctl; int i, err; - err = stream_open(inode, file); - if (err < 0) - return err; + stream_open(inode, file); card = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_CONTROL); if (!card) { diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c index 591cff800329..f5bcc7896542 100644 --- a/sound/core/oss/mixer_oss.c +++ b/sound/core/oss/mixer_oss.c @@ -30,9 +30,7 @@ static int snd_mixer_oss_open(struct inode *inode, struct file *file) struct snd_mixer_oss_file *fmixer; int err; - err = nonseekable_open(inode, file); - if (err < 0) - return err; + nonseekable_open(inode, file); card = snd_lookup_oss_minor_data(iminor(inode), SNDRV_OSS_DEVICE_TYPE_MIXER); diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index 9826d9a0be6d..0d7818eb3e14 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -2495,9 +2495,7 @@ static int snd_pcm_oss_open(struct inode *inode, struct file *file) int nonblock; wait_queue_entry_t wait; - err = nonseekable_open(inode, file); - if (err < 0) - return err; + nonseekable_open(inode, file); pcm = snd_lookup_oss_minor_data(iminor(inode), SNDRV_OSS_DEVICE_TYPE_PCM); diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index fe597f7d522d..4d665b4148d7 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -545,7 +545,7 @@ void snd_pcm_set_sync_per_card(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, const unsigned char *id, unsigned int len) { - *(__u32 *)params->sync = cpu_to_le32(substream->pcm->card->number); + *(__le32 *)params->sync = cpu_to_le32(substream->pcm->card->number); len = min(12, len); memcpy(params->sync + 4, id, len); memset(params->sync + 4 + len, 0, 12 - len); diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 7dc0060617f1..3462cd5275a2 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -2873,9 +2873,10 @@ static int snd_pcm_open_file(struct file *file, static int snd_pcm_playback_open(struct inode *inode, struct file *file) { struct snd_pcm *pcm; - int err = nonseekable_open(inode, file); - if (err < 0) - return err; + int err; + + nonseekable_open(inode, file); + pcm = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_PCM_PLAYBACK); err = snd_pcm_open(file, pcm, SNDRV_PCM_STREAM_PLAYBACK); @@ -2887,9 +2888,10 @@ static int snd_pcm_playback_open(struct inode *inode, struct file *file) static int snd_pcm_capture_open(struct inode *inode, struct file *file) { struct snd_pcm *pcm; - int err = nonseekable_open(inode, file); - if (err < 0) - return err; + int err; + + nonseekable_open(inode, file); + pcm = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_PCM_CAPTURE); err = snd_pcm_open(file, pcm, SNDRV_PCM_STREAM_CAPTURE); @@ -3907,6 +3909,8 @@ static vm_fault_t snd_pcm_mmap_data_fault(struct vm_fault *vmf) if (substream == NULL) return VM_FAULT_SIGBUS; runtime = substream->runtime; + if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED) + return VM_FAULT_SIGBUS; offset = vmf->pgoff << PAGE_SHIFT; dma_bytes = PAGE_ALIGN(runtime->dma_bytes); if (offset > dma_bytes - PAGE_SIZE) diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c index 4dfd9d53e6d3..789254a88e53 100644 --- a/sound/core/rawmidi.c +++ b/sound/core/rawmidi.c @@ -441,9 +441,7 @@ static int snd_rawmidi_open(struct inode *inode, struct file *file) if ((file->f_flags & O_APPEND) && !(file->f_flags & O_NONBLOCK)) return -EINVAL; /* invalid combination */ - err = stream_open(inode, file); - if (err < 0) - return err; + stream_open(inode, file); if (maj == snd_major) { rmidi = snd_lookup_minor_data(iminor(inode), diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 28782e1776fa..8fe872367568 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -296,11 +296,8 @@ static int snd_seq_open(struct inode *inode, struct file *file) int c, mode; /* client id */ struct snd_seq_client *client; struct snd_seq_user_client *user; - int err; - err = stream_open(inode, file); - if (err < 0) - return err; + stream_open(inode, file); scoped_guard(mutex, ®ister_mutex) { client = seq_create_client1(-1, SNDRV_SEQ_DEFAULT_EVENTS); diff --git a/sound/core/seq/seq_memory.c b/sound/core/seq/seq_memory.c index ca9f6db0022c..209b08c2a940 100644 --- a/sound/core/seq/seq_memory.c +++ b/sound/core/seq/seq_memory.c @@ -364,7 +364,7 @@ int snd_seq_event_dup(struct snd_seq_pool *pool, struct snd_seq_event *event, size = snd_seq_event_packet_size(event); memcpy(&cell->ump, event, size); #if IS_ENABLED(CONFIG_SND_SEQ_UMP) - if (size < sizeof(cell->event)) + if (size < sizeof(cell->ump)) cell->ump.raw.extra = 0; #endif diff --git a/sound/core/sound.c b/sound/core/sound.c index 8d05fe0d263b..3e1969a52b21 100644 --- a/sound/core/sound.c +++ b/sound/core/sound.c @@ -395,15 +395,21 @@ int __init snd_minor_info_init(void) static int __init alsa_sound_init(void) { + int err; + snd_major = major; snd_ecards_limit = cards_limit; - if (register_chrdev(major, "alsa", &snd_fops)) { + + err = register_chrdev(major, "alsa", &snd_fops); + if (err < 0) { pr_err("ALSA core: unable to register native major device number %d\n", major); - return -EIO; + return err; } - if (snd_info_init() < 0) { + + err = snd_info_init(); + if (err < 0) { unregister_chrdev(major, "alsa"); - return -ENOMEM; + return err; } #ifdef CONFIG_SND_DEBUG diff --git a/sound/core/timer.c b/sound/core/timer.c index 51c6ac4df9f4..937d7996f7ab 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1531,11 +1531,8 @@ static int realloc_user_queue(struct snd_timer_user *tu, int size) static int snd_timer_user_open(struct inode *inode, struct file *file) { struct snd_timer_user *tu; - int err; - err = stream_open(inode, file); - if (err < 0) - return err; + stream_open(inode, file); tu = kzalloc_obj(*tu); if (tu == NULL) diff --git a/sound/firewire/dice/dice.c b/sound/firewire/dice/dice.c index f7a50bae4b55..58f14aadc73d 100644 --- a/sound/firewire/dice/dice.c +++ b/sound/firewire/dice/dice.c @@ -148,7 +148,7 @@ static int dice_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent snd_dice_detect_formats_t detect_formats; int err; - if (!entry->driver_data && entry->vendor_id != OUI_SSL) { + if (!entry->driver_data_ptr && entry->vendor_id != OUI_SSL) { err = check_dice_category(unit); if (err < 0) return -ENODEV; @@ -164,10 +164,10 @@ static int dice_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent dev_set_drvdata(&unit->device, dice); dice->card = card; - if (!entry->driver_data) + if (!entry->driver_data_ptr) detect_formats = snd_dice_stream_detect_current_formats; else - detect_formats = (snd_dice_detect_formats_t)entry->driver_data; + detect_formats = entry->driver_data_ptr; // Below models are compliant to IEC 61883-1/6 and have no quirk at high sampling transfer // frequency. @@ -255,7 +255,7 @@ static void dice_bus_reset(struct fw_unit *unit) .model_id = (model), \ .specifier_id = (vendor), \ .version = DICE_INTERFACE, \ - .driver_data = (kernel_ulong_t)(data), \ + .driver_data_ptr = (data), \ } static const struct ieee1394_device_id dice_id_table[] = { @@ -267,7 +267,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_MAUDIO, .model_id = 0x000010, - .driver_data = (kernel_ulong_t)snd_dice_detect_extension_formats, + .driver_data_ptr = snd_dice_detect_extension_formats, }, /* M-Audio Profire 610 has a different value in version field. */ { @@ -275,7 +275,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_MAUDIO, .model_id = 0x000011, - .driver_data = (kernel_ulong_t)snd_dice_detect_extension_formats, + .driver_data_ptr = snd_dice_detect_extension_formats, }, /* TC Electronic Konnekt 24D. */ { @@ -283,7 +283,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000020, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Konnekt 8. */ { @@ -291,7 +291,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000021, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Studio Konnekt 48. */ { @@ -299,7 +299,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000022, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Konnekt Live. */ { @@ -307,7 +307,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000023, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Desktop Konnekt 6. */ { @@ -315,7 +315,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000024, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Impact Twin. */ { @@ -323,7 +323,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000027, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* TC Electronic Digital Konnekt x32. */ { @@ -331,7 +331,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_TCELECTRONIC, .model_id = 0x000030, - .driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats, + .driver_data_ptr = snd_dice_detect_tcelectronic_formats, }, /* Alesis iO14/iO26. */ { @@ -339,7 +339,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_ALESIS, .model_id = MODEL_ALESIS_IO_BOTH, - .driver_data = (kernel_ulong_t)snd_dice_detect_alesis_formats, + .driver_data_ptr = snd_dice_detect_alesis_formats, }, // Alesis MasterControl. { @@ -347,7 +347,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_ALESIS, .model_id = 0x000002, - .driver_data = (kernel_ulong_t)snd_dice_detect_alesis_mastercontrol_formats, + .driver_data_ptr = snd_dice_detect_alesis_mastercontrol_formats, }, /* Mytek Stereo 192 DSD-DAC. */ { @@ -355,7 +355,7 @@ static const struct ieee1394_device_id dice_id_table[] = { IEEE1394_MATCH_MODEL_ID, .vendor_id = OUI_MYTEK, .model_id = 0x000002, - .driver_data = (kernel_ulong_t)snd_dice_detect_mytek_formats, + .driver_data_ptr = snd_dice_detect_mytek_formats, }, // Solid State Logic, Duende Classic and Mini. // NOTE: each field of GUID in config ROM is not compliant to standard @@ -469,7 +469,7 @@ static const struct ieee1394_device_id dice_id_table[] = { .model_id = OUI_TEAC, .specifier_id = OUI_TEAC, .version = 0x800006, - .driver_data = (kernel_ulong_t)snd_dice_detect_teac_formats, + .driver_data_ptr = snd_dice_detect_teac_formats, }, { } }; diff --git a/sound/firewire/fireface/ff.c b/sound/firewire/fireface/ff.c index 5d2c4fbf4434..13472822d2be 100644 --- a/sound/firewire/fireface/ff.c +++ b/sound/firewire/fireface/ff.c @@ -70,7 +70,7 @@ static int snd_ff_probe(struct fw_unit *unit, const struct ieee1394_device_id *e init_waitqueue_head(&ff->hwdep_wait); ff->unit_version = entry->version; - ff->spec = (const struct snd_ff_spec *)entry->driver_data; + ff->spec = entry->driver_data_ptr; err = snd_ff_transaction_register(ff); if (err < 0) @@ -186,7 +186,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = { .specifier_id = OUI_RME, .version = SND_FF_UNIT_VERSION_FF800, .model_id = 0x101800, - .driver_data = (kernel_ulong_t)&spec_ff800, + .driver_data_ptr = &spec_ff800, }, /* Fireface 400 */ { @@ -198,7 +198,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = { .specifier_id = OUI_RME, .version = SND_FF_UNIT_VERSION_FF400, .model_id = 0x101800, - .driver_data = (kernel_ulong_t)&spec_ff400, + .driver_data_ptr = &spec_ff400, }, // Fireface UFX. { @@ -210,7 +210,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = { .specifier_id = OUI_RME, .version = SND_FF_UNIT_VERSION_UFX, .model_id = 0x101800, - .driver_data = (kernel_ulong_t)&spec_ufx_802, + .driver_data_ptr = &spec_ufx_802, }, // Fireface UCX. { @@ -222,7 +222,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = { .specifier_id = OUI_RME, .version = SND_FF_UNIT_VERSION_UCX, .model_id = 0x101800, - .driver_data = (kernel_ulong_t)&spec_ucx, + .driver_data_ptr = &spec_ucx, }, // Fireface 802. { @@ -234,7 +234,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = { .specifier_id = OUI_RME, .version = SND_FF_UNIT_VERSION_802, .model_id = 0x101800, - .driver_data = (kernel_ulong_t)&spec_ufx_802, + .driver_data_ptr = &spec_ufx_802, }, {} }; diff --git a/sound/firewire/isight.c b/sound/firewire/isight.c index 2b7f071d593b..f16e2e223494 100644 --- a/sound/firewire/isight.c +++ b/sound/firewire/isight.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -179,7 +178,8 @@ static void isight_packet(struct fw_iso_context *context, u32 cycle, if (likely(length >= 16 && payload->signature == cpu_to_be32(0x73676874/*"sght"*/))) { count = be32_to_cpu(payload->sample_count); - if (likely(count <= (length - 16) / 4)) { + if (likely(count <= (length - 16) / 4 && + count <= MAX_FRAMES_PER_PACKET)) { total = be32_to_cpu(payload->sample_total); if (unlikely(total != isight->total_samples)) { if (!isight->first_packet) diff --git a/sound/firewire/motu/motu.c b/sound/firewire/motu/motu.c index fd2a9dddbfa6..1fec6c8cdf6c 100644 --- a/sound/firewire/motu/motu.c +++ b/sound/firewire/motu/motu.c @@ -78,7 +78,7 @@ static int motu_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent dev_set_drvdata(&unit->device, motu); motu->card = card; - motu->spec = (const struct snd_motu_spec *)entry->driver_data; + motu->spec = entry->driver_data_ptr; mutex_init(&motu->mutex); spin_lock_init(&motu->lock); init_waitqueue_head(&motu->hwdep_wait); @@ -148,7 +148,7 @@ static void motu_bus_update(struct fw_unit *unit) snd_motu_transaction_reregister(motu); } -#define SND_MOTU_DEV_ENTRY(model, data) \ +#define SND_MOTU_DEV_ENTRY(model, data_ptr) \ { \ .match_flags = IEEE1394_MATCH_VENDOR_ID | \ IEEE1394_MATCH_SPECIFIER_ID | \ @@ -156,7 +156,7 @@ static void motu_bus_update(struct fw_unit *unit) .vendor_id = OUI_MOTU, \ .specifier_id = OUI_MOTU, \ .version = model, \ - .driver_data = (kernel_ulong_t)data, \ + .driver_data_ptr = data_ptr, \ } static const struct ieee1394_device_id motu_id_table[] = { diff --git a/sound/firewire/oxfw/oxfw.c b/sound/firewire/oxfw/oxfw.c index 5039bd79b18e..38a3c3b150df 100644 --- a/sound/firewire/oxfw/oxfw.c +++ b/sound/firewire/oxfw/oxfw.c @@ -95,7 +95,7 @@ static int name_card(struct snd_oxfw *oxfw, const struct ieee1394_device_id *ent /* to apply card definitions */ if (entry->vendor_id == VENDOR_GRIFFIN || entry->vendor_id == VENDOR_LACIE) { - info = (const struct compat_info *)entry->driver_data; + info = entry->driver_data_ptr; d = info->driver_name; v = info->vendor_name; m = info->model_name; @@ -321,7 +321,7 @@ static const struct compat_info lacie_speakers = { .model_id = model, \ .specifier_id = SPECIFIER_1394TA, \ .version = VERSION_AVC, \ - .driver_data = (kernel_ulong_t)data, \ + .driver_data_ptr = data, \ } static const struct ieee1394_device_id oxfw_id_table[] = { diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index e3b6aaabe3a9..3d92262763f6 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -291,6 +291,7 @@ enum { CXT_FIXUP_HEADSET_MIC, CXT_FIXUP_HP_MIC_NO_PRESENCE, CXT_PINCFG_SWS_JS201D, + CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, CXT_FIXUP_ACER_SWIFT_HP, @@ -826,6 +827,12 @@ static const struct hda_pintbl cxt_pincfg_lemote[] = { {} }; +/* Lenovo IdeaPad Slim 5 16AKP10 with SN6140 */ +static const struct hda_pintbl cxt_pincfg_lenovo_ideapad_slim5_16akp10[] = { + { 0x1a, 0x95a60130 }, /* Internal mic, fixed/always-connected */ + {} +}; + /* SuoWoSi/South-holding JS201D with sn6140 */ static const struct hda_pintbl cxt_pincfg_sws_js201d[] = { { 0x16, 0x03211040 }, /* hp out */ @@ -1006,6 +1013,10 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_PINS, .v.pins = cxt_pincfg_sws_js201d, }, + [CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10] = { + .type = HDA_FIXUP_PINS, + .v.pins = cxt_pincfg_lenovo_ideapad_slim5_16akp10, + }, [CXT_PINCFG_TOP_SPEAKER] = { .type = HDA_FIXUP_PINS, .v.pins = (const struct hda_pintbl[]) { @@ -1114,6 +1125,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x17aa, 0x21da, "Lenovo X220", CXT_PINCFG_LENOVO_TP410), SND_PCI_QUIRK(0x17aa, 0x21db, "Lenovo X220-tablet", CXT_PINCFG_LENOVO_TP410), SND_PCI_QUIRK(0x17aa, 0x38af, "Lenovo IdeaPad Z560", CXT_FIXUP_MUTE_LED_EAPD), + SND_PCI_QUIRK(0x17aa, 0x38b6, "Lenovo IdeaPad Slim 5 16AKP10", CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10), SND_PCI_QUIRK(0x17aa, 0x3905, "Lenovo G50-30", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x17aa, 0x390b, "Lenovo G50-80", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x17aa, 0x3975, "Lenovo U300s", CXT_FIXUP_STEREO_DMIC), diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index 1f4d646724ed..0b6816018a42 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -2233,8 +2233,10 @@ void snd_hda_hdmi_acomp_pin_eld_notify(void *audio_ptr, int port, int dev_id) /* skip notification during system suspend (but not in runtime PM); * the state will be updated at resume */ - if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) + if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) { + codec->acomp_requested_resume = 1; return; + } snd_hda_hdmi_check_presence_and_report(codec, pin_nid, dev_id); } diff --git a/sound/hda/codecs/helpers/razer_blade16_2025.c b/sound/hda/codecs/helpers/razer_blade16_2025.c new file mode 100644 index 000000000000..1f2e06d8fc13 --- /dev/null +++ b/sound/hda/codecs/helpers/razer_blade16_2025.c @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Razer Blade 16 (2025) external smart-amplifier programming and fixup, + * to be included from the codec driver. + * + * The internal speakers are driven by an external smart amplifier whose + * crossover, voicing and protection live in a DSP reached over a vendor + * coef mailbox on NID 0x20. These tables hold the programming captured + * from the Windows driver, reduced to the NID-0x20 traffic this machine + * needs (everything else the HDA parser rebuilds). + * + * Each entry is either a plain codec coef write or a mailbox write: + * strobe == 0: coef[addr] = lo + * strobe != 0: amp[addr] = (hi << 16) | lo, with the bank selected on + * port 0x89 and the write committed by the strobe opcode. + */ + +struct alc298_razer_blade16_2025_op { + u16 bank; /* mailbox bank (port 0x89); unused for a plain coef */ + u16 addr; /* mailbox register, or plain coef index */ + u16 hi; /* mailbox high word; unused for a plain coef */ + u16 lo; /* mailbox low word, or plain coef value */ + u16 strobe; /* mailbox commit opcode; 0 marks a plain coef write */ +}; + +/* program the external DSP at init; the amp is woken/parked separately */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_init[] = { + { 0x0000, 0xf102, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xf103, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc000, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 }, + { 0x0000, 0xf20d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xf212, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xc001, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc003, 0x0000, 0x0022, 0xb031 }, + { 0x0000, 0xc004, 0x0000, 0x0044, 0xb031 }, + { 0x0000, 0xc005, 0x0000, 0x0044, 0xb031 }, + { 0x0000, 0xc007, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xc00e, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xf223, 0x0000, 0x007f, 0xb031 }, + { 0x0000, 0xf224, 0x0000, 0x00db, 0xb031 }, + { 0x0000, 0xf225, 0x0000, 0x00ee, 0xb031 }, + { 0x0000, 0xf226, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xf227, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xf21a, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xf242, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc120, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xc321, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc200, 0x0000, 0x00d8, 0xb031 }, + { 0x0000, 0xc201, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xc202, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xc400, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc401, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xc402, 0x0000, 0x00e0, 0xb031 }, + { 0x0000, 0xc403, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc404, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xc406, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc407, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc408, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xc300, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xdf5f, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xdf60, 0x0000, 0x00a7, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x0084, 0xb031 }, + { 0x0000, 0xc206, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xf10a, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0065, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 }, + { 0x0000, 0xf105, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 }, + { 0x0000, 0xe706, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xe707, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xe806, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xe807, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xce04, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xce05, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce06, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xce07, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xcf04, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xcf05, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xcf06, 0x0000, 0x0031, 0xb031 }, + { 0x0000, 0xcf07, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xc130, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xe000, 0x0000, 0x00a8, 0xb031 }, + { 0x4100, 0x1888, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xea00, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xed00, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xcc2c, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc2d, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xcc2e, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xcc2f, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcd2c, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd2d, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xcd2e, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xcd2f, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcc24, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc25, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xcc26, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xcc27, 0x0000, 0x0085, 0xb031 }, + { 0x0000, 0xcd24, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd25, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xcd26, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xcd27, 0x0000, 0x0085, 0xb031 }, + { 0x0000, 0xcc20, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc21, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc22, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcd20, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd21, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd22, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xcc16, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xcc17, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd16, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xcd17, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc29, 0x0000, 0x005d, 0xb031 }, + { 0x0000, 0xcc2a, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xcd29, 0x0000, 0x005d, 0xb031 }, + { 0x0000, 0xcd2a, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xcc31, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xcc32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc33, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc34, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd31, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xcd32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd33, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd34, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc36, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xcc37, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc38, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc39, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd36, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xcd37, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd38, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcd39, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xcc09, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc0a, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xcc0b, 0x0000, 0x007c, 0xb031 }, + { 0x0000, 0xcc0c, 0x0000, 0x005b, 0xb031 }, + { 0x0000, 0xcd09, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd0a, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xcd0b, 0x0000, 0x007c, 0xb031 }, + { 0x0000, 0xcd0c, 0x0000, 0x005b, 0xb031 }, + { 0x0000, 0xcc0e, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcc0f, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xcc10, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xcc11, 0x0000, 0x002d, 0xb031 }, + { 0x0000, 0xcd0e, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcd0f, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xcd10, 0x0000, 0x003e, 0xb031 }, + { 0x0000, 0xcd11, 0x0000, 0x002d, 0xb031 }, + { 0x0000, 0xccd6, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccd7, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdd6, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcdd7, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xccd8, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccd9, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdd8, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcdd9, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xccda, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xccdb, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xcdda, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xcddb, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xc320, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xc321, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xe604, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xdd00, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xdc19, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdc1a, 0x0000, 0x006a, 0xb031 }, + { 0x0000, 0xdc1b, 0x0000, 0x00aa, 0xb031 }, + { 0x0000, 0xdc1c, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xdc1d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdc1e, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xdc1f, 0x0000, 0x0062, 0xb031 }, + { 0x0000, 0xdc20, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xde19, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xde1a, 0x0000, 0x006a, 0xb031 }, + { 0x0000, 0xde1b, 0x0000, 0x00aa, 0xb031 }, + { 0x0000, 0xde1c, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xde1d, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xde1e, 0x0000, 0x0027, 0xb031 }, + { 0x0000, 0xde1f, 0x0000, 0x0062, 0xb031 }, + { 0x0000, 0xde20, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xdb32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdd32, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdb33, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xdd33, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xdb34, 0x0000, 0x001a, 0xb031 }, + { 0x0000, 0xdd34, 0x0000, 0x001a, 0xb031 }, + { 0x0000, 0xdb15, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdd15, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdb17, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xdd17, 0x0000, 0x003f, 0xb031 }, + { 0x0000, 0xdb94, 0x0000, 0x0070, 0xb031 }, + { 0x0000, 0xdd94, 0x0000, 0x0070, 0xb031 }, + { 0x0000, 0xdb19, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdd19, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x001c, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xdb04, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xdb05, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdd04, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xdd05, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xdbbb, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xdbbc, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xdbbd, 0x0000, 0x00f3, 0xb031 }, + { 0x0000, 0xdbbe, 0x0000, 0x00cf, 0xb031 }, + { 0x0000, 0xddbb, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xddbc, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xddbd, 0x0000, 0x00f3, 0xb031 }, + { 0x0000, 0xddbe, 0x0000, 0x00cf, 0xb031 }, + { 0x0000, 0xdb01, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xdd01, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xdb08, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdd08, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xdc52, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xde52, 0x0000, 0x00ef, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xdf0a, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xdf0b, 0x0000, 0x007f, 0xb031 }, + { 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdf01, 0x0000, 0x0073, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xebcb, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb10, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb11, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xeb12, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xeb13, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xeb14, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb15, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xeb16, 0x0000, 0x0069, 0xb031 }, + { 0x0000, 0xeb21, 0x0000, 0x0011, 0xb031 }, + { 0x0000, 0xeb22, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xeb23, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb24, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb25, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xeb26, 0x0000, 0x0055, 0xb031 }, + { 0x0000, 0xeb27, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb28, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb29, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xeb2a, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xeb2b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xeb30, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb31, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xeb32, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xeb33, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb34, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb35, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xeb36, 0x0000, 0x0024, 0xb031 }, + { 0x0000, 0xeb37, 0x0000, 0x0068, 0xb031 }, + { 0x0000, 0xeb40, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb41, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb42, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xeb43, 0x0000, 0x00e8, 0xb031 }, + { 0x0000, 0xeb44, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb45, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xeb46, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xeb47, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb48, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb49, 0x0000, 0x00c5, 0xb031 }, + { 0x0000, 0xeb4a, 0x0000, 0x0032, 0xb031 }, + { 0x0000, 0xeb4b, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeb50, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb51, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xeb52, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xeb53, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb54, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb55, 0x0000, 0x007d, 0xb031 }, + { 0x0000, 0xeb56, 0x0000, 0x00d9, 0xb031 }, + { 0x0000, 0xeb57, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xeb60, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xeb61, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xeb62, 0x0000, 0x00b5, 0xb031 }, + { 0x0000, 0xeb63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xeb64, 0x0000, 0x00fa, 0xb031 }, + { 0x0000, 0xeb65, 0x0000, 0x0086, 0xb031 }, + { 0x0000, 0xeb66, 0x0000, 0x00a8, 0xb031 }, + { 0x0000, 0xeb67, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xeb68, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xeb69, 0x0000, 0x009a, 0xb031 }, + { 0x0000, 0xeb6a, 0x0000, 0x002f, 0xb031 }, + { 0x0000, 0xeb6b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xebb6, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeb70, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb71, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xeb72, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xeb73, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb74, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xeb75, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xeb76, 0x0000, 0x002c, 0xb031 }, + { 0x0000, 0xeb77, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xeb80, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb81, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb82, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xeb83, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb84, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xeb85, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xeb86, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xeb87, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xeb88, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xeb89, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xeb8a, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xeb8b, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xebb8, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeb90, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xeb91, 0x0000, 0x008b, 0xb031 }, + { 0x0000, 0xeb92, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xeb93, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xeb94, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xeb95, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xeb96, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xeb97, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xeba0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeba1, 0x0000, 0x0091, 0xb031 }, + { 0x0000, 0xeba2, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xeba3, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeba4, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xeba5, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeba6, 0x0000, 0x0093, 0xb031 }, + { 0x0000, 0xeba7, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xeba8, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xeba9, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xebaa, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xebab, 0x0000, 0x006c, 0xb031 }, + { 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 }, + { 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xeb00, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xeb01, 0x0000, 0x00a9, 0xb031 }, + { 0x0000, 0xeb02, 0x0000, 0x00df, 0xb031 }, + { 0x0000, 0xeb03, 0x0000, 0x007b, 0xb031 }, + { 0x0000, 0xeb05, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xeb06, 0x0000, 0x005a, 0xb031 }, + { 0x0000, 0xeb07, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xd0cb, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xd0b0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd010, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd011, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd012, 0x0000, 0x0092, 0xb031 }, + { 0x0000, 0xd013, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd014, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd015, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xd016, 0x0000, 0x0072, 0xb031 }, + { 0x0000, 0xd017, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xd020, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd021, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd022, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd023, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd024, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd025, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd026, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xd027, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xd028, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd029, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd02a, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd02b, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xce6a, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xdb04, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xdb05, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xcc00, 0x0000, 0x00c1, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca62, 0x0000, 0x0077, 0xb031 }, + { 0x0000, 0xca65, 0x0000, 0x0082, 0xb031 }, + { 0x0000, 0xca68, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xca75, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xca7b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc860, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc861, 0x0000, 0x00e5, 0xb031 }, + { 0x0000, 0xc862, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xc863, 0x0000, 0x00c6, 0xb031 }, + { 0x0000, 0xc864, 0x0000, 0x00ba, 0xb031 }, + { 0x0000, 0xc865, 0x0000, 0x0061, 0xb031 }, + { 0x0000, 0xc867, 0x0000, 0x0017, 0xb031 }, + { 0x0000, 0xc868, 0x0000, 0x00ed, 0xb031 }, + { 0x0000, 0xc869, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc86a, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xc86b, 0x0000, 0x00e2, 0xb031 }, + { 0x0000, 0xc875, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xc87b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xc836, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc8a0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc833, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc834, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc83f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc83e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc83a, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xc83b, 0x0000, 0x0089, 0xb031 }, + { 0x0000, 0xc830, 0x0000, 0x0013, 0xb031 }, + { 0x0000, 0xc826, 0x0000, 0x00d1, 0xb031 }, + { 0x0000, 0xc890, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc823, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc824, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xc82f, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xc82e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc820, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc816, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc880, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xc813, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc814, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc81f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc81e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc81a, 0x0000, 0x0066, 0xb031 }, + { 0x0000, 0xc81b, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc810, 0x0000, 0x000d, 0xb031 }, + { 0x0000, 0xc846, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc8d0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc844, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc84f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc84e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc84a, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc84b, 0x0000, 0x0056, 0xb031 }, + { 0x0000, 0xc800, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc500, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc501, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc502, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc503, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc504, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc505, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc506, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc507, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc510, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc511, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc512, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc513, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc514, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc515, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc516, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc517, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc518, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xd251, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xecb0, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec10, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec11, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xec12, 0x0000, 0x00ab, 0xb031 }, + { 0x0000, 0xec13, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xec14, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec15, 0x0000, 0x004c, 0xb031 }, + { 0x0000, 0xec16, 0x0000, 0x0069, 0xb031 }, + { 0x0000, 0xec21, 0x0000, 0x0011, 0xb031 }, + { 0x0000, 0xec22, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xec23, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xec24, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec25, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xec26, 0x0000, 0x0055, 0xb031 }, + { 0x0000, 0xec27, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec28, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec29, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xec2a, 0x0000, 0x0051, 0xb031 }, + { 0x0000, 0xec2b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xecb2, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec30, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec31, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xec32, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xec33, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec34, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec35, 0x0000, 0x00bb, 0xb031 }, + { 0x0000, 0xec36, 0x0000, 0x0024, 0xb031 }, + { 0x0000, 0xec37, 0x0000, 0x0068, 0xb031 }, + { 0x0000, 0xec40, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec41, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec42, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xec43, 0x0000, 0x00e8, 0xb031 }, + { 0x0000, 0xec44, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec45, 0x0000, 0x004f, 0xb031 }, + { 0x0000, 0xec46, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xec47, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec48, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec49, 0x0000, 0x00c5, 0xb031 }, + { 0x0000, 0xec4a, 0x0000, 0x0032, 0xb031 }, + { 0x0000, 0xec4b, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xecb4, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec50, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec51, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xec52, 0x0000, 0x00b4, 0xb031 }, + { 0x0000, 0xec53, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec54, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec55, 0x0000, 0x007d, 0xb031 }, + { 0x0000, 0xec56, 0x0000, 0x00d9, 0xb031 }, + { 0x0000, 0xec57, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xec60, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xec61, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xec62, 0x0000, 0x00b5, 0xb031 }, + { 0x0000, 0xec63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xec64, 0x0000, 0x00fa, 0xb031 }, + { 0x0000, 0xec65, 0x0000, 0x0086, 0xb031 }, + { 0x0000, 0xec66, 0x0000, 0x00a8, 0xb031 }, + { 0x0000, 0xec67, 0x0000, 0x0040, 0xb031 }, + { 0x0000, 0xec68, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xec69, 0x0000, 0x009a, 0xb031 }, + { 0x0000, 0xec6a, 0x0000, 0x002f, 0xb031 }, + { 0x0000, 0xec6b, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xecb6, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xec70, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec71, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xec72, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xec73, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec74, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xec75, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xec76, 0x0000, 0x002c, 0xb031 }, + { 0x0000, 0xec77, 0x0000, 0x0050, 0xb031 }, + { 0x0000, 0xec80, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec81, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec82, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xec83, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec84, 0x0000, 0x00f5, 0xb031 }, + { 0x0000, 0xec85, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xec86, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xec87, 0x0000, 0x00d0, 0xb031 }, + { 0x0000, 0xec88, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xec89, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xec8a, 0x0000, 0x0016, 0xb031 }, + { 0x0000, 0xec8b, 0x0000, 0x0028, 0xb031 }, + { 0x0000, 0xecb8, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xec90, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xec91, 0x0000, 0x008b, 0xb031 }, + { 0x0000, 0xec92, 0x0000, 0x00bd, 0xb031 }, + { 0x0000, 0xec93, 0x0000, 0x0064, 0xb031 }, + { 0x0000, 0xec94, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xec95, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xec96, 0x0000, 0x00e7, 0xb031 }, + { 0x0000, 0xec97, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xeca0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeca1, 0x0000, 0x0091, 0xb031 }, + { 0x0000, 0xeca2, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xeca3, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xeca4, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xeca5, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xeca6, 0x0000, 0x0093, 0xb031 }, + { 0x0000, 0xeca7, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xeca8, 0x0000, 0x0003, 0xb031 }, + { 0x0000, 0xeca9, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xecaa, 0x0000, 0x0099, 0xb031 }, + { 0x0000, 0xecab, 0x0000, 0x006c, 0xb031 }, + { 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 }, + { 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 }, + { 0x0000, 0xec00, 0x0000, 0x0005, 0xb031 }, + { 0x0000, 0xec01, 0x0000, 0x00a9, 0xb031 }, + { 0x0000, 0xec02, 0x0000, 0x00df, 0xb031 }, + { 0x0000, 0xec03, 0x0000, 0x007b, 0xb031 }, + { 0x0000, 0xec05, 0x0000, 0x0079, 0xb031 }, + { 0x0000, 0xec06, 0x0000, 0x005a, 0xb031 }, + { 0x0000, 0xec07, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xd1b0, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd110, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd111, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd112, 0x0000, 0x0092, 0xb031 }, + { 0x0000, 0xd113, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd114, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd115, 0x0000, 0x00f6, 0xb031 }, + { 0x0000, 0xd116, 0x0000, 0x0072, 0xb031 }, + { 0x0000, 0xd117, 0x0000, 0x00b0, 0xb031 }, + { 0x0000, 0xd120, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd121, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd122, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd123, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd124, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd125, 0x0000, 0x0009, 0xb031 }, + { 0x0000, 0xd126, 0x0000, 0x0090, 0xb031 }, + { 0x0000, 0xd127, 0x0000, 0x0020, 0xb031 }, + { 0x0000, 0xd128, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xd129, 0x0000, 0x00fb, 0xb031 }, + { 0x0000, 0xd12a, 0x0000, 0x0037, 0xb031 }, + { 0x0000, 0xd12b, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 }, + { 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xdd04, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xdd05, 0x0000, 0x0006, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x000f, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca82, 0x0000, 0x0077, 0xb031 }, + { 0x0000, 0xca85, 0x0000, 0x0082, 0xb031 }, + { 0x0000, 0xca88, 0x0000, 0x00c2, 0xb031 }, + { 0x0000, 0xca95, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xca9b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc960, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc961, 0x0000, 0x00e5, 0xb031 }, + { 0x0000, 0xc962, 0x0000, 0x00eb, 0xb031 }, + { 0x0000, 0xc963, 0x0000, 0x00c6, 0xb031 }, + { 0x0000, 0xc964, 0x0000, 0x00ba, 0xb031 }, + { 0x0000, 0xc965, 0x0000, 0x0061, 0xb031 }, + { 0x0000, 0xc967, 0x0000, 0x0017, 0xb031 }, + { 0x0000, 0xc968, 0x0000, 0x00ed, 0xb031 }, + { 0x0000, 0xc969, 0x0000, 0x003c, 0xb031 }, + { 0x0000, 0xc96a, 0x0000, 0x008a, 0xb031 }, + { 0x0000, 0xc96b, 0x0000, 0x00e2, 0xb031 }, + { 0x0000, 0xc975, 0x0000, 0x003a, 0xb031 }, + { 0x0000, 0xc97b, 0x0000, 0x006f, 0xb031 }, + { 0x0000, 0xc936, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc9a0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc933, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xc934, 0x0000, 0x0002, 0xb031 }, + { 0x0000, 0xc93f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc93e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc93a, 0x0000, 0x0076, 0xb031 }, + { 0x0000, 0xc93b, 0x0000, 0x0089, 0xb031 }, + { 0x0000, 0xc930, 0x0000, 0x0013, 0xb031 }, + { 0x0000, 0xc926, 0x0000, 0x00d1, 0xb031 }, + { 0x0000, 0xc990, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc923, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc924, 0x0000, 0x0004, 0xb031 }, + { 0x0000, 0xc92f, 0x0000, 0x0018, 0xb031 }, + { 0x0000, 0xc92e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc920, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc916, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc980, 0x0000, 0x00e1, 0xb031 }, + { 0x0000, 0xc913, 0x0000, 0x0000, 0xb031 }, + { 0x0000, 0xc914, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc91f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc91e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc91a, 0x0000, 0x0066, 0xb031 }, + { 0x0000, 0xc91b, 0x0000, 0x0078, 0xb031 }, + { 0x0000, 0xc910, 0x0000, 0x000d, 0xb031 }, + { 0x0000, 0xc946, 0x0000, 0x00f9, 0xb031 }, + { 0x0000, 0xc9d0, 0x0000, 0x00f1, 0xb031 }, + { 0x0000, 0xc944, 0x0000, 0x0001, 0xb031 }, + { 0x0000, 0xc94f, 0x0000, 0x0014, 0xb031 }, + { 0x0000, 0xc94e, 0x0000, 0x0094, 0xb031 }, + { 0x0000, 0xc94a, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc94b, 0x0000, 0x0056, 0xb031 }, + { 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xc600, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc601, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc602, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc603, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc604, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc605, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc606, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc607, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc610, 0x0000, 0x0007, 0xb031 }, + { 0x0000, 0xc611, 0x0000, 0x00da, 0xb031 }, + { 0x0000, 0xc612, 0x0000, 0x000e, 0xb031 }, + { 0x0000, 0xc613, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc614, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xc615, 0x0000, 0x0026, 0xb031 }, + { 0x0000, 0xc616, 0x0000, 0x004a, 0xb031 }, + { 0x0000, 0xc617, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xc618, 0x0000, 0x0008, 0xb031 }, + { 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xc802, 0x0000, 0x00d2, 0xb031 }, + { 0x0000, 0xc902, 0x0000, 0x00d2, 0xb031 }, + { 0x0000, 0xc800, 0x0000, 0x00f0, 0xb031 }, + { 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 }, + { 0x0000, 0xd252, 0x0000, 0x0030, 0xb031 }, + { 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 }, + { 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 }, + { 0x0000, 0xdbb5, 0x0041, 0xd27f, 0xb037 }, + { 0x0000, 0xddb5, 0x0040, 0x6e3b, 0xb037 }, + { 0x0000, 0xdb93, 0x0000, 0x009e, 0xb033 }, + { 0x0000, 0xdd93, 0x0000, 0x009e, 0xb033 }, + { 0x0000, 0xdb12, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xdd12, 0x0000, 0x00c0, 0xb031 }, + { 0x0000, 0xdb08, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdd08, 0x0000, 0x0080, 0xb031 }, + { 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 }, + { 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 }, + { 0x0000, 0x0010, 0x0000, 0x0f21, 0x0000 }, +}; + +/* wake the amp (ea00=0x43, c121=0x0b, f109=0xe0) */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_wake[] = { + { 0x0000, 0xea00, 0x0000, 0x0043, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 }, +}; + +/* park the amp (ea00=0x47, c121=0x0a, f109=0xa0) */ +static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_sleep[] = { + { 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 }, + { 0x0000, 0xc121, 0x0000, 0x000a, 0xb031 }, + { 0x0000, 0xf109, 0x0000, 0x00a0, 0xb031 }, +}; + +/* + * Razer Blade 16 (2025): Realtek ALC298 driving an external smart amp. + * Crossover, voicing and speaker protection live in an external DSP + * reached over a vendor-coef mailbox on NID 0x20; the captured DSP + * programming is in the tables above. NID 0x14, the BIOS-disabled + * tweeter pin, is re-exposed via a pin-config override so the parser + * drives it as a normal internal speaker (shared volume/mute, auto-mute + * on headphone insertion). + * + * alc298_razer_blade16_2025_apply() runs one op-table: a plain coef write + * (strobe == 0) goes through alc_write_coef_idx(), while a mailbox op + * selects a bank and streams four PROC_COEF words (addr/hi/lo/commit) + * into a single coef index (0x23) on NID 0x20 to reach the external DSP + * -- that one-index/many-words form is why it cannot be expressed as an + * alc_process_coef_fw() table (one value per index). + */ +static void alc298_razer_blade16_2025_apply(struct hda_codec *codec, + const struct alc298_razer_blade16_2025_op *seq, + int num) +{ + int i; + + for (i = 0; i < num; i++) { + if (!seq[i].strobe) { + alc_write_coef_idx(codec, seq[i].addr, seq[i].lo); + continue; + } + alc_write_coef_idx(codec, 0x89, seq[i].bank); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].addr); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].hi); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].lo); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].strobe); + } +} + +static void alc298_razer_blade16_2025_pcm_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + /* power the external amp only while a stream is running */ + switch (action) { + case HDA_GEN_PCM_ACT_PREPARE: + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake)); + break; + case HDA_GEN_PCM_ACT_CLEANUP: + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep)); + break; + } +} + +static void alc298_fixup_razer_blade16_2025(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + /* + * Pin every output to a fixed converter so the routing does not + * rely on the generic parser's DAC-allocation heuristics: without + * a fixed assignment the parser brings up only one speaker pin and + * leaves the other silent, and the allocation could change between + * kernel versions. Both speaker pins share DAC 0x03 through mixer + * 0x0d: that is the only converter that drives the woofer (NID + * 0x17) on this machine -- feeding it from DAC 0x02 or the digital + * converter 0x06 leaves the woofer silent. Both pins carry the + * same stereo stream and the external amp does the crossover. + * Headphone NID 0x21 keeps DAC 0x02. + */ + static const hda_nid_t preferred_pairs[] = { + 0x21, 0x02, + 0x14, 0x03, + 0x17, 0x03, + 0 + }; + struct alc_spec *spec = codec->spec; + + switch (action) { + case HDA_FIXUP_ACT_PRE_PROBE: + spec->gen.preferred_dacs = preferred_pairs; + break; + case HDA_FIXUP_ACT_PROBE: + spec->gen.pcm_playback_hook = alc298_razer_blade16_2025_pcm_hook; + break; + case HDA_FIXUP_ACT_INIT: + /* + * Wake the amp, program its DSP at boot and on resume, then + * park it. It is woken again for playback and parked + * afterwards from the pcm_playback_hook. + */ + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake)); + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_init, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_init)); + alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep, + ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep)); + break; + } +} diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 5d09baa6d32e..b26ed9c1f09f 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -3357,6 +3357,9 @@ static void alc287_fixup_acer_micmute_led(struct hda_codec *codec, /* for alc285_fixup_ideapad_s740_coef() */ #include "../helpers/ideapad_s740.c" +/* for alc298_fixup_razer_blade16_2025() */ +#include "../helpers/razer_blade16_2025.c" + static const struct coef_fw alc256_fixup_set_coef_defaults_coefs[] = { WRITE_COEF(0x10, 0x0020), WRITE_COEF(0x24, 0x0000), WRITE_COEF(0x26, 0x0000), WRITE_COEF(0x29, 0x3000), @@ -4041,6 +4044,8 @@ enum { ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, ALC298_FIXUP_LG_GRAM_STYLE_14, + ALC298_FIXUP_RAZER_BLADE16_2025_PINS, + ALC298_FIXUP_RAZER_BLADE16_2025, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -4206,6 +4211,20 @@ static void alc287_fixup_lenovo_yoga_book_9i(struct hda_codec *codec, } static const struct hda_fixup alc269_fixups[] = { + [ALC298_FIXUP_RAZER_BLADE16_2025_PINS] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x14, 0x90170121 }, /* tweeter as internal speaker, seq 1 */ + { 0x17, 0x90170120 }, /* woofer as internal speaker, seq 0 */ + { } + }, + .chained = true, + .chain_id = ALC298_FIXUP_RAZER_BLADE16_2025, + }, + [ALC298_FIXUP_RAZER_BLADE16_2025] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_razer_blade16_2025, + }, [ALC269_FIXUP_GPIO2] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_gpio2, @@ -6807,11 +6826,13 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC287_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x160e, "Acer PT316-51S", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1025, 0x161f, "Acer S40-54", ALC256_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1640, "Acer Aspire A315-44P", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1679, "Acer Nitro 16 AN16-41", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x171e, "Acer Nitro ANV15-51", ALC245_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x173a, "Acer Swift SFG14-73", ALC245_FIXUP_ACER_MICMUTE_LED), + SND_PCI_QUIRK(0x1025, 0x1758, "Acer Nitro ANV15-41", ALC245_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1826, "Acer Helios ZPC", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x182c, "Acer Helios ZPD", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x1844, "Acer Helios ZPS", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), @@ -6986,9 +7007,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8537, "HP ProBook 440 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8548, "HP EliteBook x360 830 G6", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x854d, "HP EliteBook 840 G6", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x856a, "HP Pavilion 15-cs1xxx", ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B), SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11), SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360), + SND_PCI_QUIRK(0x103c, 0x85f0, "HP Laptop 15-dw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8603, "HP Omen 17-cb0xxx", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x860c, "HP ZBook 17 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT), SND_PCI_QUIRK(0x103c, 0x860f, "HP ZBook 15 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT), @@ -7065,6 +7088,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x88d1, "HP Pavilion 15-eh1xxx (mainboard 88D1)", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x88dd, "HP Pavilion 15z-ec200", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x88eb, "HP Victus 16-e0xxx", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x88ee, "HP Victus 16-e0xxx (MB 88EE)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x890e, "HP 255 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8919, "HP Pavilion Aero Laptop 13-be0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -7326,6 +7350,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8f0e, "HP ZBook X G2i 16W", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8f2d, "HP Auster 14", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8f2e, "HP Auster 14", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8f37, "HP EliteBook 6 G2i", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8f38, "HP EliteBook 6 G2i", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8f39, "HP EliteBook 6 G2i", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8f3a, "HP EliteBook 6 G2i", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8f3c, "HP EliteBook 6 G2a", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8f3d, "HP EliteBook 6 G2a", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8f40, "HP ZBook 8 G2a 14", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED), @@ -7597,6 +7625,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1558, 0x51b3, "Clevo NS70AU", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x5630, "Clevo NP50RNJS", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x5700, "Clevo X560WN[RST]", ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1558, 0x6480, "Clevo V6xxAW", ALC245_FIXUP_CLEVO_NOISY_MIC), SND_PCI_QUIRK(0x1558, 0x70a1, "Clevo NB70T[HJK]", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x70b3, "Clevo NK70SB", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1558, 0x70f2, "Clevo NH79EPY", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE), @@ -7753,11 +7782,13 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3865, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3866, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3869, "Lenovo Yoga7 14IAL7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + HDA_CODEC_QUIRK(0x17aa, 0x386a, "Lenovo Yoga 7 16IAP7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x386e, "Legion Y9000X 2022 IAH7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x386e, "Yoga Pro 7 14ARP8", ALC285_FIXUP_SPEAKER2_TO_DAC1), HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C), + SND_PCI_QUIRK(0x17aa, 0x3874, "Legion 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3877, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3878, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x387d, "Yoga S780-16 pro Quad AAC", ALC287_FIXUP_TAS2781_I2C), @@ -7852,6 +7883,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC), + SND_PCI_QUIRK(0x1a58, 0x300e, "Razer Blade 16 (2025)", + ALC298_FIXUP_RAZER_BLADE16_2025_PINS), SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20), SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI), SND_PCI_QUIRK(0x1b35, 0x1237, "CZC L101", ALC269_FIXUP_CZC_L101), @@ -7911,6 +7944,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop 13 Pro PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0xf111, 0x010f, "Framework Laptop 13 PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0xf111, 0x0010, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST), #if 0 /* Below is a quirk table taken from the old code. diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c b/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c index 96d8cc6c2324..fdf406e92fca 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c @@ -6,7 +6,6 @@ // // Author: Lucas Tanure -#include #include #include diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c index 732ae534db36..416d7bf3e289 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c @@ -128,6 +128,7 @@ static const struct cs35l41_config cs35l41_config_table[] = { { "17AA3866", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA386E", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, 2, -1, 0, 0, 0 }, { "17AA386F", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, + { "17AA3874", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA3877", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA3878", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA38A9", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, 2, -1, 0, 0, 0 }, @@ -554,6 +555,7 @@ static const struct cs35l41_prop_model cs35l41_prop_model_table[] = { { "CSC3551", "17AA3866", generic_dsd_config }, { "CSC3551", "17AA386E", generic_dsd_config }, { "CSC3551", "17AA386F", generic_dsd_config }, + { "CSC3551", "17AA3874", generic_dsd_config }, { "CSC3551", "17AA3877", generic_dsd_config }, { "CSC3551", "17AA3878", generic_dsd_config }, { "CSC3551", "17AA38A9", generic_dsd_config }, diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_spi.c b/sound/hda/codecs/side-codecs/cs35l41_hda_spi.c index 2acbaf8467a0..aab2066a20eb 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda_spi.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda_spi.c @@ -6,7 +6,6 @@ // // Author: Lucas Tanure -#include #include #include diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c index 93bf12116626..69a22fdfeedb 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index 3978d58ad020..4899ea372798 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index ef533770179b..b9ded149ea11 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -2967,8 +2967,11 @@ static void hda_codec_pm_complete(struct device *dev) dev->power.power_state = PMSG_RESUME; if (pm_runtime_suspended(dev) && (codec->jackpoll_interval || - hda_codec_need_resume(codec) || codec->forced_resume)) + hda_codec_need_resume(codec) || codec->forced_resume || + codec->acomp_requested_resume)) { + codec->acomp_requested_resume = 0; pm_request_resume(dev); + } } static int hda_codec_pm_suspend(struct device *dev) diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c index 4b03c64e72ab..192f5c044b7a 100644 --- a/sound/hda/controllers/intel.c +++ b/sound/hda/controllers/intel.c @@ -100,6 +100,8 @@ enum { #define ATIHDMI_NUM_CAPTURE 0 #define ATIHDMI_NUM_PLAYBACK 8 +/* Hygon HD Audio controller */ +#define PCI_DEVICE_ID_HYGON_18H_M05H_HDA 0x14a9 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX; static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR; @@ -241,6 +243,7 @@ enum { AZX_DRIVER_ZHAOXIN, AZX_DRIVER_ZHAOXINHDMI, AZX_DRIVER_LOONGSON, + AZX_DRIVER_HYGON, AZX_DRIVER_GENERIC, AZX_NUM_DRIVERS, /* keep this as last entry */ }; @@ -360,6 +363,7 @@ static const char * const driver_short_names[] = { [AZX_DRIVER_ZHAOXIN] = "HDA Zhaoxin", [AZX_DRIVER_ZHAOXINHDMI] = "HDA Zhaoxin HDMI", [AZX_DRIVER_LOONGSON] = "HDA Loongson", + [AZX_DRIVER_HYGON] = "HDA Hygon", [AZX_DRIVER_GENERIC] = "HD-Audio Generic", }; @@ -1926,6 +1930,10 @@ static int azx_first_init(struct azx *chip) if (chip->driver_type == AZX_DRIVER_ZHAOXINHDMI) bus->polling_mode = 1; + if (chip->driver_type == AZX_DRIVER_HYGON && + chip->pci->device == PCI_DEVICE_ID_HYGON_18H_M05H_HDA) + bus->access_sdnctl_in_dword = 1; + bus->remap_addr = pcim_iomap_region(pci, 0, "ICH HD audio"); if (IS_ERR(bus->remap_addr)) return PTR_ERR(bus->remap_addr); @@ -2836,6 +2844,9 @@ static const struct pci_device_id azx_ids[] = { .driver_data = AZX_DRIVER_LOONGSON | AZX_DCAPS_NO_TCSEL }, { PCI_VDEVICE(LOONGSON, PCI_DEVICE_ID_LOONGSON_HDMI), .driver_data = AZX_DRIVER_LOONGSON | AZX_DCAPS_NO_TCSEL }, + /* Hygon HDAudio */ + { PCI_VDEVICE(HYGON, PCI_DEVICE_ID_HYGON_18H_M05H_HDA), + .driver_data = AZX_DRIVER_HYGON | AZX_DCAPS_POSFIX_LPIB | AZX_DCAPS_NO_MSI }, { 0, } }; MODULE_DEVICE_TABLE(pci, azx_ids); diff --git a/sound/hda/core/controller.c b/sound/hda/core/controller.c index 69e11d62bbfa..6312ad7af71d 100644 --- a/sound/hda/core/controller.c +++ b/sound/hda/core/controller.c @@ -511,7 +511,10 @@ void snd_hdac_bus_exit_link_reset(struct hdac_bus *bus) { unsigned long timeout; - snd_hdac_chip_updateb(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET); + if (bus->access_sdnctl_in_dword) + snd_hdac_chip_updatel(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET); + else + snd_hdac_chip_updateb(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET); timeout = jiffies + msecs_to_jiffies(100); while (!snd_hdac_chip_readb(bus, GCTL) && time_before(jiffies, timeout)) @@ -576,7 +579,10 @@ static void azx_int_disable(struct hdac_bus *bus) /* disable interrupts in stream descriptor */ list_for_each_entry(azx_dev, &bus->stream_list, list) - snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_INT_MASK, 0); + if (bus->access_sdnctl_in_dword) + snd_hdac_stream_updatel(azx_dev, SD_CTL, SD_INT_MASK, 0); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_INT_MASK, 0); /* disable SIE for all streams & disable controller CIE and GIE */ snd_hdac_chip_writel(bus, INTCTL, 0); diff --git a/sound/hda/core/hda_bus_type.c b/sound/hda/core/hda_bus_type.c index eb72a7af2e56..a4afd41b6f84 100644 --- a/sound/hda/core/hda_bus_type.c +++ b/sound/hda/core/hda_bus_type.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include diff --git a/sound/hda/core/stream.c b/sound/hda/core/stream.c index b471a038b314..0091c4ef6486 100644 --- a/sound/hda/core/stream.c +++ b/sound/hda/core/stream.c @@ -146,8 +146,12 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev) stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream); else stripe_ctl = 0; - snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, - stripe_ctl); + if (bus->access_sdnctl_in_dword) + snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, + stripe_ctl); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, + stripe_ctl); } /* set DMA start and interrupt mask */ if (bus->access_sdnctl_in_dword) @@ -166,11 +170,22 @@ EXPORT_SYMBOL_GPL(snd_hdac_stream_start); */ static void snd_hdac_stream_clear(struct hdac_stream *azx_dev) { - snd_hdac_stream_updateb(azx_dev, SD_CTL, - SD_CTL_DMA_START | SD_INT_MASK, 0); - snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ - if (azx_dev->stripe) - snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + struct hdac_bus *bus = azx_dev->bus; + + if (bus->access_sdnctl_in_dword) { + snd_hdac_stream_updatel(azx_dev, SD_CTL, + SD_CTL_DMA_START | SD_INT_MASK, 0); + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ + if (azx_dev->stripe) + snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + } else { + snd_hdac_stream_updateb(azx_dev, SD_CTL, + SD_CTL_DMA_START | SD_INT_MASK, 0); + snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */ + if (azx_dev->stripe) + snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); + } + azx_dev->running = false; } @@ -225,12 +240,16 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev) { unsigned char val; int dma_run_state; + struct hdac_bus *bus = azx_dev->bus; snd_hdac_stream_clear(azx_dev); dma_run_state = snd_hdac_stream_readb(azx_dev, SD_CTL) & SD_CTL_DMA_START; - snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET); + if (bus->access_sdnctl_in_dword) + snd_hdac_stream_updatel(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET); /* wait for hardware to report that the stream entered reset */ snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, (val & SD_CTL_STREAM_RESET), 3, 300); @@ -238,7 +257,10 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev) if (azx_dev->bus->dma_stop_delay && dma_run_state) udelay(azx_dev->bus->dma_stop_delay); - snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0); + if (bus->access_sdnctl_in_dword) + snd_hdac_stream_updatel(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0); + else + snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0); /* wait for hardware to report that the stream is out of reset */ snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, !(val & SD_CTL_STREAM_RESET), 3, 300); diff --git a/sound/pci/emu10k1/emupcm.c b/sound/pci/emu10k1/emupcm.c index 9023f3444d20..b8749e0131ad 100644 --- a/sound/pci/emu10k1/emupcm.c +++ b/sound/pci/emu10k1/emupcm.c @@ -1181,19 +1181,17 @@ static int snd_emu10k1_playback_open(struct snd_pcm_substream *substream) runtime->private_free = snd_emu10k1_pcm_free_substream; runtime->hw = snd_emu10k1_playback; err = snd_emu10k1_playback_set_constraints(runtime); - if (err < 0) { - kfree(epcm); - return err; - } + if (err < 0) + goto free_epcm; + if (emu->card_capabilities->emu_model) sample_rate = emu->emu1010.word_clock; else sample_rate = 48000; err = snd_pcm_hw_rule_noresample(runtime, sample_rate); - if (err < 0) { - kfree(epcm); - return err; - } + if (err < 0) + goto free_epcm; + mix = &emu->pcm_mixer[substream->number]; for (i = 0; i < 8; i++) mix->send_routing[0][i] = mix->send_routing[1][i] = mix->send_routing[2][i] = i; @@ -1204,6 +1202,10 @@ static int snd_emu10k1_playback_open(struct snd_pcm_substream *substream) mix->epcm = epcm; snd_emu10k1_pcm_mixer_notify(emu, substream->number, 1); return 0; + +free_epcm: + kfree(epcm); + return err; } static int snd_emu10k1_playback_close(struct snd_pcm_substream *substream) diff --git a/sound/sh/aica.c b/sound/sh/aica.c index 9438c3a68ee9..8196e1bf0416 100644 --- a/sound/sh/aica.c +++ b/sound/sh/aica.c @@ -564,10 +564,9 @@ static int snd_aica_probe(struct platform_device *devptr) return -ENOMEM; err = snd_card_new(&devptr->dev, index, SND_AICA_DRIVER, THIS_MODULE, 0, &dreamcastcard->card); - if (unlikely(err < 0)) { - kfree(dreamcastcard); - return err; - } + if (unlikely(err < 0)) + goto free_card; + strscpy(dreamcastcard->card->driver, "snd_aica"); strscpy(dreamcastcard->card->shortname, SND_AICA_DRIVER); strscpy(dreamcastcard->card->longname, @@ -593,6 +592,7 @@ static int snd_aica_probe(struct platform_device *devptr) return 0; freedreamcast: snd_card_free(dreamcastcard->card); +free_card: kfree(dreamcastcard); return err; } diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c index 0d977f4f758d..93a2182d4e86 100644 --- a/sound/soc/amd/acp-config.c +++ b/sound/soc/amd/acp-config.c @@ -360,5 +360,17 @@ struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_machines[] = { }; EXPORT_SYMBOL(snd_soc_acpi_amd_acp70_sof_machines); +struct snd_soc_acpi_mach snd_soc_acpi_amd_acp7x_sof_machines[] = { + { + .id = "AMDI1010", + .drv_name = "acp7x-dsp", + .pdata = &acp_quirk_data, + .fw_filename = "sof-acp7x.ri", + .sof_tplg_filename = "sof-acp7x.tplg", + }, + {}, +}; +EXPORT_SYMBOL(snd_soc_acpi_amd_acp7x_sof_machines); + MODULE_DESCRIPTION("AMD ACP Machine Configuration Module"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index e8b6819cc4b4..9726a9d33ec6 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -432,7 +432,7 @@ static int soc_card_dai_links_create(struct snd_soc_card *card) if (!soc_aux) return -ENOMEM; - ret = asoc_sdw_parse_sdw_endpoints(card, soc_aux, soc_dais, soc_ends, &num_confs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, soc_aux, soc_dais, soc_ends, &num_confs); if (ret < 0) return ret; diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c index a423853f3a97..963ce6fd4012 100644 --- a/sound/soc/amd/acp/acp-sdw-sof-mach.c +++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c @@ -303,7 +303,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) if (!sof_aux) return -ENOMEM; - ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_devs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_devs); if (ret < 0) return ret; diff --git a/sound/soc/amd/mach-config.h b/sound/soc/amd/mach-config.h index 5b6362103ca0..b602a983feb7 100644 --- a/sound/soc/amd/mach-config.h +++ b/sound/soc/amd/mach-config.h @@ -28,6 +28,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp63_sof_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_sdw_machines[]; +extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp7x_sof_machines[]; struct config_entry { u32 flags; diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index a405cc2c11a3..d6df7de70b27 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -528,6 +528,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Raider A18 HX A9WJG"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m15 R7 AMD"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/atmel/sam9x5_wm8731.c b/sound/soc/atmel/sam9x5_wm8731.c index 1b5ef4e9d2b8..a603e4a57d50 100644 --- a/sound/soc/atmel/sam9x5_wm8731.c +++ b/sound/soc/atmel/sam9x5_wm8731.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include diff --git a/sound/soc/au1x/psc-ac97.c b/sound/soc/au1x/psc-ac97.c index 94698e08a513..0f9d80883116 100644 --- a/sound/soc/au1x/psc-ac97.c +++ b/sound/soc/au1x/psc-ac97.c @@ -210,9 +210,7 @@ static int au1xpsc_ac97_hw_params(struct snd_pcm_substream *substream, { struct au1xpsc_audio_data *pscdata = snd_soc_dai_get_drvdata(dai); unsigned long r, ro, stat; - int chans, t, stype = substream->stream; - - chans = params_channels(params); + int t, stype = substream->stream; r = ro = __raw_readl(AC97_CFG(pscdata)); stat = __raw_readl(AC97_STAT(pscdata)); diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index 252f683be3c1..a41d0f50825a 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -615,8 +615,8 @@ config SND_SOC_AK4613 depends on I2C config SND_SOC_AK4619 - tristate "AKM AK4619 CODEC" - depends on I2C + tristate "AKM AK4619 CODEC" + depends on I2C config SND_SOC_AK4642 tristate "AKM AK4642 CODEC" @@ -1201,16 +1201,16 @@ config SND_SOC_JZ4725B_CODEC will be called snd-soc-jz4725b-codec. config SND_SOC_JZ4760_CODEC - depends on MACH_INGENIC || COMPILE_TEST - depends on OF - select REGMAP - tristate "Ingenic JZ4760 internal CODEC" - help - Enable support for the internal CODEC found in the JZ4760 SoC - from Ingenic. + depends on MACH_INGENIC || COMPILE_TEST + depends on OF + select REGMAP + tristate "Ingenic JZ4760 internal CODEC" + help + Enable support for the internal CODEC found in the JZ4760 SoC + from Ingenic. - This driver can also be built as a module. If so, the module - will be called snd-soc-jz4760-codec. + This driver can also be built as a module. If so, the module + will be called snd-soc-jz4760-codec. config SND_SOC_JZ4770_CODEC depends on MACH_INGENIC || COMPILE_TEST @@ -1888,7 +1888,7 @@ config SND_SOC_RT5670 depends on I2C config SND_SOC_RT5677 - tristate + tristate "Realtek RT5677 Codec" depends on I2C select REGMAP_I2C select REGMAP_IRQ @@ -2296,8 +2296,8 @@ config SND_SOC_TLV320ADC3XXX depends on I2C depends on GPIOLIB help - Enable support for Texas Instruments TLV320ADC3001 and TLV320ADC3101 - ADCs. + Enable support for Texas Instruments TLV320ADC3001 and TLV320ADC3101 + ADCs. config SND_SOC_TLV320AIC23 tristate @@ -2893,16 +2893,18 @@ config SND_SOC_TPA6130A2 depends on I2C config SND_SOC_LPASS_MACRO_COMMON - tristate + tristate config SND_SOC_LPASS_WSA_MACRO depends on COMMON_CLK + depends on PM_CLK select REGMAP_MMIO select SND_SOC_LPASS_MACRO_COMMON tristate "Qualcomm WSA Macro in LPASS(Low Power Audio SubSystem)" config SND_SOC_LPASS_VA_MACRO depends on COMMON_CLK + depends on PM_CLK select REGMAP_MMIO select SND_SOC_LPASS_MACRO_COMMON tristate "Qualcomm VA Macro in LPASS(Low Power Audio SubSystem)" diff --git a/sound/soc/codecs/adau1372-i2c.c b/sound/soc/codecs/adau1372-i2c.c index 4217b7fc349c..bdb3e3a8509d 100644 --- a/sound/soc/codecs/adau1372-i2c.c +++ b/sound/soc/codecs/adau1372-i2c.c @@ -7,7 +7,6 @@ */ #include -#include #include #include #include diff --git a/sound/soc/codecs/adau1372-spi.c b/sound/soc/codecs/adau1372-spi.c index 656bd1fabeb3..a12961e2fb1e 100644 --- a/sound/soc/codecs/adau1372-spi.c +++ b/sound/soc/codecs/adau1372-spi.c @@ -6,7 +6,6 @@ * Author: Lars-Peter Clausen */ -#include #include #include #include diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c index 879afeb81c42..cc174ec3a1f8 100644 --- a/sound/soc/codecs/adau1372.c +++ b/sound/soc/codecs/adau1372.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/sound/soc/codecs/adau1761-i2c.c b/sound/soc/codecs/adau1761-i2c.c index a329e5bddb99..ae73136d0a6e 100644 --- a/sound/soc/codecs/adau1761-i2c.c +++ b/sound/soc/codecs/adau1761-i2c.c @@ -7,7 +7,6 @@ */ #include -#include #include #include #include diff --git a/sound/soc/codecs/adau1761-spi.c b/sound/soc/codecs/adau1761-spi.c index 7c9242c2ff94..eb6f63d63783 100644 --- a/sound/soc/codecs/adau1761-spi.c +++ b/sound/soc/codecs/adau1761-spi.c @@ -6,7 +6,6 @@ * Author: Lars-Peter Clausen */ -#include #include #include #include diff --git a/sound/soc/codecs/adau1781-i2c.c b/sound/soc/codecs/adau1781-i2c.c index 0e7954148af7..3ab624417f12 100644 --- a/sound/soc/codecs/adau1781-i2c.c +++ b/sound/soc/codecs/adau1781-i2c.c @@ -7,7 +7,6 @@ */ #include -#include #include #include #include diff --git a/sound/soc/codecs/adau1781-spi.c b/sound/soc/codecs/adau1781-spi.c index 1a09633d5a88..0e6d42b10077 100644 --- a/sound/soc/codecs/adau1781-spi.c +++ b/sound/soc/codecs/adau1781-spi.c @@ -6,7 +6,6 @@ * Author: Lars-Peter Clausen */ -#include #include #include #include diff --git a/sound/soc/codecs/adau1977-i2c.c b/sound/soc/codecs/adau1977-i2c.c index fc7ed5c1dd74..d1c6c4ddf506 100644 --- a/sound/soc/codecs/adau1977-i2c.c +++ b/sound/soc/codecs/adau1977-i2c.c @@ -7,7 +7,6 @@ */ #include -#include #include #include #include diff --git a/sound/soc/codecs/adau1977-spi.c b/sound/soc/codecs/adau1977-spi.c index e7e95e5d1911..878cde9d1014 100644 --- a/sound/soc/codecs/adau1977-spi.c +++ b/sound/soc/codecs/adau1977-spi.c @@ -6,7 +6,6 @@ * Author: Lars-Peter Clausen */ -#include #include #include #include diff --git a/sound/soc/codecs/adau7118-hw.c b/sound/soc/codecs/adau7118-hw.c index 45a5d2dcc0f2..92b226b8b4bb 100644 --- a/sound/soc/codecs/adau7118-hw.c +++ b/sound/soc/codecs/adau7118-hw.c @@ -6,7 +6,6 @@ // Copyright 2019 Analog Devices Inc. #include -#include #include #include "adau7118.h" diff --git a/sound/soc/codecs/ak4104.c b/sound/soc/codecs/ak4104.c index a33cb329865c..6ea7cb78cd44 100644 --- a/sound/soc/codecs/ak4104.c +++ b/sound/soc/codecs/ak4104.c @@ -5,7 +5,6 @@ * Copyright (c) 2009 Daniel Mack */ -#include #include #include #include diff --git a/sound/soc/codecs/audio-iio-aux.c b/sound/soc/codecs/audio-iio-aux.c index 066e401912b0..964b9a5b2990 100644 --- a/sound/soc/codecs/audio-iio-aux.c +++ b/sound/soc/codecs/audio-iio-aux.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/cs4234.c b/sound/soc/codecs/cs4234.c index 89c424dd838b..a889fbd519a8 100644 --- a/sound/soc/codecs/cs4234.c +++ b/sound/soc/codecs/cs4234.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/cs4270.c b/sound/soc/codecs/cs4270.c index 47cb10eb21bb..47b2f903a32c 100644 --- a/sound/soc/codecs/cs4270.c +++ b/sound/soc/codecs/cs4270.c @@ -19,7 +19,6 @@ * - Power management is supported */ -#include #include #include #include diff --git a/sound/soc/codecs/cs42l42-sdw.c b/sound/soc/codecs/cs42l42-sdw.c index b8256ce0b8fb..ad1256910a18 100644 --- a/sound/soc/codecs/cs42l42-sdw.c +++ b/sound/soc/codecs/cs42l42-sdw.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/cs42l43.c b/sound/soc/codecs/cs42l43.c index f0d6ff0b2976..1d133577702e 100644 --- a/sound/soc/codecs/cs42l43.c +++ b/sound/soc/codecs/cs42l43.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/cs42xx8-i2c.c b/sound/soc/codecs/cs42xx8-i2c.c index 31debe2d8231..4427d3214dee 100644 --- a/sound/soc/codecs/cs42xx8-i2c.c +++ b/sound/soc/codecs/cs42xx8-i2c.c @@ -9,7 +9,6 @@ #include #include -#include #include #include diff --git a/sound/soc/codecs/cs42xx8-spi.c b/sound/soc/codecs/cs42xx8-spi.c index b86fe2fe771e..2e4b8e6c4081 100644 --- a/sound/soc/codecs/cs42xx8-spi.c +++ b/sound/soc/codecs/cs42xx8-spi.c @@ -6,7 +6,6 @@ * */ -#include #include #include #include diff --git a/sound/soc/codecs/cs4349.c b/sound/soc/codecs/cs4349.c index ced1270c4d68..6ac6d306b054 100644 --- a/sound/soc/codecs/cs4349.c +++ b/sound/soc/codecs/cs4349.c @@ -7,7 +7,6 @@ * Authors: Tim Howe */ -#include #include #include #include diff --git a/sound/soc/codecs/cs530x.c b/sound/soc/codecs/cs530x.c index 18b5ff75feec..2c7e33135911 100644 --- a/sound/soc/codecs/cs530x.c +++ b/sound/soc/codecs/cs530x.c @@ -1093,6 +1093,29 @@ static int cs530x_component_probe(struct snd_soc_component *component) return 0; } +static bool cs530x_mclk_freq_is_valid(struct cs530x_priv *cs530x, + unsigned int freq) +{ + /* + * All these chips support 48 kHz- and 44.1 kHz-related sample rates, + * but they differ in what MCLK frequency is required for achieving + * the sample rate. + */ + switch (cs530x->devtype) { + case CS4282: + case CS4302: + case CS4304: + case CS4308: + return freq == 49152000 || freq == 45158400; + case CS5302: + case CS5304: + case CS5308: + return freq == 24576000 || freq == 22579200; + } + + return false; +} + static int cs530x_set_sysclk(struct snd_soc_component *component, int clk_id, int source, unsigned int freq, int dir) { @@ -1101,11 +1124,7 @@ static int cs530x_set_sysclk(struct snd_soc_component *component, int clk_id, switch (source) { case CS530X_SYSCLK_SRC_MCLK: - switch (freq) { - case CS530X_SYSCLK_REF_45_1MHZ: - case CS530X_SYSCLK_REF_49_1MHZ: - break; - default: + if (!cs530x_mclk_freq_is_valid(cs530x, freq)) { dev_err(component->dev, "Invalid MCLK source rate %d\n", freq); return -EINVAL; } diff --git a/sound/soc/codecs/cs530x.h b/sound/soc/codecs/cs530x.h index 1e2f6a7a589c..18aa4dfd0c86 100644 --- a/sound/soc/codecs/cs530x.h +++ b/sound/soc/codecs/cs530x.h @@ -200,12 +200,6 @@ /* IN_VOL_CTL5 and OUT_VOL_CTL5 */ #define CS530X_INOUT_VU BIT(0) -/* MCLK Reference Source Frequency */ -/* 41KHz related */ -#define CS530X_SYSCLK_REF_45_1MHZ 45158400 -/* 48KHz related */ -#define CS530X_SYSCLK_REF_49_1MHZ 49152000 - /* System Clock Source */ #define CS530X_SYSCLK_SRC_MCLK 0 #define CS530X_SYSCLK_SRC_PLL 1 diff --git a/sound/soc/codecs/es8316.c b/sound/soc/codecs/es8316.c index 6a428387e496..3abe77423f29 100644 --- a/sound/soc/codecs/es8316.c +++ b/sound/soc/codecs/es8316.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/es8323.c b/sound/soc/codecs/es8323.c index d067f7bda03a..b926340256be 100644 --- a/sound/soc/codecs/es8323.c +++ b/sound/soc/codecs/es8323.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c index 9838fe42cb6f..aaa6646ad4c5 100644 --- a/sound/soc/codecs/es8328.c +++ b/sound/soc/codecs/es8328.c @@ -405,6 +405,11 @@ static const struct snd_soc_dapm_route es8328_dapm_routes[] = { { "Mic Bias", NULL, "Mic Bias Gen" }, + { "LINPUT1", NULL, "Mic Bias" }, + { "RINPUT1", NULL, "Mic Bias" }, + { "LINPUT2", NULL, "Mic Bias" }, + { "RINPUT2", NULL, "Mic Bias" }, + { "Left Mixer", NULL, "Left DAC" }, { "Left Mixer", "Left Bypass Switch", "Left Line Mux" }, { "Left Mixer", "Right Playback Switch", "Right DAC" }, diff --git a/sound/soc/codecs/es8389.c b/sound/soc/codecs/es8389.c index 3484c87853cb..0c7567e2ffc2 100644 --- a/sound/soc/codecs/es8389.c +++ b/sound/soc/codecs/es8389.c @@ -36,6 +36,10 @@ struct es8389_private { unsigned int sysclk; int mastermode; + u8 hpfl; + u8 hpfr; + u32 hpf_freq; + u32 capture_rate; u8 mclk_src; u8 vddd; int version; @@ -50,10 +54,29 @@ static const char * const es8389_core_supplies[] = { static bool es8389_volatile_register(struct device *dev, unsigned int reg) { - if ((reg <= 0xff)) - return true; - else + switch (reg) { + case ES8389_ADCL_VOL: + case ES8389_ADCR_VOL: + case ES8389_MIC1_GAIN: + case ES8389_MIC2_GAIN: + case ES8389_DACL_VOL: + case ES8389_DACR_VOL: + case ES8389_ALC_ON: + case ES8389_ALC_CTL: + case ES8389_ALC_TARGET: + case ES8389_ALC_GAIN: + case ES8389_ADC_MUTE: + case ES8389_OSR_VOL: + case ES8389_DAC_INV: + case ES8389_MIX_VOL: + case ES8389_DAC_MIX: + case ES8389_ADC_RESET: + case ES8389_ADC_MODE: + case ES8389_DMIC_EN: return false; + default: + return true; + } } static const DECLARE_TLV_DB_SCALE(dac_vol_tlv, -9550, 50, 0); @@ -63,6 +86,92 @@ static const DECLARE_TLV_DB_SCALE(mix_vol_tlv, -9500, 100, 0); static const DECLARE_TLV_DB_SCALE(alc_target_tlv, -3200, 200, 0); static const DECLARE_TLV_DB_SCALE(alc_max_level, -3200, 200, 0); +static const u32 hpf_table[10][10] = { + {1020, 754, 624, 559, 527, 511, 502, 498, 497, 496}, + {754, 495, 368, 306, 274, 259, 251, 247, 246, 244}, + {624, 368, 243, 182, 151, 136, 128, 124, 123, 121}, + {559, 306, 182, 120, 90, 75, 68, 63, 62, 60}, + {527, 274, 151, 90, 60, 45, 38, 33, 32, 31}, + {511, 259, 136, 75, 45, 30, 23, 19, 18, 17}, + {502, 251, 128, 68, 38, 23, 16, 13, 11, 11}, + {498, 247, 124, 63, 33, 19, 13, 10, 8, 8}, + {497, 246, 123, 62, 32, 18, 11, 8, 8, 0}, + {496, 244, 121, 60, 31, 17, 11, 8, 0, 0} +}; + +static bool find_best_hpf_freq(u32 target_hz, u8 *hpf1, u8 *hpf2, u32 *out) +{ + int best_row = -1, best_col = -1; + u32 min_diff = U32_MAX; + u32 f, diff; + int i, j; + + if ((target_hz > 1020) | (target_hz < 0)) + return false; + + for (i = 0; i < 10; i++) { + for (j = i; j < 10; j++) { + f = hpf_table[i][j]; + + diff = (target_hz > f) ? (target_hz - f) : (f - target_hz); + if (diff < min_diff) { + min_diff = diff; + best_row = i; + best_col = j; + *out = f; + } + } + } + + *hpf1 = best_col + ES8389_HPF_OFFSET; + *hpf2 = best_row + ES8389_HPF_OFFSET; + + return true; +} + +static int es8389_hpf_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct es8389_private *es8389 = snd_soc_component_get_drvdata(component); + + ucontrol->value.integer.value[0] = es8389->hpf_freq; + return 0; +} + +static int es8389_hpf_set(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct es8389_private *es8389 = snd_soc_component_get_drvdata(component); + u32 freq; + bool hpf; + + if (es8389->hpf_freq == ucontrol->value.integer.value[0]) + return 0; + + if (es8389->capture_rate) { + freq = (ucontrol->value.integer.value[0] * 48000) / es8389->capture_rate; + + hpf = find_best_hpf_freq(freq, &es8389->hpfl, &es8389->hpfr, &es8389->hpf_freq); + if (!hpf) + return -EBUSY; + + if (es8389->hpf_freq != ucontrol->value.integer.value[0]) + dev_dbg(component->dev, "At the %u Hz sampling rate, %ld Hz could not be obtained." + "the frequency has been set to the closest value, %u Hz\n", + es8389->capture_rate, ucontrol->value.integer.value[0], es8389->hpf_freq); + + regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, es8389->hpfl); + regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, es8389->hpfr); + } else { + es8389->hpf_freq = ucontrol->value.integer.value[0]; + dev_dbg(component->dev, "PCM_STREAM_CAPTURE is not active.retain the input frequency\n"); + } + + return 1; +} + static int es8389_dmic_set(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { @@ -143,6 +252,16 @@ static const struct soc_enum alc_ramprate = static const struct soc_enum alc_winsize = SOC_ENUM_SINGLE(ES8389_ALC_CTL, 0, 16, winsize); +static const char *const es8389_adcl_mux_txt[] = { + "Normal", + "ADC2 channel to ADC1 channel", +}; + +static const char *const es8389_adcr_mux_txt[] = { + "Normal", + "ADC1 channel to ADC2 channel", +}; + static const char *const es8389_outl_mux_txt[] = { "Normal", "DAC2 channel to DAC1 channel", @@ -170,6 +289,20 @@ static const unsigned int es8389_pga_values[] = { 1, 5, 6 }; +static const struct soc_enum es8389_adcl_mux_enum = + SOC_ENUM_SINGLE(ES8389_ADC_MODE, 5, + ARRAY_SIZE(es8389_adcl_mux_txt), es8389_adcl_mux_txt); + +static const struct snd_kcontrol_new es8389_adcl_mux_controls = + SOC_DAPM_ENUM("INPUTL MUX", es8389_adcl_mux_enum); + +static const struct soc_enum es8389_adcr_mux_enum = + SOC_ENUM_SINGLE(ES8389_ADC_MODE, 4, + ARRAY_SIZE(es8389_adcr_mux_txt), es8389_adcr_mux_txt); + +static const struct snd_kcontrol_new es8389_adcr_mux_controls = + SOC_DAPM_ENUM("INPUTR MUX", es8389_adcr_mux_enum); + static const struct soc_enum es8389_outl_mux_enum = SOC_ENUM_SINGLE(ES8389_DAC_MIX, 5, ARRAY_SIZE(es8389_outl_mux_txt), es8389_outl_mux_txt); @@ -240,6 +373,8 @@ static const struct snd_kcontrol_new es8389_snd_controls[] = { SOC_DOUBLE("ADC OSR Volume ON Switch", ES8389_ADC_MUTE, 6, 7, 1, 0), SOC_SINGLE_TLV("ADC OSR Volume", ES8389_OSR_VOL, 0, 0xFF, 0, adc_vol_tlv), SOC_DOUBLE("ADC OUTPUT Invert Switch", ES8389_ADC_HPF2, 5, 6, 1, 0), + SOC_SINGLE_EXT("ADC HPF Freq Select", SND_SOC_NOPM, 0, 1020, 0, + es8389_hpf_get, es8389_hpf_set), SOC_SINGLE_TLV("DACL Playback Volume", ES8389_DACL_VOL, 0, 0xFF, 0, dac_vol_tlv), SOC_SINGLE_TLV("DACR Playback Volume", ES8389_DACR_VOL, 0, 0xFF, 0, dac_vol_tlv), @@ -298,6 +433,8 @@ static const struct snd_soc_dapm_widget es8389_dapm_widgets[] = { &es8389_adc_mixer_controls[0], ARRAY_SIZE(es8389_adc_mixer_controls)), SND_SOC_DAPM_MUX("ADC MUX", SND_SOC_NOPM, 0, 0, &es8389_dmic_mux_controls), + SND_SOC_DAPM_MUX("INPUTL MUX", SND_SOC_NOPM, 0, 0, &es8389_adcl_mux_controls), + SND_SOC_DAPM_MUX("INPUTR MUX", SND_SOC_NOPM, 0, 0, &es8389_adcr_mux_controls), SND_SOC_DAPM_MUX("OUTL MUX", SND_SOC_NOPM, 0, 0, &es8389_outl_mux_controls), SND_SOC_DAPM_MUX("OUTR MUX", SND_SOC_NOPM, 0, 0, &es8389_outr_mux_controls), @@ -311,10 +448,15 @@ static const struct snd_soc_dapm_route es8389_dapm_routes[] = { {"ADCL", NULL, "PGAL"}, {"ADCR", NULL, "PGAR"}, + {"INPUTL MUX", "Normal", "ADCL"}, + {"INPUTL MUX", "ADC2 channel to ADC1 channel", "ADCR"}, + {"INPUTR MUX", "Normal", "ADCR"}, + {"INPUTR MUX", "ADC1 channel to ADC2 channel", "ADCL"}, + {"ADC Mixer", "DACL ADCL Mixer", "DACL"}, {"ADC Mixer", "DACR ADCR Mixer", "DACR"}, - {"ADC Mixer", NULL, "ADCL"}, - {"ADC Mixer", NULL, "ADCR"}, + {"ADC Mixer", NULL, "INPUTL MUX"}, + {"ADC Mixer", NULL, "INPUTR MUX"}, {"ADC MUX", "AMIC", "ADC Mixer"}, {"ADC MUX", "DMIC", "DMIC"}, @@ -415,52 +557,54 @@ static const struct _coeff_div coeff_div[] = { {36, 576000, 16000, 0x00, 0x55, 0x84, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {48, 768000, 16000, 0x02, 0x57, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {50, 800000, 16000, 0x00, 0x7E, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {64, 1024000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {72, 1152000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {64, 1024000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {72, 1152000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {96, 1536000, 16000, 0x02, 0x55, 0x84, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {128, 2048000, 16000, 0x00, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {144, 2304000, 16000, 0x00, 0x51, 0x00, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {150, 2400000, 16000, 0x02, 0x7E, 0x01, 0xC9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {192, 3072000, 16000, 0x02, 0x65, 0x25, 0xE0, 0x00, 0xE1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {256, 4096000, 16000, 0x00, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {256, 4096000, 16000, 0x00, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {300, 4800000, 16000, 0x02, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {384, 6144000, 16000, 0x02, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {512, 8192000, 16000, 0x01, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {512, 8192000, 16000, 0x01, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {750, 12000000, 16000, 0x0E, 0x7E, 0x01, 0xC9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {768, 12288000, 16000, 0x02, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {1024, 16384000, 16000, 0x03, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {768, 12288000, 16000, 0x02, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {1024, 16384000, 16000, 0x03, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {1152, 18432000, 16000, 0x08, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {1200, 19200000, 16000, 0x0B, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {1500, 24000000, 16000, 0x0E, 0x26, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, - {1536, 24576000, 16000, 0x05, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, + {1536, 24576000, 16000, 0x05, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {1625, 26000000, 16000, 0x40, 0x6E, 0x05, 0xC8, 0x01, 0xC2, 0x90, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2}, {800, 19200000, 24000, 0x07, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x1A, 0x49, 0x14, 2, 2}, {375, 12000000, 32000, 0x0E, 0x2E, 0x05, 0xC8, 0x00, 0xC2, 0x80, 0x40, 0x01, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 0}, - {600, 19200000, 32000, 0x05, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 2}, - {32, 1411200, 44100, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {600, 19200000, 32000, 0x05, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 2}, + {32, 1411200, 44100, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {64, 2822400, 44100, 0x00, 0x51, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {128, 5644800, 44100, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {256, 11289600, 44100, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {512, 22579200, 44100, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {32, 1536000, 48000, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {128, 5644800, 44100, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {256, 11289600, 44100, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {512, 22579200, 44100, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {32, 1536000, 48000, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {48, 2304000, 48000, 0x02, 0x55, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {50, 2400000, 48000, 0x00, 0x76, 0x01, 0xC8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {64, 3072000, 48000, 0x00, 0x51, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {100, 4800000, 48000, 0x00, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {100, 4800000, 48000, 0x00, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {125, 6000000, 48000, 0x04, 0x6E, 0x05, 0xC8, 0x10, 0xC2, 0x80, 0x00, 0x01, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {128, 6144000, 48000, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {200, 9600000, 48000, 0x01, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {128, 6144000, 48000, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {200, 9600000, 48000, 0x01, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {250, 12000000, 48000, 0x04, 0x76, 0x01, 0xC8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {256, 12288000, 48000, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {384, 18432000, 48000, 0x02, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {400, 19200000, 48000, 0x03, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {500, 24000000, 48000, 0x04, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, - {512, 24576000, 48000, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {256, 12288000, 48000, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {384, 18432000, 48000, 0x02, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {400, 19200000, 48000, 0x03, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {500, 24000000, 48000, 0x04, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, + {512, 24576000, 48000, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {800, 38400000, 48000, 0x18, 0x45, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2}, {128, 11289600, 88200, 0x00, 0x50, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x32, 0x89, 0x25, 2, 2}, - {64, 6144000, 96000, 0x00, 0x41, 0x00, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2}, + {64, 6144000, 96000, 0x00, 0x41, 0x00, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2}, {96, 9216000, 96000, 0x02, 0x43, 0x00, 0xC0, 0x10, 0xC0, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2}, {256, 24576000, 96000, 0x00, 0x40, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2}, {128, 24576000, 192000, 0x00, 0x50, 0x00, 0xC0, 0x18, 0xC1, 0x81, 0xC0, 0x00, 0x8F, 0x7F, 0xBF, 0xC0, 0x3F, 0x7F, 0x80, 0x12, 0xC0, 0x3F, 0xF9, 0x3F, 2, 2}, + {64, 12288000, 192000, 0x00, 0x41, 0x00, 0xC0, 0x18, 0xC1, 0x80, 0x00, 0x00, 0x8F, 0x7F, 0xEF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x3F, 0xF9, 0x3F, 1, 0}, }; static inline int get_coeff(u8 vddd, u8 dmic, int mclk, int rate) @@ -564,6 +708,8 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream, int coeff, ret; u8 dmic_enable, state = 0; unsigned int regv; + u32 freq; + bool hpf; switch (params_format(params)) { case SNDRV_PCM_FORMAT_S16_LE: @@ -590,7 +736,7 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream, if (es8389->mclk_src == ES8389_SCLK_PIN) { regmap_update_bits(es8389->regmap, ES8389_MASTER_CLK, - ES8389_MCLK_SOURCE, es8389->mclk_src); + ES8389_MCLK_MASK, ES8389_MCLK_FROM_SCLK); es8389->sysclk = params_channels(params) * params_width(params) * params_rate(params); } @@ -641,6 +787,28 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream, dev_warn(component->dev, "Clock coefficients do not match"); } + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { + es8389->capture_rate = params_rate(params); + freq = (es8389->hpf_freq * 48000) / params_rate(params); + hpf = find_best_hpf_freq(freq, &es8389->hpfl, &es8389->hpfr, &es8389->hpf_freq); + if (!hpf) { + dev_err(component->dev, "The HPF frequency is invalid\n"); + return -EINVAL; + } + } + + return 0; +} + +static int es8389_pcm_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct es8389_private *es8389 = snd_soc_component_get_drvdata(component); + + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + es8389->capture_rate = 0; + return 0; } @@ -721,8 +889,8 @@ static int es8389_mute(struct snd_soc_dai *dai, int mute, int direction) regmap_update_bits(es8389->regmap, ES8389_DAC_FORMAT_MUTE, 0x03, 0x00); } else { - regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, 0x0a); - regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, 0x0a); + regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, es8389->hpfl); + regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, es8389->hpfr); regmap_update_bits(es8389->regmap, ES8389_ADC_FORMAT_MUTE, 0x03, 0x00); } @@ -738,6 +906,7 @@ static int es8389_mute(struct snd_soc_dai *dai, int mute, int direction) static const struct snd_soc_dai_ops es8389_ops = { .hw_params = es8389_pcm_hw_params, + .hw_free = es8389_pcm_hw_free, .set_fmt = es8389_set_dai_fmt, .set_sysclk = es8389_set_dai_sysclk, .set_tdm_slot = es8389_set_tdm_slot, @@ -771,7 +940,7 @@ static void es8389_init(struct snd_soc_component *component) regmap_read(es8389->regmap, ES8389_MAX_REGISTER, ®); es8389->version = reg; - regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x00); + regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x56); regmap_write(es8389->regmap, ES8389_RESET, 0x7E); regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x38); regmap_write(es8389->regmap, ES8389_ADC_HPF1, 0x64); @@ -823,7 +992,7 @@ static void es8389_init(struct snd_soc_component *component) regmap_write(es8389->regmap, ES8389_SCLK_DIV, 0x04); regmap_write(es8389->regmap, ES8389_LRCK_DIV1, 0x01); regmap_write(es8389->regmap, ES8389_LRCK_DIV2, 0x00); - regmap_write(es8389->regmap, ES8389_OSC_CLK, 0x00); + regmap_write(es8389->regmap, ES8389_OSC_CLK, 0x10); regmap_write(es8389->regmap, ES8389_ADC_OSR, 0x1F); regmap_write(es8389->regmap, ES8389_ADC_DSP, 0x7F); regmap_write(es8389->regmap, ES8389_ADC_MUTE, 0xC0); @@ -861,13 +1030,13 @@ static int es8389_resume(struct snd_soc_component *component) regcache_cache_only(es8389->regmap, false); regcache_cache_bypass(es8389->regmap, true); regmap_read(es8389->regmap, ES8389_RESET, ®v); - regcache_cache_bypass(es8389->regmap, false); if (regv == 0xff) es8389_init(component); else es8389_set_bias_level(component, SND_SOC_BIAS_ON); + regcache_cache_bypass(es8389->regmap, false); regcache_sync(es8389->regmap); return 0; @@ -913,6 +1082,7 @@ static int es8389_probe(struct snd_soc_component *component) return ret; } + es8389->hpf_freq = ES8389_HPF_DEFAULT; es8389_init(component); es8389_set_bias_level(component, SND_SOC_BIAS_STANDBY); diff --git a/sound/soc/codecs/es8389.h b/sound/soc/codecs/es8389.h index d21e72f876a6..de39e9bf96e9 100644 --- a/sound/soc/codecs/es8389.h +++ b/sound/soc/codecs/es8389.h @@ -106,6 +106,9 @@ #define ES8389_MIC_SEL_MASK (7 << 4) #define ES8389_MIC_DEFAULT (1 << 4) +#define ES8389_HPF_DEFAULT 16 +#define ES8389_HPF_OFFSET 4 + #define ES8389_MASTER_MODE_EN (1 << 0) #define ES8389_TDM_OFF (0 << 0) @@ -116,9 +119,11 @@ #define ES8389_TDM_SLOT (0x70 << 0) #define ES8389_TDM_SHIFT 4 -#define ES8389_MCLK_SOURCE (1 << 6) -#define ES8389_MCLK_PIN (1 << 6) -#define ES8389_SCLK_PIN (0 << 6) +#define ES8389_MCLK_MASK (3 << 6) +#define ES8389_MCLK_FROM_SCLK (1 << 6) +#define ES8389_MCLK_SOURCE ES8389_MCLK_PIN +#define ES8389_MCLK_PIN 0 +#define ES8389_SCLK_PIN 1 /* ES8389_FMT */ #define ES8389_S24_LE (0 << 5) diff --git a/sound/soc/codecs/es9356.c b/sound/soc/codecs/es9356.c index 670e918b56a4..1122455aab77 100644 --- a/sound/soc/codecs/es9356.c +++ b/sound/soc/codecs/es9356.c @@ -7,7 +7,6 @@ // #include -#include #include #include #include @@ -1111,8 +1110,10 @@ static int es9356_sdca_dev_resume(struct device *dev) es9356->disable_irq = false; ret = sdw_slave_wait_for_init(slave, es9356_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(es9356->regmap, false); regcache_sync(es9356->regmap); diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c index 680e341aa7f1..1ab5f8a26e03 100644 --- a/sound/soc/codecs/hdac_hda.c +++ b/sound/soc/codecs/hdac_hda.c @@ -642,10 +642,8 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev) &hdac_hda_codec, hdac_hda_dais, ARRAY_SIZE(hdac_hda_dais)); - if (ret < 0) { + if (ret < 0) dev_err(&hdev->dev, "%s: failed to register HDA codec %d\n", __func__, ret); - return ret; - } snd_hdac_ext_bus_link_put(hdev->bus, hlink); diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c index 3220f9226e0b..38073a70fa61 100644 --- a/sound/soc/codecs/hdac_hdmi.c +++ b/sound/soc/codecs/hdac_hdmi.c @@ -1866,8 +1866,10 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev) snd_hdac_ext_bus_link_get(hdev->bus, hlink); hdmi_priv = devm_kzalloc(&hdev->dev, sizeof(*hdmi_priv), GFP_KERNEL); - if (hdmi_priv == NULL) + if (hdmi_priv == NULL) { + snd_hdac_ext_bus_link_put(hdev->bus, hlink); return -ENOMEM; + } snd_hdac_register_chmap_ops(hdev, &hdmi_priv->chmap); hdmi_priv->chmap.ops.get_chmap = hdac_hdmi_get_chmap; @@ -1876,8 +1878,10 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev) hdmi_priv->chmap.ops.get_spk_alloc = hdac_hdmi_get_spk_alloc; hdmi_priv->hdev = hdev; - if (!hdac_id) + if (!hdac_id) { + snd_hdac_ext_bus_link_put(hdev->bus, hlink); return -ENODEV; + } if (hdac_id->driver_data) hdmi_priv->drv_data = @@ -1902,6 +1906,8 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev) if (ret < 0) { dev_err(&hdev->dev, "Failed in parse and map nid with err: %d\n", ret); + snd_hdac_ext_bus_link_put(hdev->bus, hlink); + snd_hdac_display_power(hdev->bus, hdev->addr, false); return ret; } snd_hdac_refresh_widgets(hdev); diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c index 528d5b167ecf..946051698420 100644 --- a/sound/soc/codecs/lpass-va-macro.c +++ b/sound/soc/codecs/lpass-va-macro.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -244,6 +245,11 @@ static const struct va_macro_data sm8250_va_data = { .version = LPASS_CODEC_VERSION_1_0, }; +static const struct va_macro_data sc7280_va_data = { + .has_swr_master = false, + .has_npl_clk = false, +}; + static const struct va_macro_data sm8450_va_data = { .has_swr_master = true, .has_npl_clk = true, @@ -1346,34 +1352,43 @@ static int fsgen_gate_enable(struct clk_hw *hw) { struct va_macro *va = to_va_macro(hw); struct regmap *regmap = va->regmap; - int ret; + int ret, rpm_ret; - if (va->has_swr_master) { - ret = clk_prepare_enable(va->mclk); - if (ret) - return ret; - } + ret = pm_runtime_resume_and_get(va->dev); + if (ret < 0) + return ret; ret = va_macro_mclk_enable(va, true); + if (ret) { + rpm_ret = pm_runtime_put_autosuspend(va->dev); + if (rpm_ret < 0) + dev_warn(va->dev, + "runtime PM put failed in fsgen enable unwind: %d\n", + rpm_ret); + return ret; + } if (va->has_swr_master) regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, CDC_VA_SWR_CLK_EN_MASK, CDC_VA_SWR_CLK_ENABLE); - return ret; + return 0; } static void fsgen_gate_disable(struct clk_hw *hw) { struct va_macro *va = to_va_macro(hw); struct regmap *regmap = va->regmap; + int ret; if (va->has_swr_master) regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, CDC_VA_SWR_CLK_EN_MASK, 0x0); va_macro_mclk_enable(va, false); - if (va->has_swr_master) - clk_disable_unprepare(va->mclk); + + ret = pm_runtime_put_autosuspend(va->dev); + if (ret < 0) + dev_warn(va->dev, "runtime PM put failed in fsgen disable: %d\n", ret); } static int fsgen_gate_is_enabled(struct clk_hw *hw) @@ -1534,6 +1549,7 @@ static int va_macro_probe(struct platform_device *pdev) void __iomem *base; u32 sample_rate = 0; int ret; + int rpm_ret; va = devm_kzalloc(dev, sizeof(*va), GFP_KERNEL); if (!va) @@ -1601,22 +1617,24 @@ static int va_macro_probe(struct platform_device *pdev) clk_set_rate(va->npl, 2 * VA_MACRO_MCLK_FREQ); } - ret = clk_prepare_enable(va->macro); + ret = devm_pm_clk_create(dev); if (ret) goto err; - ret = clk_prepare_enable(va->dcodec); - if (ret) - goto err_dcodec; + ret = of_pm_clk_add_clks(dev); + if (ret < 0) + goto err; - ret = clk_prepare_enable(va->mclk); + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + ret = devm_pm_runtime_enable(dev); if (ret) - goto err_mclk; + goto err; - if (va->has_npl_clk) { - ret = clk_prepare_enable(va->npl); - if (ret) - goto err_npl; + rpm_ret = pm_runtime_resume_and_get(dev); + if (rpm_ret < 0) { + ret = rpm_ret; + goto err; } /** @@ -1629,7 +1647,7 @@ static int va_macro_probe(struct platform_device *pdev) /* read version from register */ ret = va_macro_set_lpass_codec_version(va); if (ret) - goto err_clkout; + goto err_rpm_put; } if (va->has_swr_master) { @@ -1659,35 +1677,28 @@ static int va_macro_probe(struct platform_device *pdev) va_macro_dais, ARRAY_SIZE(va_macro_dais)); if (ret) - goto err_clkout; - - pm_runtime_set_autosuspend_delay(dev, 3000); - pm_runtime_use_autosuspend(dev); - pm_runtime_mark_last_busy(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + goto err_rpm_put; ret = va_macro_register_fsgen_output(va); if (ret) - goto err_clkout; + goto err_rpm_put; va->fsgen = devm_clk_hw_get_clk(dev, &va->hw, "fsgen"); if (IS_ERR(va->fsgen)) { ret = PTR_ERR(va->fsgen); - goto err_clkout; + goto err_rpm_put; } + rpm_ret = pm_runtime_put_autosuspend(dev); + if (rpm_ret < 0) + dev_warn(dev, "runtime PM put failed after probe: %d\n", rpm_ret); + return 0; -err_clkout: - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); -err_npl: - clk_disable_unprepare(va->mclk); -err_mclk: - clk_disable_unprepare(va->dcodec); -err_dcodec: - clk_disable_unprepare(va->macro); +err_rpm_put: + rpm_ret = pm_runtime_put_sync_suspend(dev); + if (rpm_ret < 0) + dev_warn(dev, "runtime PM sync suspend failed in probe unwind: %d\n", rpm_ret); err: lpass_macro_pds_exit(va->pds); @@ -1698,27 +1709,23 @@ static void va_macro_remove(struct platform_device *pdev) { struct va_macro *va = dev_get_drvdata(&pdev->dev); - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); - - clk_disable_unprepare(va->mclk); - clk_disable_unprepare(va->dcodec); - clk_disable_unprepare(va->macro); - lpass_macro_pds_exit(va->pds); } static int va_macro_runtime_suspend(struct device *dev) { struct va_macro *va = dev_get_drvdata(dev); + int ret; regcache_cache_only(va->regmap, true); - regcache_mark_dirty(va->regmap); - if (va->has_npl_clk) - clk_disable_unprepare(va->npl); + ret = pm_clk_suspend(dev); + if (ret) { + regcache_cache_only(va->regmap, false); + return ret; + } - clk_disable_unprepare(va->mclk); + regcache_mark_dirty(va->regmap); return 0; } @@ -1726,25 +1733,28 @@ static int va_macro_runtime_suspend(struct device *dev) static int va_macro_runtime_resume(struct device *dev) { struct va_macro *va = dev_get_drvdata(dev); - int ret; + int ret, sret; - ret = clk_prepare_enable(va->mclk); + ret = pm_clk_resume(dev); if (ret) { - dev_err(va->dev, "unable to prepare mclk\n"); + regcache_cache_only(va->regmap, true); + regcache_mark_dirty(va->regmap); return ret; } - if (va->has_npl_clk) { - ret = clk_prepare_enable(va->npl); - if (ret) { - clk_disable_unprepare(va->mclk); - dev_err(va->dev, "unable to prepare npl\n"); - return ret; - } - } - regcache_cache_only(va->regmap, false); - regcache_sync(va->regmap); + + ret = regcache_sync(va->regmap); + if (ret) { + regcache_cache_only(va->regmap, true); + regcache_mark_dirty(va->regmap); + sret = pm_clk_suspend(dev); + if (sret) + dev_err(va->dev, + "failed to suspend clocks after regcache sync failure: %d\n", + sret); + return ret; + } return 0; } @@ -1755,7 +1765,7 @@ static const struct dev_pm_ops va_macro_pm_ops = { }; static const struct of_device_id va_macro_dt_match[] = { - { .compatible = "qcom,sc7280-lpass-va-macro", .data = &sm8250_va_data }, + { .compatible = "qcom,sc7280-lpass-va-macro", .data = &sc7280_va_data }, { .compatible = "qcom,sm6115-lpass-va-macro", .data = &sm8450_va_data }, { .compatible = "qcom,sm8250-lpass-va-macro", .data = &sm8250_va_data }, { .compatible = "qcom,sm8450-lpass-va-macro", .data = &sm8450_va_data }, diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c index 5ad0448af649..f511816aa4a0 100644 --- a/sound/soc/codecs/lpass-wsa-macro.c +++ b/sound/soc/codecs/lpass-wsa-macro.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -2529,15 +2530,13 @@ static const struct snd_soc_dapm_route wsa_audio_map[] = { static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable) { struct regmap *regmap = wsa->regmap; + int ret; if (enable) { - int ret; - - ret = clk_prepare_enable(wsa->mclk); - if (ret) { - dev_err(wsa->dev, "failed to enable mclk\n"); + ret = pm_runtime_resume_and_get(wsa->dev); + if (ret < 0) return ret; - } + wsa_macro_mclk_enable(wsa, true); regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, @@ -2548,7 +2547,10 @@ static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable) regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, CDC_WSA_SWR_CLK_EN_MASK, 0); wsa_macro_mclk_enable(wsa, false); - clk_disable_unprepare(wsa->mclk); + + ret = pm_runtime_put_autosuspend(wsa->dev); + if (ret < 0) + dev_warn(wsa->dev, "runtime PM put failed: %d\n", ret); } return 0; @@ -2656,7 +2658,7 @@ static int wsa_macro_register_mclk_output(struct wsa_macro *wsa) init.num_parents = 1; wsa->hw.init = &init; hw = &wsa->hw; - ret = clk_hw_register(wsa->dev, hw); + ret = devm_clk_hw_register(wsa->dev, hw); if (ret) return ret; @@ -2774,25 +2776,23 @@ static int wsa_macro_probe(struct platform_device *pdev) clk_set_rate(wsa->mclk, WSA_MACRO_MCLK_FREQ); clk_set_rate(wsa->npl, WSA_MACRO_MCLK_FREQ); - ret = clk_prepare_enable(wsa->macro); + ret = devm_pm_clk_create(dev); if (ret) - goto err; - - ret = clk_prepare_enable(wsa->dcodec); - if (ret) - goto err_dcodec; + return ret; - ret = clk_prepare_enable(wsa->mclk); - if (ret) - goto err_mclk; + ret = of_pm_clk_add_clks(dev); + if (ret < 0) + return ret; - ret = clk_prepare_enable(wsa->npl); + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + ret = devm_pm_runtime_enable(dev); if (ret) - goto err_npl; + return ret; - ret = clk_prepare_enable(wsa->fsgen); - if (ret) - goto err_fsgen; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + return ret; /* reset swr ip */ regmap_update_bits(wsa->regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, @@ -2809,56 +2809,37 @@ static int wsa_macro_probe(struct platform_device *pdev) wsa_macro_dai, ARRAY_SIZE(wsa_macro_dai)); if (ret) - goto err_clkout; - - pm_runtime_set_autosuspend_delay(dev, 3000); - pm_runtime_use_autosuspend(dev); - pm_runtime_mark_last_busy(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + goto err_rpm_put; ret = wsa_macro_register_mclk_output(wsa); if (ret) - goto err_clkout; + goto err_rpm_put; - return 0; + ret = pm_runtime_put_autosuspend(dev); + if (ret < 0) + dev_warn(dev, "runtime PM put failed after probe: %d\n", ret); -err_clkout: - clk_disable_unprepare(wsa->fsgen); -err_fsgen: - clk_disable_unprepare(wsa->npl); -err_npl: - clk_disable_unprepare(wsa->mclk); -err_mclk: - clk_disable_unprepare(wsa->dcodec); -err_dcodec: - clk_disable_unprepare(wsa->macro); -err: + return 0; +err_rpm_put: + if (pm_runtime_put_sync_suspend(dev) < 0) + dev_warn(dev, "runtime PM sync suspend failed in probe unwind\n"); return ret; - -} - -static void wsa_macro_remove(struct platform_device *pdev) -{ - struct wsa_macro *wsa = dev_get_drvdata(&pdev->dev); - - clk_disable_unprepare(wsa->macro); - clk_disable_unprepare(wsa->dcodec); - clk_disable_unprepare(wsa->mclk); - clk_disable_unprepare(wsa->npl); - clk_disable_unprepare(wsa->fsgen); } static int wsa_macro_runtime_suspend(struct device *dev) { struct wsa_macro *wsa = dev_get_drvdata(dev); + int ret; regcache_cache_only(wsa->regmap, true); - regcache_mark_dirty(wsa->regmap); - clk_disable_unprepare(wsa->fsgen); - clk_disable_unprepare(wsa->npl); - clk_disable_unprepare(wsa->mclk); + ret = pm_clk_suspend(dev); + if (ret) { + regcache_cache_only(wsa->regmap, false); + return ret; + } + + regcache_mark_dirty(wsa->regmap); return 0; } @@ -2866,36 +2847,29 @@ static int wsa_macro_runtime_suspend(struct device *dev) static int wsa_macro_runtime_resume(struct device *dev) { struct wsa_macro *wsa = dev_get_drvdata(dev); - int ret; + int ret, sret; - ret = clk_prepare_enable(wsa->mclk); + ret = pm_clk_resume(dev); if (ret) { - dev_err(dev, "unable to prepare mclk\n"); + regcache_cache_only(wsa->regmap, true); + regcache_mark_dirty(wsa->regmap); return ret; } + regcache_cache_only(wsa->regmap, false); - ret = clk_prepare_enable(wsa->npl); - if (ret) { - dev_err(dev, "unable to prepare mclkx2\n"); - goto err_npl; - } - - ret = clk_prepare_enable(wsa->fsgen); + ret = regcache_sync(wsa->regmap); if (ret) { - dev_err(dev, "unable to prepare fsgen\n"); - goto err_fsgen; + regcache_cache_only(wsa->regmap, true); + regcache_mark_dirty(wsa->regmap); + sret = pm_clk_suspend(dev); + if (sret) + dev_err(dev, + "failed to suspend clocks after regcache sync failure: %d\n", + sret); + return ret; } - regcache_cache_only(wsa->regmap, false); - regcache_sync(wsa->regmap); - return 0; -err_fsgen: - clk_disable_unprepare(wsa->npl); -err_npl: - clk_disable_unprepare(wsa->mclk); - - return ret; } static const struct dev_pm_ops wsa_macro_pm_ops = { @@ -2929,7 +2903,6 @@ static struct platform_driver wsa_macro_driver = { .pm = pm_ptr(&wsa_macro_pm_ops), }, .probe = wsa_macro_probe, - .remove = wsa_macro_remove, }; module_platform_driver(wsa_macro_driver); diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c index da416329b038..bccce322ccc3 100644 --- a/sound/soc/codecs/max98090.c +++ b/sound/soc/codecs/max98090.c @@ -2341,6 +2341,7 @@ static irqreturn_t max98090_interrupt(int irq, void *data) * * @component: MAX98090 component * @jack: jack to report detection events on + * @data: can be used if codec driver need extra data for configuring jack * * Enable microphone detection via IRQ on the MAX98090. If GPIOs are * being used to bring out signals to the processor then only platform diff --git a/sound/soc/codecs/max98357a.c b/sound/soc/codecs/max98357a.c index cc811f58c9d2..b0f8043fb9e2 100644 --- a/sound/soc/codecs/max98357a.c +++ b/sound/soc/codecs/max98357a.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/max98373-i2c.c b/sound/soc/codecs/max98373-i2c.c index 20de379d08de..8805bd01153c 100644 --- a/sound/soc/codecs/max98373-i2c.c +++ b/sound/soc/codecs/max98373-i2c.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/max98373-sdw.c b/sound/soc/codecs/max98373-sdw.c index 6829fa07c9ec..8fe9c58e1a62 100644 --- a/sound/soc/codecs/max98373-sdw.c +++ b/sound/soc/codecs/max98373-sdw.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -272,8 +271,10 @@ static int max98373_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, MAX98373_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(max98373->regmap, false); regcache_sync(max98373->regmap); diff --git a/sound/soc/codecs/max98388.c b/sound/soc/codecs/max98388.c index 2576841b7de2..a4c57152d25f 100644 --- a/sound/soc/codecs/max98388.c +++ b/sound/soc/codecs/max98388.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index 2bbedf84ee5d..66309e87fdbd 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -952,11 +952,17 @@ static int max98390_suspend(struct device *dev) static int max98390_resume(struct device *dev) { struct max98390_priv *max98390 = dev_get_drvdata(dev); + int ret; dev_dbg(dev, "%s:Enter\n", __func__); regcache_cache_only(max98390->regmap, false); - regcache_sync(max98390->regmap); + ret = regcache_sync(max98390->regmap); + if (ret) { + regcache_cache_only(max98390->regmap, true); + regcache_mark_dirty(max98390->regmap); + return ret; + } return 0; } diff --git a/sound/soc/codecs/mt6351.c b/sound/soc/codecs/mt6351.c index 2a5e963fb2b5..1768c249650d 100644 --- a/sound/soc/codecs/mt6351.c +++ b/sound/soc/codecs/mt6351.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include diff --git a/sound/soc/codecs/mt6358.c b/sound/soc/codecs/mt6358.c index a787accb88e8..ed8cbc63fa77 100644 --- a/sound/soc/codecs/mt6358.c +++ b/sound/soc/codecs/mt6358.c @@ -6,7 +6,6 @@ // Author: KaiChieh Chuang #include -#include #include #include #include diff --git a/sound/soc/codecs/pcm3168a-i2c.c b/sound/soc/codecs/pcm3168a-i2c.c index 334f344761aa..dd24027836b2 100644 --- a/sound/soc/codecs/pcm3168a-i2c.c +++ b/sound/soc/codecs/pcm3168a-i2c.c @@ -10,7 +10,6 @@ #include #include #include -#include #include diff --git a/sound/soc/codecs/pcm512x.c b/sound/soc/codecs/pcm512x.c index fdef98ce52f1..fe3b5011fa16 100644 --- a/sound/soc/codecs/pcm512x.c +++ b/sound/soc/codecs/pcm512x.c @@ -633,7 +633,7 @@ static int pcm512x_dai_startup_slave(struct snd_pcm_substream *substream, struct regmap *regmap = pcm512x->regmap; if (IS_ERR(pcm512x->sclk)) { - dev_info(dev, "No SCLK, using BCLK: %ld\n", + dev_info_once(dev, "No SCLK, using BCLK: %ld\n", PTR_ERR(pcm512x->sclk)); /* Disable reporting of missing SCLK as an error */ diff --git a/sound/soc/codecs/rt1017-sdca-sdw.c b/sound/soc/codecs/rt1017-sdca-sdw.c index d62e8a253676..95405cea8143 100644 --- a/sound/soc/codecs/rt1017-sdca-sdw.c +++ b/sound/soc/codecs/rt1017-sdca-sdw.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -779,8 +778,10 @@ static int rt1017_sdca_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT1017_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt1017->regmap, false); regcache_sync(rt1017->regmap); diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c index 39e06a3a7560..2f30497498c7 100644 --- a/sound/soc/codecs/rt1308-sdw.c +++ b/sound/soc/codecs/rt1308-sdw.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -774,8 +773,10 @@ static int rt1308_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT1308_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt1308->regmap, false); regcache_sync_region(rt1308->regmap, 0xc000, 0xcfff); diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c index 1828fd9d5af6..ca318dbd946e 100644 --- a/sound/soc/codecs/rt1316-sdw.c +++ b/sound/soc/codecs/rt1316-sdw.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -751,8 +750,10 @@ static int rt1316_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT1316_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt1316->regmap, false); regcache_sync(rt1316->regmap); diff --git a/sound/soc/codecs/rt1318-sdw.c b/sound/soc/codecs/rt1318-sdw.c index 51bd11b92a55..c038ac0e3b76 100644 --- a/sound/soc/codecs/rt1318-sdw.c +++ b/sound/soc/codecs/rt1318-sdw.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c index 13493b85f3c9..18d7ebff505f 100644 --- a/sound/soc/codecs/rt1320-sdw.c +++ b/sound/soc/codecs/rt1320-sdw.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -411,6 +410,402 @@ static const struct reg_sequence rt1321_blind_write[] = { { 0x41001988, 0x03 }, }; +static const struct reg_sequence rt1321_va1_blind_write[] = { + { 0x0000c003, 0xf0 }, + { 0x0000c01b, 0xfc }, + { 0x0000c5c3, 0xf2 }, + { 0x0000c5c2, 0x00 }, + { 0x0000c5c1, 0x10 }, + { 0x0000c5c0, 0x04 }, + { 0x0000c5c7, 0x03 }, + { 0x0000c5c6, 0x10 }, + { 0x0000c526, 0x47 }, + { 0x0000c5c4, 0x12 }, + { 0x0000c5c5, 0x60 }, + { 0x0000c520, 0x10 }, + { 0x0000c521, 0x32 }, + { 0x0000c5c7, 0x00 }, + { 0x0000c5c8, 0x03 }, + { 0x0000c5d3, 0x08 }, + { 0x0000c5d2, 0x0a }, + { 0x0000c5d1, 0x49 }, + { 0x0000c5d0, 0x0f }, + { 0x0000c580, 0x10 }, + { 0x0000c581, 0x32 }, + { 0x0000c582, 0x01 }, + { 0x0000c682, 0x60 }, + { 0x0000c019, 0x10 }, + { 0x0000c5f0, 0x01 }, + { 0x0000c5f7, 0x22 }, + { 0x0000c5f6, 0x22 }, + { 0x0000c057, 0x51 }, + { 0x0000c054, 0x55 }, + { 0x0000c053, 0x55 }, + { 0x0000c052, 0x55 }, + { 0x0000c051, 0x01 }, + { 0x0000c050, 0x15 }, + { 0x0000c060, 0x99 }, + { 0x0000c030, 0x55 }, + { 0x0000c061, 0x55 }, + { 0x0000c063, 0x55 }, + { 0x0000c065, 0xa5 }, + { 0x0000c06b, 0x0a }, + { 0x0000ca05, 0xd6 }, + { 0x0000ca06, 0x11 }, + { 0x0000ca07, 0x1e }, + { 0x0000ca25, 0xd6 }, + { 0x0000ca26, 0x11 }, + { 0x0000ca27, 0x1e }, + { 0x0000cd00, 0x05 }, + { 0x0000cd81, 0x49 }, + { 0x0000cd82, 0x49 }, + { 0x0000c604, 0x40 }, + { 0x0000c609, 0x40 }, + { 0x0000c046, 0xf7 }, + { 0x0000c045, 0xff }, + { 0x0000c044, 0xff }, + { 0x0000c043, 0xff }, + { 0x0000c042, 0xff }, + { 0x0000c041, 0xff }, + { 0x0000c040, 0xff }, + { 0x0000c049, 0xff }, + { 0x0000c028, 0x3f }, + { 0x0000c020, 0x3f }, + { 0x0000c032, 0x13 }, + { 0x0000c033, 0x01 }, + { 0x0000cc10, 0x01 }, + { 0x0000dc20, 0x03 }, + { 0x0000de03, 0x05 }, + { 0x0000dc00, 0x00 }, + { 0x0000c700, 0xf0 }, + { 0x0000c701, 0x13 }, + { 0x0000c900, 0xc3 }, + { 0x0000c570, 0x08 }, + { 0x0000c086, 0x02 }, + { 0x0000c085, 0x7f }, + { 0x0000c084, 0x00 }, + { 0x0000c081, 0xff }, + { 0x0000f084, 0x0f }, + { 0x0000f083, 0xff }, + { 0x0000f082, 0xff }, + { 0x0000f081, 0xff }, + { 0x0000f080, 0xff }, + { 0x20003003, 0x3f }, + { 0x20005818, 0x81 }, + { 0x20009018, 0x81 }, + { 0x2000301c, 0x81 }, + { 0x0000c003, 0xc0 }, + { 0x0000c047, 0x80 }, + { 0x0000d541, 0x80 }, + { 0x0000d487, 0x0b }, + { 0x0000d487, 0x3b }, + { 0x0000d486, 0xc3 }, + { 0x0000d470, 0x89 }, + { 0x0000d471, 0x3a }, + { 0x0000d472, 0x3d }, + { 0x0000d474, 0x11 }, + { 0x0000d475, 0x32 }, + { 0x0000d476, 0x64 }, + { 0x0000d477, 0x10 }, + { 0x0000d478, 0xff }, + { 0x0000d479, 0x20 }, + { 0x0000d47a, 0x10 }, + { 0x0000d73c, 0xb7 }, + { 0x0000d73d, 0xd7 }, + { 0x0000d73e, 0x00 }, + { 0x0000d73f, 0x10 }, + { 0x1000cd56, 0x00 }, + { 0x3fc2dfc0, 0x03 }, + { 0x3fc2dfc1, 0x00 }, + { 0x3fc2dfc2, 0x00 }, + { 0x3fc2dfc3, 0x00 }, + { 0x3fc2dfc4, 0x01 }, + { 0x3fc2dfc5, 0x00 }, + { 0x3fc2dfc6, 0x00 }, + { 0x3fc2dfc7, 0x00 }, + { 0x3fc2df80, 0x00 }, + { 0x3fc2df81, 0x00 }, + { 0x3fc2df82, 0x00 }, + { 0x3fc2df83, 0x00 }, + { 0x0000d541, 0x40 }, + { 0x0000d486, 0x43 }, + { 0x1000db00, 0x04 }, + { 0x1000db01, 0x00 }, + { 0x1000db02, 0x10 }, + { 0x1000db03, 0x00 }, + { 0x1000db04, 0x00 }, + { 0x1000db05, 0x45 }, + { 0x1000db06, 0x0d }, + { 0x1000db07, 0x01 }, + { 0x1000db08, 0x00 }, + { 0x1000db09, 0x00 }, + { 0x1000db0a, 0xbf }, + { 0x1000db0b, 0x0b }, + { 0x1000db0c, 0x11 }, + { 0x1000db0d, 0x00 }, + { 0x1000db0e, 0x00 }, + { 0x1000db0f, 0x00 }, + { 0x1000db10, 0x2c }, + { 0x1000db11, 0xfa }, + { 0x1000db12, 0x00 }, + { 0x1000db13, 0x00 }, + { 0x1000db14, 0x09 }, + { 0x0000d540, 0x21 }, + { 0x0000c570, 0x08 }, + { 0x0000d714, 0x17 }, + { 0x0000c5c3, 0xf2 }, + { 0x0000c5c8, 0x03 }, + { 0x20009012, 0x00 }, + { 0x0000dd08, 0x17 }, + { 0x0000dd09, 0x0e }, + { 0x0000dd0a, 0x17 }, + { 0x0000dd0b, 0x0e }, + { 0x0000c570, 0x08 }, + { 0x0000d471, 0x3a }, + { 0x0000db00, 0x00 }, + { 0x0000db01, 0x00 }, + { 0x0000db02, 0x73 }, + { 0x0000db03, 0x00 }, + { 0x0000db04, 0x00 }, + { 0x0000db05, 0x00 }, + { 0x0000db06, 0x00 }, + { 0x0000db07, 0x00 }, + { 0x0000db08, 0x7f }, + { 0x0000db09, 0x00 }, + { 0x0000db1a, 0x00 }, + { 0x0000db1b, 0x00 }, + { 0x0000db19, 0x00 }, +}; + +static const struct reg_sequence rt1321_va2_blind_write[] = { + { 0x0000c003, 0xf0 }, + { 0x0000c01b, 0xfc }, + { 0x0000c5c3, 0xf2 }, + { 0x0000c5c2, 0x00 }, + { 0x0000c5c1, 0x10 }, + { 0x0000c5c0, 0x04 }, + { 0x0000c5c7, 0x03 }, + { 0x0000c5c6, 0x10 }, + { 0x0000c526, 0x47 }, + { 0x0000c5c4, 0x12 }, + { 0x0000c5c5, 0x60 }, + { 0x0000c520, 0x10 }, + { 0x0000c521, 0x32 }, + { 0x0000c5c7, 0x00 }, + { 0x0000c5c8, 0x03 }, + { 0x0000c5d3, 0x08 }, + { 0x0000c5d2, 0x0a }, + { 0x0000c5d1, 0x49 }, + { 0x0000c5d0, 0x0f }, + { 0x0000c580, 0x10 }, + { 0x0000c581, 0x32 }, + { 0x0000c582, 0x01 }, + { 0x0000c682, 0x60 }, + { 0x0000c019, 0x10 }, + { 0x0000c5f0, 0x01 }, + { 0x0000c5f7, 0x22 }, + { 0x0000c5f6, 0x22 }, + { 0x0000c057, 0x51 }, + { 0x0000c054, 0x55 }, + { 0x0000c053, 0x55 }, + { 0x0000c052, 0x55 }, + { 0x0000c051, 0x01 }, + { 0x0000c050, 0x15 }, + { 0x0000c060, 0x99 }, + { 0x0000c030, 0x55 }, + { 0x0000c061, 0x55 }, + { 0x0000c063, 0x55 }, + { 0x0000c065, 0xa5 }, + { 0x0000c06b, 0x0a }, + { 0x0000ca05, 0xd6 }, + { 0x0000ca06, 0x11 }, + { 0x0000ca07, 0x1e }, + { 0x0000ca25, 0xd6 }, + { 0x0000ca26, 0x11 }, + { 0x0000ca27, 0x1e }, + { 0x0000cd00, 0x05 }, + { 0x0000cd81, 0x49 }, + { 0x0000cd82, 0x49 }, + { 0x0000c604, 0x40 }, + { 0x0000c609, 0x40 }, + { 0x0000c046, 0xf7 }, + { 0x0000c045, 0xff }, + { 0x0000c044, 0xff }, + { 0x0000c043, 0xff }, + { 0x0000c042, 0xff }, + { 0x0000c041, 0xff }, + { 0x0000c040, 0xff }, + { 0x0000c049, 0xff }, + { 0x0000c028, 0x3f }, + { 0x0000c020, 0x3f }, + { 0x0000c032, 0x13 }, + { 0x0000c033, 0x01 }, + { 0x0000cc10, 0x01 }, + { 0x0000dc20, 0x03 }, + { 0x0000de03, 0x05 }, + { 0x0000dc00, 0x00 }, + { 0x0000c700, 0xf0 }, + { 0x0000c701, 0x13 }, + { 0x0000c900, 0xc3 }, + { 0x0000c570, 0x08 }, + { 0x0000c086, 0x02 }, + { 0x0000c085, 0x7f }, + { 0x0000c084, 0x00 }, + { 0x0000c081, 0xff }, + { 0x0000f084, 0x0f }, + { 0x0000f083, 0xff }, + { 0x0000f082, 0xff }, + { 0x0000f081, 0xff }, + { 0x0000f080, 0xff }, + { 0x20003003, 0x3f }, + { 0x20005818, 0x81 }, + { 0x20009018, 0x81 }, + { 0x2000301c, 0x81 }, + { 0x0000c003, 0xc0 }, + { 0x0000c047, 0x80 }, + { 0x0000d541, 0x80 }, + { 0x0000d487, 0x0b }, + { 0x0000d487, 0x3b }, + { 0x0000d486, 0xc3 }, + { 0x0000d470, 0x89 }, + { 0x0000d471, 0x3a }, + { 0x0000d472, 0x3d }, + { 0x0000d474, 0x11 }, + { 0x0000d475, 0x32 }, + { 0x0000d476, 0x64 }, + { 0x0000d477, 0x10 }, + { 0x0000d478, 0xff }, + { 0x0000d479, 0x20 }, + { 0x0000d47a, 0x10 }, + { 0x10008000, 0x67 }, + { 0x10008001, 0x80 }, + { 0x10008002, 0x00 }, + { 0x10008003, 0x00 }, + { 0x1000cd56, 0x00 }, + { 0x0000d486, 0x43 }, + { 0x3fc2dfc3, 0x00 }, + { 0x3fc2dfc2, 0x00 }, + { 0x3fc2dfc1, 0x00 }, + { 0x3fc2dfc0, 0x03 }, + { 0x3fc2dfc7, 0x00 }, + { 0x3fc2dfc6, 0x00 }, + { 0x3fc2dfc5, 0x00 }, + { 0x3fc2dfc4, 0x01 }, + { 0x3fc2dfa3, 0x00 }, + { 0x3fc2dfa2, 0x00 }, + { 0x3fc2dfa1, 0x00 }, + { 0x3fc2dfa0, 0x00 }, + { 0x3fc2df80, 0x10 }, + { 0x3fc2df81, 0x20 }, + { 0x3fc2df82, 0x00 }, + { 0x3fc2df83, 0x00 }, + { 0x3fc2df84, 0x50 }, + { 0x3fc2df85, 0x19 }, + { 0x3fc2df86, 0x00 }, + { 0x3fc2df87, 0x00 }, + { 0x3fc2df88, 0x52 }, + { 0x3fc2df89, 0x23 }, + { 0x3fc2df8a, 0x00 }, + { 0x3fc2df8b, 0x00 }, + { 0x3fc2df8c, 0xe0 }, + { 0x3fc2df8d, 0x2e }, + { 0x3fc2df8e, 0x00 }, + { 0x3fc2df8f, 0x00 }, + { 0x3fc2df90, 0xe0 }, + { 0x3fc2df91, 0x2e }, + { 0x3fc2df92, 0x00 }, + { 0x3fc2df93, 0x00 }, + { 0x3fc2df94, 0x01 }, + { 0x3fc2df95, 0x08 }, + { 0x3fc2df96, 0x00 }, + { 0x3fc2df97, 0x00 }, + { 0x3fc2df40, 0x80 }, + { 0x3fc2df41, 0xbb }, + { 0x3fc2df42, 0x00 }, + { 0x3fc2df43, 0x00 }, + { 0x3fc2df44, 0xc0 }, + { 0x3fc2df45, 0x99 }, + { 0x3fc2df46, 0x01 }, + { 0x3fc2df47, 0x00 }, + { 0x3fc2df48, 0x00 }, + { 0x3fc2df49, 0x00 }, + { 0x3fc2df4a, 0x00 }, + { 0x3fc2df4b, 0x00 }, + { 0x3fc2df4c, 0x00 }, + { 0x3fc2df4d, 0x00 }, + { 0x3fc2df4e, 0x00 }, + { 0x3fc2df4f, 0x00 }, + { 0x3fc2df50, 0x01 }, + { 0x3fc2df51, 0x00 }, + { 0x3fc2df52, 0x00 }, + { 0x3fc2df53, 0x00 }, + { 0x3fc2df54, 0x01 }, + { 0x3fc2df55, 0x00 }, + { 0x3fc2df56, 0x00 }, + { 0x3fc2df57, 0x00 }, + { 0x3fc2df58, 0x00 }, + { 0x3fc2df59, 0x00 }, + { 0x3fc2df5a, 0x00 }, + { 0x3fc2df5b, 0x00 }, + { 0x3fc2df5c, 0x01 }, + { 0x3fc2df5d, 0x00 }, + { 0x3fc2df5e, 0x00 }, + { 0x3fc2df5f, 0x00 }, + { 0x3fc2df60, 0x00 }, + { 0x3fc2df61, 0x00 }, + { 0x3fc2df62, 0x00 }, + { 0x3fc2df63, 0x00 }, + { 0x3fc2df64, 0x00 }, + { 0x3fc2df65, 0x00 }, + { 0x3fc2df66, 0x00 }, + { 0x3fc2df67, 0x10 }, + { 0x3fc2df68, 0x01 }, + { 0x3fc2df69, 0x00 }, + { 0x3fc2df6a, 0x00 }, + { 0x3fc2df6b, 0x00 }, + { 0x3fc2df6c, 0x01 }, + { 0x3fc2df6d, 0x00 }, + { 0x3fc2df6e, 0x00 }, + { 0x3fc2df6f, 0x00 }, + { 0x3fc2df70, 0x04 }, + { 0x3fc2df71, 0x00 }, + { 0x3fc2df72, 0x00 }, + { 0x3fc2df73, 0x00 }, + { 0x3fc2df74, 0x01 }, + { 0x3fc2df75, 0x00 }, + { 0x3fc2df76, 0x00 }, + { 0x3fc2df77, 0x00 }, + { 0x1000db00, 0x04 }, + { 0x1000db01, 0x00 }, + { 0x1000db02, 0x10 }, + { 0x1000db03, 0x00 }, + { 0x1000db04, 0x00 }, + { 0x1000db05, 0x45 }, + { 0x1000db06, 0x0d }, + { 0x1000db07, 0x01 }, + { 0x1000db08, 0x00 }, + { 0x1000db09, 0x00 }, + { 0x1000db0a, 0xbf }, + { 0x1000db0b, 0x0b }, + { 0x1000db0c, 0x11 }, + { 0x1000db0d, 0x00 }, + { 0x1000db0e, 0x00 }, + { 0x1000db0f, 0x00 }, + { 0x1000db10, 0x2c }, + { 0x1000db11, 0xfa }, + { 0x1000db12, 0x00 }, + { 0x1000db13, 0x00 }, + { 0x1000db14, 0x09 }, + { 0x0000d540, 0x21 }, + { 0x0000d714, 0x17 }, + { 0x0000dd0b, 0x0d }, + { 0x0000dd0a, 0xff }, + { 0x0000dd09, 0x0d }, + { 0x0000dd08, 0xff }, + { 0x0000c5fb, 0x12 }, + { 0x0000c570, 0x08 }, +}; + static const struct reg_default rt1320_reg_defaults[] = { { SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_PDE11, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 }, { SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_FU113, RT1320_SDCA_CTL_FU_MUTE, CH_01), 0x01 }, @@ -498,10 +893,15 @@ static bool rt1320_readable_register(struct device *dev, unsigned int reg) case RT1321_PATCH_MAIN_VER ... RT1321_PATCH_BETA_VER: case 0x1000f008: case 0x1000f021: - case 0x2000300f: + case 0x20003000 ... 0x2000300f: case 0x2000301c: - case 0x2000900f: + case 0x20003040 ... 0x20003059: + case 0x20005800 ... 0x2000580f: + case 0x20005818: + case 0x20005840 ... 0x20005859: + case 0x20009000 ... 0x2000900f: case 0x20009018: + case 0x20009040 ... 0x20009059: case 0x3fc000c0 ... 0x3fc2dfc8: case 0x3fe00000 ... 0x3fe36fff: /* 0x40801508/0x40801809/0x4080180a/0x40801909/0x4080190a */ @@ -597,15 +997,20 @@ static bool rt1320_volatile_register(struct device *dev, unsigned int reg) case 0x1000c000 ... 0x1000dfff: case 0x1000f008: case 0x1000f021: - case 0x2000300f: + case 0x2000300e ... 0x2000300f: case 0x2000301c: - case 0x2000900f: + case 0x20003040 ... 0x20003059: + case 0x2000580e ... 0x2000580f: + case 0x20005818: + case 0x20005840 ... 0x20005859: + case 0x2000900e ... 0x2000900f: case 0x20009018: + case 0x20009040 ... 0x20009059: case 0x3fc2ab80 ... 0x3fc2ac4c: case 0x3fc2b780: case 0x3fc2bf80 ... 0x3fc2bf83: case 0x3fc2bfc0 ... 0x3fc2bfc8: - case 0x3fc2d300 ... 0x3fc2d354: + case 0x3fc2d300 ... 0x3fc2d3cc: case 0x3fc2dfc0 ... 0x3fc2dfc8: case 0x3fe2e000 ... 0x3fe2e003: case SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_PDE11, RT1320_SDCA_CTL_ACTUAL_POWER_STATE, 0): @@ -893,6 +1298,42 @@ static int rt1320_check_fw_ready(struct rt1320_sdw_priv *rt1320) return 0; } +static int rt1320_dspfw_status(struct rt1320_sdw_priv *rt1320) +{ + struct device *dev = &rt1320->sdw_slave->dev; + unsigned int fw_status_addr, fw_ready; + unsigned int dspfw_run; + + switch (rt1320->dev_id) { + case RT1320_DEV_ID: + fw_status_addr = RT1320_DSPFW_STATUS_ADDR; + break; + case RT1321_DEV_ID: + fw_status_addr = RT1321_DSPFW_STATUS_ADDR; + break; + default: + dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); + return -EINVAL; + } + + regmap_read(rt1320->regmap, fw_status_addr, &fw_ready); + fw_ready &= 0x1; + + if (rt1320->dev_id == RT1321_DEV_ID) { + regmap_read(rt1320->regmap, 0xf01e, &dspfw_run); + dspfw_run &= 0x1; + fw_ready = (!dspfw_run && fw_ready); + } + + if (fw_ready) { + dev_dbg(dev, "%s, DSP FW was already\n", __func__); + return 1; + } + + dev_dbg(dev, "%s, DSP FW is NOT ready. Please load DSP FW first\n", __func__); + return 0; +} + static int rt1320_check_power_state_ready(struct rt1320_sdw_priv *rt1320, enum rt1320_power_state ps) { struct device *dev = &rt1320->sdw_slave->dev; @@ -983,6 +1424,9 @@ static int rt1320_fw_param_protocol(struct rt1320_sdw_priv *rt1320, enum rt1320_ if (!tempbuf) return -ENOMEM; + if (rt1320->dev_id == RT1321_DEV_ID && rt1320->version_id == RT1321_VA2) + paramid += 0x0bff0000; + paramhr.moudleid = 1; paramhr.commandtype = cmdid; /* 8 is "sizeof(paramid) + sizeof(paramlength)" */ @@ -1121,45 +1565,73 @@ static void rt1320_calc_r0(struct rt1320_sdw_priv *rt1320) l_calir0, l_calir0_lo, r_calir0, r_calir0_lo); } +static int rt1320_pilot_tone_output(struct rt1320_sdw_priv *rt1320) +{ + struct device *dev = &rt1320->sdw_slave->dev; + int l_targetpostgain, r_targetpostgain; + unsigned long long factor = (1 << 12); + int l_pilotgain[9], r_pilotgain[9]; + const int postgain_step = 234; + int targetGain; + + switch (rt1320->dev_id) { + case RT1320_DEV_ID: + targetGain = -320000; + break; + case RT1321_DEV_ID: + targetGain = -420000; + if (rt1320->version_id == RT1321_VA0) + targetGain = -320000; + break; + default: + dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); + return -EINVAL; + } + + rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 70, &l_pilotgain[0], sizeof(l_pilotgain)); + rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 71, &r_pilotgain[0], sizeof(r_pilotgain)); + dev_dbg(dev, "%s, LR pilotgain %d, %d\n", __func__, l_pilotgain[2], r_pilotgain[2]); + + /* calculate pilot tone gain */ + l_pilotgain[2] = (l_pilotgain[2] * 10000) / factor; + r_pilotgain[2] = (r_pilotgain[2] * 10000) / factor; + + /* calculate post gain to meet target gain */ + l_targetpostgain = abs(targetGain - l_pilotgain[2]) / postgain_step; + r_targetpostgain = abs(targetGain - r_pilotgain[2]) / postgain_step; + l_targetpostgain = 0xfff - l_targetpostgain; + r_targetpostgain = 0xfff - r_targetpostgain; + dev_dbg(dev, "%s, LR targetpostgain=0x%x, 0x%x\n", __func__, l_targetpostgain, r_targetpostgain); + + regmap_write(rt1320->regmap, 0xdd0b, (l_targetpostgain & 0xf00) >> 8); + regmap_write(rt1320->regmap, 0xdd0a, l_targetpostgain & 0xff); + regmap_write(rt1320->regmap, 0xdd09, (r_targetpostgain & 0xf00) >> 8); + regmap_write(rt1320->regmap, 0xdd08, r_targetpostgain & 0xff); + + return 0; +} + static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320) { struct device *dev = &rt1320->sdw_slave->dev; struct rt1320_datafixpoint audfixpoint[2]; unsigned int reg_c5fb, reg_c570, reg_cd00; - unsigned int vol_reg[4], fw_ready; + unsigned int vol_reg[4]; unsigned long long l_meanr0, r_meanr0; - unsigned int fw_status_addr; int l_re[5], r_re[5]; int ret, tmp; unsigned long long factor = (1 << 27); unsigned short l_advancegain, r_advancegain; unsigned int delay_s = 7; /* delay seconds for the calibration */ + int dspfw_status; if (!rt1320->component) return; - switch (rt1320->dev_id) { - case RT1320_DEV_ID: - fw_status_addr = RT1320_DSPFW_STATUS_ADDR; - break; - case RT1321_DEV_ID: - fw_status_addr = RT1321_DSPFW_STATUS_ADDR; - break; - default: - dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); - return; - } - - /* set volume 0dB */ regmap_read(rt1320->regmap, 0xdd0b, &vol_reg[3]); regmap_read(rt1320->regmap, 0xdd0a, &vol_reg[2]); regmap_read(rt1320->regmap, 0xdd09, &vol_reg[1]); regmap_read(rt1320->regmap, 0xdd08, &vol_reg[0]); - regmap_write(rt1320->regmap, 0xdd0b, 0x0f); - regmap_write(rt1320->regmap, 0xdd0a, 0xff); - regmap_write(rt1320->regmap, 0xdd09, 0x0f); - regmap_write(rt1320->regmap, 0xdd08, 0xff); - regmap_read(rt1320->regmap, 0xc5fb, ®_c5fb); regmap_read(rt1320->regmap, 0xc570, ®_c570); regmap_read(rt1320->regmap, 0xcd00, ®_cd00); @@ -1172,9 +1644,8 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320) goto _finish_; } - regmap_read(rt1320->regmap, fw_status_addr, &fw_ready); - fw_ready &= 0x1; - if (!fw_ready) { + dspfw_status = rt1320_dspfw_status(rt1320); + if (dspfw_status <= 0) { dev_dbg(dev, "%s, DSP FW is NOT ready. Please load DSP FW first\n", __func__); goto _finish_; } @@ -1185,8 +1656,19 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320) goto _finish_; } - if (rt1320->dev_id == RT1320_DEV_ID) - regmap_write(rt1320->regmap, 0xc5fb, 0x00); + /* fine tune pilot tone output */ + ret = rt1320_pilot_tone_output(rt1320); + if (ret < 0) { + dev_dbg(dev, "%s, Failed to tune pilot tone output\n", __func__); + goto _finish_; + } + + if (rt1320->dev_id == RT1321_DEV_ID) { + regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x00); + regmap_write(rt1320->regmap, 0xc5c4, 0x12); + } + + regmap_write(rt1320->regmap, 0xc5fb, 0x00); regmap_write(rt1320->regmap, 0xc570, 0x0b); regmap_write(rt1320->regmap, 0xcd00, 0xc5); @@ -1247,6 +1729,11 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320) SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03); rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03); + if (rt1320->dev_id == RT1321_DEV_ID) { + regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x80); + regmap_write(rt1320->regmap, 0xc5c4, 0x10); + } + /* advance gain will be set when R0 load, not here */ regmap_write(rt1320->regmap, 0xdd0b, vol_reg[3]); regmap_write(rt1320->regmap, 0xdd0a, vol_reg[2]); @@ -1313,6 +1800,9 @@ static void rt1320_load_mcu_patch(struct rt1320_sdw_priv *rt1320) max_addr = 0x10007fff; break; case RT1321_DEV_ID: + if (rt1320->version_id == RT1321_VA2) + return; + filename = RT1321_VA_MCU_PATCH; min_addr = 0x10008000; max_addr = 0x10008fff; @@ -1383,30 +1873,17 @@ static void rt1320_vab_preset(struct rt1320_sdw_priv *rt1320) static void rt1320_t0_load(struct rt1320_sdw_priv *rt1320, unsigned int l_t0, unsigned int r_t0) { struct device *dev = &rt1320->sdw_slave->dev; - unsigned int factor = (1 << 22), fw_ready; + unsigned int factor = (1 << 22); int l_t0_data[38], r_t0_data[38]; - unsigned int fw_status_addr; - - switch (rt1320->dev_id) { - case RT1320_DEV_ID: - fw_status_addr = RT1320_DSPFW_STATUS_ADDR; - break; - case RT1321_DEV_ID: - fw_status_addr = RT1321_DSPFW_STATUS_ADDR; - break; - default: - dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); - return; - } + int dspfw_status; regmap_write(rt1320->regmap, SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00); rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00); - regmap_read(rt1320->regmap, fw_status_addr, &fw_ready); - fw_ready &= 0x1; - if (!fw_ready) { + dspfw_status = rt1320_dspfw_status(rt1320); + if (dspfw_status <= 0) { dev_warn(dev, "%s, DSP FW is NOT ready\n", __func__); goto _exit_; } @@ -1484,22 +1961,48 @@ static int rt1320_rae_load(struct rt1320_sdw_priv *rt1320) request_firmware(&rae_fw, rae_filename, dev); if (rae_fw) { - /* RAE CRC clear */ - regmap_write(rt1320->regmap, 0xe80b, 0x0f); - - /* RAE stop & CRC disable */ - regmap_update_bits(rt1320->regmap, 0xe803, 0xbc, 0x00); - - while (--retry) { - regmap_read(rt1320->regmap, 0xe83f, &value); - if (value & 0x40) - break; - usleep_range(1000, 1100); - } - if (!retry && !(value & 0x40)) { - dev_err(dev, "%s: RAE is not ready to load\n", __func__); - release_firmware(rae_fw); - return -ETIMEDOUT; + switch (rt1320->dev_id) { + case RT1320_DEV_ID: + /* RAE CRC clear */ + regmap_write(rt1320->regmap, 0xe80b, 0x0f); + /* RAE stop & CRC disable */ + regmap_update_bits(rt1320->regmap, 0xe803, 0xbc, 0x00); + while (--retry) { + regmap_read(rt1320->regmap, 0xe83f, &value); + if (value & 0x40) + break; + usleep_range(1000, 1100); + } + if (!retry && !(value & 0x40)) { + dev_err(dev, "%s: RAE is not ready to load\n", __func__); + release_firmware(rae_fw); + return -ETIMEDOUT; + } + break; + case RT1321_DEV_ID: + /* RAE CRC clear */ + regmap_write(rt1320->regmap, 0x2000300e, 0xc0); + regmap_write(rt1320->regmap, 0x2000300f, 0x0f); + /* RAE stop & Phase sync & CRC disable */ + regmap_update_bits(rt1320->regmap, 0x20003003, 0xfe, 0x00); + regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x00); + regmap_update_bits(rt1320->regmap, 0x2000301c, 0x01, 0x00); + /* check whether write state is ready */ + while (--retry) { + regmap_read(rt1320->regmap, 0x20003043, &value); + if (value & 0x40) + break; + usleep_range(1000, 1100); + } + if (!retry && !(value & 0x40)) { + dev_err(dev, "%s: RAE is not ready to load\n", __func__); + release_firmware(rae_fw); + return -ETIMEDOUT; + } + break; + default: + dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); + return -EINVAL; } dev_dbg(dev, "%s, rae_fw size=0x%zx\n", __func__, rae_fw->size); @@ -1560,18 +2063,34 @@ static int rt1320_rae_load(struct rt1320_sdw_priv *rt1320) goto _exit_; } - /* RAE CRC enable */ - regmap_update_bits(rt1320->regmap, 0xe803, 0x0c, 0x0c); - - /* RAE update */ - regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x00); - regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x80); - - /* RAE run */ - regmap_update_bits(rt1320->regmap, 0xe803, 0x80, 0x80); - - regmap_read(rt1320->regmap, 0xe80b, &value); - dev_dbg(dev, "%s: CAE run => 0xe80b reg = 0x%x\n", __func__, value); + switch (rt1320->dev_id) { + case RT1320_DEV_ID: + /* RAE CRC enable */ + regmap_update_bits(rt1320->regmap, 0xe803, 0x0c, 0x0c); + /* RAE update */ + regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x00); + regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x80); + /* RAE run */ + regmap_update_bits(rt1320->regmap, 0xe803, 0x80, 0x80); + regmap_read(rt1320->regmap, 0xe80b, &value); + dev_dbg(dev, "%s: CAE run => 0xe80b reg = 0x%x\n", __func__, value); + break; + case RT1321_DEV_ID: + /* RAE CRC enable */ + regmap_update_bits(rt1320->regmap, 0x20003003, 0x30, 0x30); + /* RAE update */ + regmap_update_bits(rt1320->regmap, 0x2000301c, 0x80, 0x00); + regmap_update_bits(rt1320->regmap, 0x2000301c, 0x80, 0x80); + regmap_update_bits(rt1320->regmap, 0x20009018, 0x80, 0x00); + regmap_update_bits(rt1320->regmap, 0x20009018, 0x80, 0x80); + regmap_update_bits(rt1320->regmap, 0x20005818, 0x80, 0x00); + regmap_update_bits(rt1320->regmap, 0x20005818, 0x80, 0x80); + /* RAE run */ + regmap_update_bits(rt1320->regmap, 0x2000301c, 0x01, 0x01); + /* Phase sync eanble */ + regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x80); + break; + } rt1320->rae_update_done = true; @@ -1599,8 +2118,7 @@ struct rt1320_dspfwheader { struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(rt1320->component); struct device *dev = &rt1320->sdw_slave->dev; - unsigned int val, i, fw_offset, fw_ready; - unsigned int fw_status_addr; + unsigned int val, i, fw_offset; struct rt1320_dspfwheader *fwheader; struct rt1320_imageinfo *ptr_img; struct sdw_bpt_section sec[10]; @@ -1611,20 +2129,10 @@ struct rt1320_dspfwheader { unsigned int hdr_size = 0; const char *dmi_vendor, *dmi_product, *dmi_sku; int len_vendor, len_product, len_sku; + unsigned char boot_mode = 0; /* 0: from RAM; 1: from ROM */ + unsigned char has_0x3fc00000 = 0; char filename[512]; - switch (rt1320->dev_id) { - case RT1320_DEV_ID: - fw_status_addr = RT1320_DSPFW_STATUS_ADDR; - break; - case RT1321_DEV_ID: - fw_status_addr = RT1321_DSPFW_STATUS_ADDR; - break; - default: - dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); - return; - } - dmi_vendor = dmi_get_system_info(DMI_SYS_VENDOR); dmi_product = dmi_get_system_info(DMI_PRODUCT_NAME); dmi_sku = dmi_get_system_info(DMI_PRODUCT_SKU); @@ -1654,17 +2162,12 @@ struct rt1320_dspfwheader { RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00); rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00); - regmap_read(rt1320->regmap, fw_status_addr, &fw_ready); - fw_ready &= 0x1; - if (fw_ready) { + if (rt1320_dspfw_status(rt1320)) { dev_dbg(dev, "%s, DSP FW was already\n", __func__); rt1320->fw_load_done = true; goto _exit_; } - /* change to IRAM */ - regmap_update_bits(rt1320->regmap, 0xf01e, 0x80, 0x00); - request_firmware(&fw, filename, dev); if (fw) { fwheader = (struct rt1320_dspfwheader *)fw->data; @@ -1710,9 +2213,11 @@ struct rt1320_dspfwheader { dev_fw_match = true; break; case RT1321_DEV_ID: - if (ptr_img->addr == 0x3fc00000) - if (fw_data[9] == '1') + if (ptr_img->addr == 0x3fc00000) { + if (fw_data[7] == '1') dev_fw_match = true; + has_0x3fc00000 = 1; + } break; default: dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); @@ -1722,6 +2227,22 @@ struct rt1320_dspfwheader { fw_offset += ptr_img->size; } + if (rt1320->dev_id == RT1321_DEV_ID && rt1320->version_id == RT1321_VA2) { + /* + * For 1321 VA2, if the FW doesn't include the section for address 0x3fc00000, + * it means the FW will boot from ROM and force dev_fw_match to true to download FW by BRA. + */ + if (!has_0x3fc00000) { + boot_mode = 1; + dev_fw_match = true; + } + dev_dbg(dev, "%s: Boot from %s for VA2\n", __func__, (boot_mode ? "ROM" : "RAM")); + } + + /* change to IRAM */ + if (!boot_mode) + regmap_update_bits(rt1320->regmap, 0xf01e, 0x80, 0x00); + if (dev_fw_match) { dev_dbg(dev, "%s, starting BRA downloading FW..\n", __func__); rt1320->bra_msg.dev_num = rt1320->sdw_slave->dev_num; @@ -1744,24 +2265,39 @@ struct rt1320_dspfwheader { goto _exit_; } - /* run RAM code */ - regmap_read(rt1320->regmap, 0x3fc2bfc0, &val); - val |= 0x8; - regmap_write(rt1320->regmap, 0x3fc2bfc0, val); - - /* clear frame counter */ switch (rt1320->dev_id) { case RT1320_DEV_ID: + /* run RAM code */ + regmap_read(rt1320->regmap, 0x3fc2bfc0, &val); + val |= 0x8; + regmap_write(rt1320->regmap, 0x3fc2bfc0, val); + /* clear frame counter */ regmap_write(rt1320->regmap, 0x3fc2bfcb, 0x00); regmap_write(rt1320->regmap, 0x3fc2bfca, 0x00); regmap_write(rt1320->regmap, 0x3fc2bfc9, 0x00); regmap_write(rt1320->regmap, 0x3fc2bfc8, 0x00); break; case RT1321_DEV_ID: + if (!boot_mode) { + /* run RAM code */ + regmap_read(rt1320->regmap, 0x3fc2dfc0, &val); + val |= 0x8; + regmap_write(rt1320->regmap, 0x3fc2dfc0, val); + } + /* clear frame counter */ regmap_write(rt1320->regmap, 0x3fc2dfcb, 0x00); regmap_write(rt1320->regmap, 0x3fc2dfca, 0x00); regmap_write(rt1320->regmap, 0x3fc2dfc9, 0x00); regmap_write(rt1320->regmap, 0x3fc2dfc8, 0x00); + /* enable handshake */ + regmap_write(rt1320->regmap, 0x3fc2dfc4, 0x00); + regmap_write(rt1320->regmap, 0xd470, 0xad); + /* minimum phase settings */ + regmap_write(rt1320->regmap, 0xc5c4, 0x10); + regmap_write(rt1320->regmap, 0x20003003, 0x31); + regmap_update_bits(rt1320->regmap, 0x20003002, 0x40, 0x00); + regmap_write(rt1320->regmap, 0xc5b3, 0x01); + regmap_write(rt1320->regmap, 0xc052, 0x11); break; } @@ -1844,14 +2380,35 @@ static void rt1320_vc_preset(struct rt1320_sdw_priv *rt1320) static void rt1321_preset(struct rt1320_sdw_priv *rt1320) { + const struct reg_sequence *blindwrite; unsigned int i, reg, val, delay; + unsigned int array_size; - for (i = 0; i < ARRAY_SIZE(rt1321_blind_write); i++) { - reg = rt1321_blind_write[i].reg; - val = rt1321_blind_write[i].def; - delay = rt1321_blind_write[i].delay_us; + switch (rt1320->version_id) { + case RT1321_VA0: + blindwrite = rt1321_blind_write; + array_size = ARRAY_SIZE(rt1321_blind_write); + break; + case RT1321_VA1: + blindwrite = rt1321_va1_blind_write; + array_size = ARRAY_SIZE(rt1321_va1_blind_write); + break; + case RT1321_VA2: + blindwrite = rt1321_va2_blind_write; + array_size = ARRAY_SIZE(rt1321_va2_blind_write); + break; + default: + dev_err(&rt1320->sdw_slave->dev, "%s: Unknown version ID %d\n", + __func__, rt1320->version_id); + return; + } - if (reg == 0x3fc2dfc3) + for (i = 0; i < array_size; i++) { + reg = blindwrite[i].reg; + val = blindwrite[i].def; + delay = blindwrite[i].delay_us; + + if (reg == 0x1000cd56) rt1320_load_mcu_patch(rt1320); regmap_write(rt1320->regmap, reg, val); @@ -1893,6 +2450,24 @@ static int rt1320_io_init(struct device *dev, struct sdw_slave *slave) regmap_read(rt1320->regmap, RT1320_DEV_ID_0, &val); regmap_read(rt1320->regmap, RT1320_DEV_ID_1, &tmp); rt1320->dev_id = (val << 8) | tmp; + + /* This is a workaround that reads the value twice to obtain the correct result. */ + rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_0, &val); + rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_1, &tmp); + rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_0, &val); + rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_1, &tmp); + val = (val << 8) | tmp; + + if (rt1320->dev_id == RT1321_DEV_ID) { + if (rt1320->version_id == 0x01) + rt1320->version_id = RT1321_VA2; + else if (val == RT1321_DEV_HV_VA0_ID) + rt1320->version_id = RT1321_VA0; + else if (val == RT1321_DEV_HV_VA1_ID) + rt1320->version_id = RT1321_VA1; + else + dev_err(dev, "%s: Unknown version ID 0x%x for RT1321\n", __func__, rt1320->version_id); + } } regmap_read(rt1320->regmap, @@ -2327,25 +2902,12 @@ static const DECLARE_TLV_DB_SCALE(in_vol_tlv, -1725, 75, 0); static int rt1320_r0_load(struct rt1320_sdw_priv *rt1320) { struct device *dev = regmap_get_device(rt1320->regmap); - unsigned int fw_status_addr; - unsigned int fw_ready; + int dspfw_status; int ret = 0; if (!rt1320->r0_l_reg || !rt1320->r0_r_reg) return -EINVAL; - switch (rt1320->dev_id) { - case RT1320_DEV_ID: - fw_status_addr = RT1320_DSPFW_STATUS_ADDR; - break; - case RT1321_DEV_ID: - fw_status_addr = RT1321_DSPFW_STATUS_ADDR; - break; - default: - dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id); - return -EINVAL; - } - regmap_write(rt1320->regmap, SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00); ret = rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00); @@ -2354,9 +2916,8 @@ static int rt1320_r0_load(struct rt1320_sdw_priv *rt1320) goto _timeout_; } - regmap_read(rt1320->regmap, fw_status_addr, &fw_ready); - fw_ready &= 0x1; - if (!fw_ready) { + dspfw_status = rt1320_dspfw_status(rt1320); + if (dspfw_status <= 0) { dev_dbg(dev, "%s, DSP FW is NOT ready\n", __func__); goto _timeout_; } @@ -2459,7 +3020,7 @@ static int rt1320_dspfw_load_put(struct snd_kcontrol *kcontrol, if (!rt1320->hw_init) return 0; - ret = pm_runtime_resume(component->dev); + ret = pm_runtime_resume_and_get(component->dev); if (ret < 0 && ret != -EACCES) return ret; @@ -2470,6 +3031,8 @@ static int rt1320_dspfw_load_put(struct snd_kcontrol *kcontrol, if (!ucontrol->value.integer.value[0]) rt1320->fw_load_done = false; + pm_runtime_mark_last_busy(component->dev); + pm_runtime_put_autosuspend(component->dev); return 0; } diff --git a/sound/soc/codecs/rt1320-sdw.h b/sound/soc/codecs/rt1320-sdw.h index a7b573883dd0..bc0f78e03529 100644 --- a/sound/soc/codecs/rt1320-sdw.h +++ b/sound/soc/codecs/rt1320-sdw.h @@ -17,12 +17,17 @@ #define RT1320_DEV_ID 0x6981 #define RT1321_DEV_ID 0x7045 +#define RT1321_DEV_HV_VA0_ID 0x6997 +#define RT1321_DEV_HV_VA1_ID 0x7071 /* imp-defined registers */ #define RT1320_DEV_VERSION_ID_1 0xc404 #define RT1320_DEV_ID_1 0xc405 #define RT1320_DEV_ID_0 0xc406 +#define RT1320_HV_DEV_ID_0 0xf622 +#define RT1320_HV_DEV_ID_1 0xf623 + #define RT1320_POWER_STATE 0xc560 #define RT1321_PATCH_MAIN_VER 0x1000cffe @@ -94,6 +99,12 @@ enum rt1320_version_id { RT1320_VC, }; +enum rt1321_version_id { + RT1321_VA0, + RT1321_VA1, + RT1321_VA2, +}; + #define RT1320_VER_B_ID 0x07392238 #define RT1320_VAB_MCU_PATCH "realtek/rt1320/rt1320-patch-code-vab.bin" #define RT1320_VC_MCU_PATCH "realtek/rt1320/rt1320-patch-code-vc.bin" @@ -121,6 +132,7 @@ struct rt1320_datafixpoint { int invrs; }; +/* FW parameter id 1300 */ typedef struct FwPara_HwSwGain { unsigned int SwAdvGain; unsigned int SwBasGain; diff --git a/sound/soc/codecs/rt5575-spi.c b/sound/soc/codecs/rt5575-spi.c index 9a349965435b..d5b3a57c8866 100644 --- a/sound/soc/codecs/rt5575-spi.c +++ b/sound/soc/codecs/rt5575-spi.c @@ -17,7 +17,7 @@ struct rt5575_spi_burst_write { u8 cmd; - u32 addr; + __le32 addr; u8 data[RT5575_SPI_BUF_LEN]; u8 dummy; } __packed; diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 8a9af260e5f7..e9819653b30d 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -83,6 +83,8 @@ static const struct reg_sequence rt5650_init_list[] = { {RT5645_PWR_ANLG1, 0x02}, {RT5645_IL_CMD3, 0x6728}, {RT5645_PR_BASE + 0x3a, 0x0000}, + {RT5645_CLSD_OUT_CTRL1, 0x4059}, + {RT5645_GEN_CTRL3, 0x0200}, }; static const struct reg_default rt5645_reg[] = { @@ -1855,13 +1857,9 @@ static int rt5645_spk_event(struct snd_soc_dapm_widget *w, RT5645_PWR_CLS_D_L, RT5645_PWR_CLS_D | RT5645_PWR_CLS_D_R | RT5645_PWR_CLS_D_L); - snd_soc_component_update_bits(component, RT5645_GEN_CTRL3, - RT5645_DET_CLK_MASK, RT5645_DET_CLK_MODE1); break; case SND_SOC_DAPM_PRE_PMD: - snd_soc_component_update_bits(component, RT5645_GEN_CTRL3, - RT5645_DET_CLK_MASK, RT5645_DET_CLK_DIS); snd_soc_component_write(component, RT5645_EQ_CTRL2, 0); snd_soc_component_update_bits(component, RT5645_PWR_DIG1, RT5645_PWR_CLS_D | RT5645_PWR_CLS_D_R | diff --git a/sound/soc/codecs/rt5645.h b/sound/soc/codecs/rt5645.h index bef74b29fd54..a5bfe9861b8b 100644 --- a/sound/soc/codecs/rt5645.h +++ b/sound/soc/codecs/rt5645.h @@ -118,6 +118,7 @@ #define RT5645_A_JD_CTRL1 0x94 #define RT5645_VAD_CTRL4 0x9d #define RT5645_CLSD_OUT_CTRL 0xa0 +#define RT5645_CLSD_OUT_CTRL1 0xa1 /* Function - Digital */ #define RT5645_ADC_EQ_CTRL1 0xae #define RT5645_ADC_EQ_CTRL2 0xaf diff --git a/sound/soc/codecs/rt5677.c b/sound/soc/codecs/rt5677.c index ac084ca008f3..73fc008d558a 100644 --- a/sound/soc/codecs/rt5677.c +++ b/sound/soc/codecs/rt5677.c @@ -6,6 +6,7 @@ * Author: Oder Chiou */ +#include #include #include #include @@ -4767,6 +4768,21 @@ static int rt5677_gpio_direction_in(struct gpio_chip *chip, unsigned offset) return rt5677_update_gpio_bits(rt5677, offset, m, v); } +static int rt5677_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + struct rt5677_priv *rt5677 = gpiochip_get_data(chip); + unsigned int shift = RT5677_GPIOx_DIR_SFT + (offset % 5) * 3; + unsigned int bank = offset / 5; + unsigned int reg = bank ? RT5677_GPIO_CTRL3 : RT5677_GPIO_CTRL2; + int ret; + + ret = regmap_test_bits(rt5677->regmap, reg, BIT(shift)); + if (ret < 0) + return ret; + + return ret ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + /* * Configures the GPIO as * 0 - floating @@ -4834,6 +4850,7 @@ static int rt5677_to_irq(struct gpio_chip *chip, unsigned offset) static const struct gpio_chip rt5677_template_chip = { .label = RT5677_DRV_NAME, .owner = THIS_MODULE, + .get_direction = rt5677_gpio_get_direction, .direction_output = rt5677_gpio_direction_out, .set = rt5677_gpio_set, .direction_input = rt5677_gpio_direction_in, diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c index ec2a35a0cacd..dec8c2147d68 100644 --- a/sound/soc/codecs/rt5682-sdw.c +++ b/sound/soc/codecs/rt5682-sdw.c @@ -769,8 +769,10 @@ static int rt5682_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, RT5682_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt5682->sdw_regmap, false); regcache_cache_only(rt5682->regmap, false); diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c index 30fcca210f05..bb449f08e30c 100644 --- a/sound/soc/codecs/rt700-sdw.c +++ b/sound/soc/codecs/rt700-sdw.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -459,10 +458,8 @@ static void rt700_sdw_remove(struct sdw_slave *slave) { struct rt700_priv *rt700 = dev_get_drvdata(&slave->dev); - if (rt700->hw_init) { - cancel_delayed_work_sync(&rt700->jack_detect_work); - cancel_delayed_work_sync(&rt700->jack_btn_check_work); - } + cancel_delayed_work_sync(&rt700->jack_detect_work); + cancel_delayed_work_sync(&rt700->jack_btn_check_work); pm_runtime_disable(&slave->dev); } @@ -528,8 +525,10 @@ static int rt700_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT700_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt700->regmap, false); regcache_sync_region(rt700->regmap, 0x3000, 0x8fff); diff --git a/sound/soc/codecs/rt711-sdca-sdw.c b/sound/soc/codecs/rt711-sdca-sdw.c index a8164fc3979a..e028a1c3a9ac 100644 --- a/sound/soc/codecs/rt711-sdca-sdw.c +++ b/sound/soc/codecs/rt711-sdca-sdw.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -454,8 +453,10 @@ static int rt711_sdca_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, RT711_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt711->regmap, false); regcache_sync(rt711->regmap); diff --git a/sound/soc/codecs/rt711-sdw.c b/sound/soc/codecs/rt711-sdw.c index df3c43f2ab6b..a0c6a9efa840 100644 --- a/sound/soc/codecs/rt711-sdw.c +++ b/sound/soc/codecs/rt711-sdw.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/codecs/rt712-sdca-dmic.c b/sound/soc/codecs/rt712-sdca-dmic.c index 4c5c2f5ba5ed..85779547653e 100644 --- a/sound/soc/codecs/rt712-sdca-dmic.c +++ b/sound/soc/codecs/rt712-sdca-dmic.c @@ -7,7 +7,6 @@ // #include -#include #include #include #include @@ -911,8 +910,10 @@ static int rt712_sdca_dmic_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT712_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt712->regmap, false); regcache_sync(rt712->regmap); diff --git a/sound/soc/codecs/rt712-sdca-sdw.c b/sound/soc/codecs/rt712-sdca-sdw.c index 581732180473..70d661ce2ef2 100644 --- a/sound/soc/codecs/rt712-sdca-sdw.c +++ b/sound/soc/codecs/rt712-sdca-sdw.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -467,8 +466,10 @@ static int rt712_sdca_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, RT712_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt712->regmap, false); regcache_sync(rt712->regmap); diff --git a/sound/soc/codecs/rt715-sdca-sdw.c b/sound/soc/codecs/rt715-sdca-sdw.c index 4b9815b5628d..1b183b21a4b8 100644 --- a/sound/soc/codecs/rt715-sdca-sdw.c +++ b/sound/soc/codecs/rt715-sdca-sdw.c @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -230,8 +229,10 @@ static int rt715_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT715_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt715->regmap, false); regcache_sync_region(rt715->regmap, diff --git a/sound/soc/codecs/rt715-sdw.c b/sound/soc/codecs/rt715-sdw.c index 7f83a8f1a06e..f5ec348a9628 100644 --- a/sound/soc/codecs/rt715-sdw.c +++ b/sound/soc/codecs/rt715-sdw.c @@ -9,7 +9,6 @@ */ #include #include -#include #include #include #include @@ -507,8 +506,10 @@ static int rt715_dev_resume(struct device *dev) return 0; ret = sdw_slave_wait_for_init(slave, RT715_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt715->regmap, false); regcache_sync_region(rt715->regmap, 0x3000, 0x8fff); diff --git a/sound/soc/codecs/rt721-sdca-sdw.c b/sound/soc/codecs/rt721-sdca-sdw.c index 58606209316a..041b381e582b 100644 --- a/sound/soc/codecs/rt721-sdca-sdw.c +++ b/sound/soc/codecs/rt721-sdca-sdw.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -505,8 +504,10 @@ static int rt721_sdca_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, RT721_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt721->regmap, false); regcache_sync(rt721->regmap); diff --git a/sound/soc/codecs/rt722-sdca-sdw.c b/sound/soc/codecs/rt722-sdca-sdw.c index 0f76492ff915..e68aa0350a5b 100644 --- a/sound/soc/codecs/rt722-sdca-sdw.c +++ b/sound/soc/codecs/rt722-sdca-sdw.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -552,8 +551,10 @@ static int rt722_sdca_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, RT722_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(rt722->regmap, false); regcache_sync(rt722->regmap); diff --git a/sound/soc/codecs/rt9123.c b/sound/soc/codecs/rt9123.c index 84fd3d6861de..07eaf275c9e9 100644 --- a/sound/soc/codecs/rt9123.c +++ b/sound/soc/codecs/rt9123.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/rt9123p.c b/sound/soc/codecs/rt9123p.c index d509659e735b..584fcb78cd3f 100644 --- a/sound/soc/codecs/rt9123p.c +++ b/sound/soc/codecs/rt9123p.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/rtq9124.c b/sound/soc/codecs/rtq9124.c index 186904b31434..2a041894bc0c 100644 --- a/sound/soc/codecs/rtq9124.c +++ b/sound/soc/codecs/rtq9124.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/rtq9128.c b/sound/soc/codecs/rtq9128.c index 14a2c0723d33..573200e5062f 100644 --- a/sound/soc/codecs/rtq9128.c +++ b/sound/soc/codecs/rtq9128.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/sdw-mockup.c b/sound/soc/codecs/sdw-mockup.c index b7e6546f1b5a..93f3fd1882a5 100644 --- a/sound/soc/codecs/sdw-mockup.c +++ b/sound/soc/codecs/sdw-mockup.c @@ -8,7 +8,6 @@ // #include -#include #include #include #include diff --git a/sound/soc/codecs/simple-amplifier.c b/sound/soc/codecs/simple-amplifier.c index ca0e6ce8cc37..ca53b08c0b33 100644 --- a/sound/soc/codecs/simple-amplifier.c +++ b/sound/soc/codecs/simple-amplifier.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/codecs/sma1303.c b/sound/soc/codecs/sma1303.c index c7aaf98ef71e..15758eb13f8c 100644 --- a/sound/soc/codecs/sma1303.c +++ b/sound/soc/codecs/sma1303.c @@ -7,7 +7,6 @@ // Auther: Gyuhwa Park // Kiseok Jo -#include #include #include #include @@ -1778,6 +1777,8 @@ static void sma1303_i2c_remove(struct i2c_client *client) struct sma1303_priv *sma1303 = (struct sma1303_priv *) i2c_get_clientdata(client); + if (sma1303->attr_grp) + sysfs_remove_group(sma1303->kobj, sma1303->attr_grp); cancel_delayed_work_sync(&sma1303->check_fault_work); } diff --git a/sound/soc/codecs/src4xxx-i2c.c b/sound/soc/codecs/src4xxx-i2c.c index 34b3abdb9a70..4157f7787801 100644 --- a/sound/soc/codecs/src4xxx-i2c.c +++ b/sound/soc/codecs/src4xxx-i2c.c @@ -6,7 +6,6 @@ // Author: Matt Flax #include -#include #include #include diff --git a/sound/soc/codecs/tac5xx2-sdw.c b/sound/soc/codecs/tac5xx2-sdw.c index bb12cfb6da12..ace06f5ab58c 100644 --- a/sound/soc/codecs/tac5xx2-sdw.c +++ b/sound/soc/codecs/tac5xx2-sdw.c @@ -1445,8 +1445,10 @@ static s32 tac5xx2_sdca_dev_resume(struct device *dev) } ret = sdw_slave_wait_for_init(slave, TAC5XX2_PROBE_TIMEOUT_MS); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(tac_dev->regmap, false); regcache_mark_dirty(tac_dev->regmap); diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c index 7d70e7e3f24f..3d0b116544cc 100644 --- a/sound/soc/codecs/tas2783-sdw.c +++ b/sound/soc/codecs/tas2783-sdw.c @@ -100,6 +100,8 @@ struct tas2783_prv { wait_queue_head_t fw_wait; bool fw_dl_task_done; bool fw_dl_success; + /* use fallback fw name */ + bool fw_use_fallback; }; static const struct reg_default tas2783_reg_default[] = { @@ -740,11 +742,19 @@ static void tas2783_fw_ready(const struct firmware *fmw, void *context) goto out; } + /* firmware binary not found*/ if (!fmw || !fmw->data) { - /* firmware binary not found*/ - dev_err(tas_dev->dev, - "Failed to read fw binary %s\n", - tas_dev->rca_binaryname); + if (!tas_dev->fw_use_fallback) { + tas_dev->fw_use_fallback = true; + dev_info(tas_dev->dev, + "Failed to read preferred fw binary: %s, attempting fallback binary load\n", + tas_dev->rca_binaryname); + } else { + dev_err(tas_dev->dev, + "Failed to read fallback fw binary %s\n", + tas_dev->rca_binaryname); + } + ret = -EINVAL; goto out; } @@ -1083,8 +1093,10 @@ static s32 tas2783_sdca_dev_resume(struct device *dev) int ret; ret = sdw_slave_wait_for_init(slave, TAS2783_PROBE_TIMEOUT); - if (ret) + if (ret) { + sdw_show_ping_status(slave->bus, true); return ret; + } regcache_cache_only(tas_dev->regmap, false); regcache_sync(tas_dev->regmap); @@ -1103,13 +1115,16 @@ static void tas_generate_fw_name(struct sdw_slave *slave, char *name, size_t siz bool pci_found = false; #if IS_ENABLED(CONFIG_PCI) struct device *dev = &slave->dev; + struct tas2783_prv *tas_dev = dev_get_drvdata(&slave->dev); struct pci_dev *pci = NULL; + const char *fw_uid_prefix = tas_dev->fw_use_fallback ? "" : "0x"; for (; dev; dev = dev->parent) { if (dev->bus == &pci_bus_type) { pci = to_pci_dev(dev); - scnprintf(name, size, "%04X-%1X-%1X.bin", - pci->subsystem_device, bus->link_id, unique_id); + scnprintf(name, size, "%04X-%1X-%s%1X.bin", + pci->subsystem_device, bus->link_id, + fw_uid_prefix, unique_id); pci_found = true; break; } @@ -1121,28 +1136,15 @@ static void tas_generate_fw_name(struct sdw_slave *slave, char *name, size_t siz bus->link_id, unique_id); } -static s32 tas_io_init(struct device *dev, struct sdw_slave *slave) +static s32 tas_fw_load(struct tas2783_prv *tas_dev, struct sdw_slave *slave) { - struct tas2783_prv *tas_dev = dev_get_drvdata(dev); s32 ret; u8 unique_id = tas_dev->sdw_peripheral->id.unique_id; - if (tas_dev->hw_init) - return 0; - - tas_dev->fw_dl_task_done = false; - tas_dev->fw_dl_success = false; - - ret = regmap_write(tas_dev->regmap, TAS2783_SW_RESET, 0x1); - if (ret) { - dev_err(dev, "sw reset failed, err=%d", ret); - return ret; - } - usleep_range(2000, 2200); - tas_generate_fw_name(slave, tas_dev->rca_binaryname, sizeof(tas_dev->rca_binaryname)); + tas_dev->fw_dl_task_done = false; ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT, tas_dev->rca_binaryname, tas_dev->dev, GFP_KERNEL, tas_dev, tas2783_fw_ready); @@ -1157,8 +1159,35 @@ static s32 tas_io_init(struct device *dev, struct sdw_slave *slave) msecs_to_jiffies(TIMEOUT_FW_DL_MS)); if (!ret) { dev_err(tas_dev->dev, "fw request, wait_event timeout\n"); - ret = -EAGAIN; - } else { + return -EAGAIN; + } + + return 0; +} + +static s32 tas_io_init(struct device *dev, struct sdw_slave *slave) +{ + struct tas2783_prv *tas_dev = dev_get_drvdata(dev); + s32 ret; + + if (tas_dev->hw_init) + return 0; + + tas_dev->fw_dl_success = false; + + ret = regmap_write(tas_dev->regmap, TAS2783_SW_RESET, 0x1); + if (ret) { + dev_err(dev, "sw reset failed, err=%d", ret); + return ret; + } + usleep_range(2000, 2200); + + tas_dev->fw_use_fallback = false; + ret = tas_fw_load(tas_dev, slave); + if (!ret && tas_dev->fw_use_fallback) + ret = tas_fw_load(tas_dev, slave); + + if (!ret) { if (tas_dev->sa_func_data) ret = sdca_regmap_write_init(dev, tas_dev->regmap, tas_dev->sa_func_data); diff --git a/sound/soc/codecs/tas675x.c b/sound/soc/codecs/tas675x.c index 6f89a422f3c6..82526362de7b 100644 --- a/sound/soc/codecs/tas675x.c +++ b/sound/soc/codecs/tas675x.c @@ -954,10 +954,10 @@ static const struct snd_kcontrol_new tas675x_snd_controls[] = { /* Temperature and Voltage Monitoring */ SOC_SINGLE_RO("PVDD Sense", TAS675X_PVDD_SENSE_REG, 0, 0xFF), SOC_SINGLE_RO("Global Temperature", TAS675X_TEMP_GLOBAL_REG, 0, 0xFF), - SOC_SINGLE_RO("CH1 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 6, 3), - SOC_SINGLE_RO("CH2 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 4, 3), - SOC_SINGLE_RO("CH3 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 2, 3), - SOC_SINGLE_RO("CH4 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 0, 3), + SOC_SINGLE_RO("CH1 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 0, 7), + SOC_SINGLE_RO("CH2 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 3, 7), + SOC_SINGLE_RO("CH3 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 0, 7), + SOC_SINGLE_RO("CH4 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 3, 7), /* Speaker Protection & Detection */ SOC_SINGLE("Tweeter Detection Switch", TAS675X_TWEETER_DETECT_CTRL_REG, 0, 1, 1), @@ -1133,7 +1133,7 @@ static int tas675x_hw_params(struct snd_pcm_substream *substream, * Single clock domain: SDIN and SDOUT share one SCLK/FSYNC pair, * so all active DAIs must use the same sample rate. */ - if ((tas->active_playback_dais || tas->active_capture_dais) && + if ((READ_ONCE(tas->active_playback_dais) || READ_ONCE(tas->active_capture_dais)) && tas->rate && tas->rate != rate) { dev_err(component->dev, "Rate %u conflicts with active rate %u\n", @@ -1397,14 +1397,14 @@ static int tas675x_mute_stream(struct snd_soc_dai *dai, int mute, int direction) set_bit(dai->id, &tas->active_playback_dais); /* Last playback stream */ - if (mute && !tas->active_playback_dais) { + if (mute && !READ_ONCE(tas->active_playback_dais)) { ret = tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH); regmap_read(tas->regmap, TAS675X_CLK_FAULT_LATCHED_REG, &discard); return ret; } return tas675x_set_state_all(tas, - tas->active_playback_dais ? + READ_ONCE(tas->active_playback_dais) ? TAS675X_STATE_PLAY_BOTH : TAS675X_STATE_SLEEP_BOTH); } diff --git a/sound/soc/codecs/tlv320aic26.c b/sound/soc/codecs/tlv320aic26.c index e5dfb3d752a3..84e954311c1b 100644 --- a/sound/soc/codecs/tlv320aic26.c +++ b/sound/soc/codecs/tlv320aic26.c @@ -320,8 +320,14 @@ static int aic26_probe(struct snd_soc_component *component) return 0; } +static void aic26_remove(struct snd_soc_component *component) +{ + device_remove_file(component->dev, &dev_attr_keyclick); +} + static const struct snd_soc_component_driver aic26_soc_component_dev = { .probe = aic26_probe, + .remove = aic26_remove, .controls = aic26_snd_controls, .num_controls = ARRAY_SIZE(aic26_snd_controls), .dapm_widgets = tlv320aic26_dapm_widgets, diff --git a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c index b12c1952823b..b38393a8130f 100644 --- a/sound/soc/codecs/tlv320aic3x.c +++ b/sound/soc/codecs/tlv320aic3x.c @@ -1049,11 +1049,13 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) { + static const u8 dual_rate_q[] = {4, 8, 9, 12, 16}; struct snd_soc_component *component = dai->component; struct aic3x_priv *aic3x = snd_soc_component_get_drvdata(component); int codec_clk = 0, bypass_pll = 0, fsref, last_clk = 0; u8 data, j, r, p, pll_q, pll_p = 1, pll_r = 1, pll_j = 1; u16 d, pll_d = 1; + bool dual_rate; int clk; int width = aic3x->slot_width; @@ -1079,14 +1081,25 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, /* Fsref can be 44100 or 48000 */ fsref = (params_rate(params) % 11025 == 0) ? 44100 : 48000; + dual_rate = params_rate(params) >= 64000; /* Try to find a value for Q which allows us to bypass the PLL and * generate CODEC_CLK directly. */ - for (pll_q = 2; pll_q < 18; pll_q++) - if (aic3x->sysclk / (128 * pll_q) == fsref) { - bypass_pll = 1; - break; + if (dual_rate) { + for (int i = 0; i < ARRAY_SIZE(dual_rate_q); i++) { + pll_q = dual_rate_q[i]; + if (aic3x->sysclk / (128 * pll_q) == fsref) { + bypass_pll = 1; + break; + } } + } else { + for (pll_q = 2; pll_q < 18; pll_q++) + if (aic3x->sysclk / (128 * pll_q) == fsref) { + bypass_pll = 1; + break; + } + } if (bypass_pll) { pll_q &= 0xf; @@ -1106,13 +1119,13 @@ static int aic3x_hw_params(struct snd_pcm_substream *substream, * right DAC to right channel input */ data = (LDAC2LCH | RDAC2RCH); data |= (fsref == 44100) ? FSREF_44100 : FSREF_48000; - if (params_rate(params) >= 64000) + if (dual_rate) data |= DUAL_RATE_MODE; snd_soc_component_write(component, AIC3X_CODEC_DATAPATH_REG, data); /* codec sample rate select */ data = (fsref * 20) / params_rate(params); - if (params_rate(params) < 64000) + if (!dual_rate) data /= 2; data /= 5; data -= 2; diff --git a/sound/soc/codecs/uda1334.c b/sound/soc/codecs/uda1334.c index f799772ff747..54c5cb5b3d4b 100644 --- a/sound/soc/codecs/uda1334.c +++ b/sound/soc/codecs/uda1334.c @@ -4,7 +4,6 @@ // // Based on WM8523 ALSA SoC Audio driver written by Mark Brown -#include #include #include #include diff --git a/sound/soc/codecs/wm8510.c b/sound/soc/codecs/wm8510.c index 589a89564813..137dcd3d7487 100644 --- a/sound/soc/codecs/wm8510.c +++ b/sound/soc/codecs/wm8510.c @@ -7,7 +7,6 @@ * Author: Liam Girdwood */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8523.c b/sound/soc/codecs/wm8523.c index 65108a041c92..b8832a1d61fe 100644 --- a/sound/soc/codecs/wm8523.c +++ b/sound/soc/codecs/wm8523.c @@ -7,7 +7,6 @@ * Author: Mark Brown */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8524.c b/sound/soc/codecs/wm8524.c index 6b1a7450b0ac..23daf158f35f 100644 --- a/sound/soc/codecs/wm8524.c +++ b/sound/soc/codecs/wm8524.c @@ -8,7 +8,6 @@ * Based on WM8523 ALSA SoC Audio driver written by Mark Brown */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8580.c b/sound/soc/codecs/wm8580.c index ca7bbe5d4fc3..eb374ba6e5b5 100644 --- a/sound/soc/codecs/wm8580.c +++ b/sound/soc/codecs/wm8580.c @@ -15,7 +15,6 @@ * the secondary audio interfaces are not. */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8711.c b/sound/soc/codecs/wm8711.c index 5271966b1615..2db8829661c3 100644 --- a/sound/soc/codecs/wm8711.c +++ b/sound/soc/codecs/wm8711.c @@ -9,7 +9,6 @@ * Based on wm8731.c by Richard Purdie */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8728.c b/sound/soc/codecs/wm8728.c index 6e6fd77c3020..3109a6a0df74 100644 --- a/sound/soc/codecs/wm8728.c +++ b/sound/soc/codecs/wm8728.c @@ -7,7 +7,6 @@ * Author: Mark Brown */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8731-i2c.c b/sound/soc/codecs/wm8731-i2c.c index 5d19fcc46606..f44f4d3d9394 100644 --- a/sound/soc/codecs/wm8731-i2c.c +++ b/sound/soc/codecs/wm8731-i2c.c @@ -11,7 +11,6 @@ */ #include -#include #include #include "wm8731.h" diff --git a/sound/soc/codecs/wm8731-spi.c b/sound/soc/codecs/wm8731-spi.c index c02086afa7fb..29e58e1e6b79 100644 --- a/sound/soc/codecs/wm8731-spi.c +++ b/sound/soc/codecs/wm8731-spi.c @@ -11,7 +11,6 @@ */ #include -#include #include #include "wm8731.h" diff --git a/sound/soc/codecs/wm8737.c b/sound/soc/codecs/wm8737.c index 4eb42d19bc7e..33a3f88fffb3 100644 --- a/sound/soc/codecs/wm8737.c +++ b/sound/soc/codecs/wm8737.c @@ -7,7 +7,6 @@ * Author: Mark Brown */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8753.c b/sound/soc/codecs/wm8753.c index 95b23504f68d..ac4008b4832d 100644 --- a/sound/soc/codecs/wm8753.c +++ b/sound/soc/codecs/wm8753.c @@ -26,7 +26,6 @@ * an alsa kcontrol. This allows the PCM to remain open. */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8770.c b/sound/soc/codecs/wm8770.c index d382b476c89c..b8b4d1e823e6 100644 --- a/sound/soc/codecs/wm8770.c +++ b/sound/soc/codecs/wm8770.c @@ -7,7 +7,6 @@ * Author: Dimitris Papastamos */ -#include #include #include #include diff --git a/sound/soc/codecs/wm8776.c b/sound/soc/codecs/wm8776.c index f3b02c77314f..a8e4f71c77d1 100644 --- a/sound/soc/codecs/wm8776.c +++ b/sound/soc/codecs/wm8776.c @@ -9,7 +9,6 @@ * TODO: Input ALC/limiter support */ -#include #include #include #include diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c index 5fda9b647c70..0b28bcfa47fe 100644 --- a/sound/soc/fsl/fsl_asrc.c +++ b/sound/soc/fsl/fsl_asrc.c @@ -222,10 +222,9 @@ static int fsl_asrc_request_pair(int channels, struct fsl_asrc_pair *pair) enum asrc_pair_index index = ASRC_INVALID_PAIR; struct fsl_asrc *asrc = pair->asrc; struct device *dev = &asrc->pdev->dev; - unsigned long lock_flags; int i, ret = 0; - spin_lock_irqsave(&asrc->lock, lock_flags); + guard(spinlock_irqsave)(&asrc->lock); for (i = ASRC_PAIR_A; i < ASRC_PAIR_MAX_NUM; i++) { if (asrc->pair[i] != NULL) @@ -250,8 +249,6 @@ static int fsl_asrc_request_pair(int channels, struct fsl_asrc_pair *pair) pair->index = index; } - spin_unlock_irqrestore(&asrc->lock, lock_flags); - return ret; } @@ -265,19 +262,16 @@ static void fsl_asrc_release_pair(struct fsl_asrc_pair *pair) { struct fsl_asrc *asrc = pair->asrc; enum asrc_pair_index index = pair->index; - unsigned long lock_flags; /* Make sure the pair is disabled */ regmap_update_bits(asrc->regmap, REG_ASRCTR, ASRCTR_ASRCEi_MASK(index), 0); - spin_lock_irqsave(&asrc->lock, lock_flags); + guard(spinlock_irqsave)(&asrc->lock); asrc->channel_avail += pair->channels; asrc->pair[index] = NULL; pair->error = 0; - - spin_unlock_irqrestore(&asrc->lock, lock_flags); } /** diff --git a/sound/soc/fsl/fsl_asrc_dma.c b/sound/soc/fsl/fsl_asrc_dma.c index 5aa96af994c4..38f2b7c63133 100644 --- a/sound/soc/fsl/fsl_asrc_dma.c +++ b/sound/soc/fsl/fsl_asrc_dma.c @@ -288,6 +288,26 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, config_be.dst_addr_width = buswidth; config_be.dst_maxburst = dma_params_be->maxburst; + /* + * For eDMA, the back-end may report a maxburst size that is not evenly + * divisible by the channel count. This causes the DMA transfer length + * to misalign with the FIFO boundary, resulting in wrong data and + * audible noise. Align maxburst to the nearest valid boundary: + * - If maxburst >= channel count, override to the channel count so + * each transfer equals exactly one audio frame. + * - If maxburst < channel count, override to 1 to avoid partial-frame + * transfers. + */ + if (asrc->use_edma && (dma_params_be->maxburst % params_channels(params))) { + if (dma_params_be->maxburst >= params_channels(params)) { + config_be.src_maxburst = params_channels(params); + config_be.dst_maxburst = params_channels(params); + } else { + config_be.src_maxburst = 1; + config_be.dst_maxburst = 1; + } + } + memset(&audio_config, 0, sizeof(audio_config)); config_be.peripheral_config = &audio_config; config_be.peripheral_size = sizeof(audio_config); diff --git a/sound/soc/fsl/fsl_aud2htx.c b/sound/soc/fsl/fsl_aud2htx.c index da401561e2de..8f2aa8f7d4e8 100644 --- a/sound/soc/fsl/fsl_aud2htx.c +++ b/sound/soc/fsl/fsl_aud2htx.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index f819f33ec46b..d4140e170edf 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -286,7 +286,6 @@ static int fsl_audmix_dai_trigger(struct snd_pcm_substream *substream, int cmd, struct snd_soc_dai *dai) { struct fsl_audmix *priv = snd_soc_dai_get_drvdata(dai); - unsigned long lock_flags; /* Capture stream shall not be handled */ if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) @@ -296,16 +295,14 @@ static int fsl_audmix_dai_trigger(struct snd_pcm_substream *substream, int cmd, case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - spin_lock_irqsave(&priv->lock, lock_flags); - priv->tdms |= BIT(dai->driver->id); - spin_unlock_irqrestore(&priv->lock, lock_flags); + scoped_guard(spinlock_irqsave, &priv->lock) + priv->tdms |= BIT(dai->driver->id); break; case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - spin_lock_irqsave(&priv->lock, lock_flags); - priv->tdms &= ~BIT(dai->driver->id); - spin_unlock_irqrestore(&priv->lock, lock_flags); + scoped_guard(spinlock_irqsave, &priv->lock) + priv->tdms &= ~BIT(dai->driver->id); break; default: return -EINVAL; @@ -457,6 +454,9 @@ static const struct of_device_id fsl_audmix_ids[] = { }; MODULE_DEVICE_TABLE(of, fsl_audmix_ids); +static int fsl_audmix_runtime_resume(struct device *dev); +static int fsl_audmix_runtime_suspend(struct device *dev); + static int fsl_audmix_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -488,13 +488,25 @@ static int fsl_audmix_probe(struct platform_device *pdev) spin_lock_init(&priv->lock); platform_set_drvdata(pdev, priv); pm_runtime_enable(dev); + if (!pm_runtime_enabled(dev)) { + ret = fsl_audmix_runtime_resume(dev); + if (ret) + goto err_disable_pm; + } + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto err_pm_get_sync; + + /* To enable regmap cache only when runtime PM enabled */ + pm_runtime_put(dev); ret = devm_snd_soc_register_component(dev, &fsl_audmix_component, fsl_audmix_dai, ARRAY_SIZE(fsl_audmix_dai)); if (ret) { dev_err(dev, "failed to register ASoC DAI\n"); - goto err_disable_pm; + goto err_pm_get_sync; } /* @@ -506,12 +518,15 @@ static int fsl_audmix_probe(struct platform_device *pdev) if (IS_ERR(priv->pdev)) { ret = PTR_ERR(priv->pdev); dev_err(dev, "failed to register platform: %d\n", ret); - goto err_disable_pm; + goto err_pm_get_sync; } } return 0; +err_pm_get_sync: + if (!pm_runtime_status_suspended(dev)) + fsl_audmix_runtime_suspend(dev); err_disable_pm: pm_runtime_disable(dev); return ret; @@ -522,6 +537,8 @@ static void fsl_audmix_remove(struct platform_device *pdev) struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev); pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + fsl_audmix_runtime_suspend(&pdev->dev); if (priv->pdev) platform_device_unregister(priv->pdev); diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index 114a6c0b6b73..edfd943197a0 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -1025,7 +1025,6 @@ static int fsl_easrc_config_context(struct fsl_asrc *easrc, unsigned int ctx_id) struct fsl_easrc_ctx_priv *ctx_priv; struct fsl_asrc_pair *ctx; struct device *dev; - unsigned long lock_flags; int ret; if (!easrc) @@ -1053,9 +1052,8 @@ static int fsl_easrc_config_context(struct fsl_asrc *easrc, unsigned int ctx_id) if (ret) return ret; - spin_lock_irqsave(&easrc->lock, lock_flags); - ret = fsl_easrc_config_slot(easrc, ctx->index); - spin_unlock_irqrestore(&easrc->lock, lock_flags); + scoped_guard(spinlock_irqsave, &easrc->lock) + ret = fsl_easrc_config_slot(easrc, ctx->index); if (ret) return ret; @@ -1301,13 +1299,12 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx) enum asrc_pair_index index = ASRC_INVALID_PAIR; struct fsl_asrc *easrc = ctx->asrc; struct device *dev; - unsigned long lock_flags; int ret = 0; int i; dev = &easrc->pdev->dev; - spin_lock_irqsave(&easrc->lock, lock_flags); + guard(spinlock_irqsave)(&easrc->lock); for (i = ASRC_PAIR_A; i < EASRC_CTX_MAX_NUM; i++) { if (easrc->pair[i]) @@ -1331,8 +1328,6 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx) easrc->channel_avail -= channels; } - spin_unlock_irqrestore(&easrc->lock, lock_flags); - return ret; } @@ -1343,7 +1338,6 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx) */ static void fsl_easrc_release_context(struct fsl_asrc_pair *ctx) { - unsigned long lock_flags; struct fsl_asrc *easrc; if (!ctx) @@ -1351,14 +1345,12 @@ static void fsl_easrc_release_context(struct fsl_asrc_pair *ctx) easrc = ctx->asrc; - spin_lock_irqsave(&easrc->lock, lock_flags); + guard(spinlock_irqsave)(&easrc->lock); fsl_easrc_release_slot(easrc, ctx->index); easrc->channel_avail += ctx->channels; easrc->pair[ctx->index] = NULL; - - spin_unlock_irqrestore(&easrc->lock, lock_flags); } /* @@ -2292,15 +2284,13 @@ static int fsl_easrc_runtime_suspend(struct device *dev) { struct fsl_asrc *easrc = dev_get_drvdata(dev); struct fsl_easrc_priv *easrc_priv = easrc->private; - unsigned long lock_flags; regcache_cache_only(easrc->regmap, true); clk_disable_unprepare(easrc->mem_clk); - spin_lock_irqsave(&easrc->lock, lock_flags); - easrc_priv->firmware_loaded = 0; - spin_unlock_irqrestore(&easrc->lock, lock_flags); + scoped_guard(spinlock_irqsave, &easrc->lock) + easrc_priv->firmware_loaded = 0; return 0; } @@ -2311,7 +2301,6 @@ static int fsl_easrc_runtime_resume(struct device *dev) struct fsl_easrc_priv *easrc_priv = easrc->private; struct fsl_easrc_ctx_priv *ctx_priv; struct fsl_asrc_pair *ctx; - unsigned long lock_flags; int ret; int i; @@ -2323,13 +2312,11 @@ static int fsl_easrc_runtime_resume(struct device *dev) regcache_mark_dirty(easrc->regmap); regcache_sync(easrc->regmap); - spin_lock_irqsave(&easrc->lock, lock_flags); - if (easrc_priv->firmware_loaded) { - spin_unlock_irqrestore(&easrc->lock, lock_flags); - goto skip_load; + scoped_guard(spinlock_irqsave, &easrc->lock) { + if (easrc_priv->firmware_loaded) + return 0; + easrc_priv->firmware_loaded = 1; } - easrc_priv->firmware_loaded = 1; - spin_unlock_irqrestore(&easrc->lock, lock_flags); ret = fsl_easrc_get_firmware(easrc); if (ret) { @@ -2377,9 +2364,6 @@ static int fsl_easrc_runtime_resume(struct device *dev) goto disable_mem_clk; } -skip_load: - return 0; - disable_mem_clk: clk_disable_unprepare(easrc->mem_clk); return ret; diff --git a/sound/soc/fsl/fsl_esai.c b/sound/soc/fsl/fsl_esai.c index cde0b0c6c1ef..4a530a6c33f0 100644 --- a/sound/soc/fsl/fsl_esai.c +++ b/sound/soc/fsl/fsl_esai.c @@ -709,10 +709,9 @@ static void fsl_esai_hw_reset(struct work_struct *work) { struct fsl_esai *esai_priv = container_of(work, struct fsl_esai, work); bool tx = true, rx = false, enabled[2]; - unsigned long lock_flags; u32 tfcr, rfcr; - spin_lock_irqsave(&esai_priv->lock, lock_flags); + guard(spinlock_irqsave)(&esai_priv->lock); /* Save the registers */ regmap_read(esai_priv->regmap, REG_ESAI_TFCR, &tfcr); regmap_read(esai_priv->regmap, REG_ESAI_RFCR, &rfcr); @@ -750,8 +749,6 @@ static void fsl_esai_hw_reset(struct work_struct *work) fsl_esai_trigger_start(esai_priv, tx); if (enabled[rx]) fsl_esai_trigger_start(esai_priv, rx); - - spin_unlock_irqrestore(&esai_priv->lock, lock_flags); } static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd, @@ -759,7 +756,6 @@ static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd, { struct fsl_esai *esai_priv = snd_soc_dai_get_drvdata(dai); bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; - unsigned long lock_flags; esai_priv->channels[tx] = substream->runtime->channels; @@ -767,16 +763,14 @@ static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd, case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - spin_lock_irqsave(&esai_priv->lock, lock_flags); - fsl_esai_trigger_start(esai_priv, tx); - spin_unlock_irqrestore(&esai_priv->lock, lock_flags); + scoped_guard(spinlock_irqsave, &esai_priv->lock) + fsl_esai_trigger_start(esai_priv, tx); break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - spin_lock_irqsave(&esai_priv->lock, lock_flags); - fsl_esai_trigger_stop(esai_priv, tx); - spin_unlock_irqrestore(&esai_priv->lock, lock_flags); + scoped_guard(spinlock_irqsave, &esai_priv->lock) + fsl_esai_trigger_stop(esai_priv, tx); break; default: return -EINVAL; diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c index 1b9be85b34c2..ad1206ed9882 100644 --- a/sound/soc/fsl/fsl_spdif.c +++ b/sound/soc/fsl/fsl_spdif.c @@ -853,17 +853,15 @@ static int fsl_spdif_subcode_get(struct snd_kcontrol *kcontrol, struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol); struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(cpu_dai); struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control; - unsigned long flags; int ret = -EAGAIN; - spin_lock_irqsave(&ctrl->ctl_lock, flags); + guard(spinlock_irqsave)(&ctrl->ctl_lock); if (ctrl->ready_buf) { int idx = (ctrl->ready_buf - 1) * SPDIF_UBITS_SIZE; memcpy(&ucontrol->value.iec958.subcode[0], &ctrl->subcode[idx], SPDIF_UBITS_SIZE); ret = 0; } - spin_unlock_irqrestore(&ctrl->ctl_lock, flags); return ret; } @@ -885,17 +883,15 @@ static int fsl_spdif_qget(struct snd_kcontrol *kcontrol, struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol); struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(cpu_dai); struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control; - unsigned long flags; int ret = -EAGAIN; - spin_lock_irqsave(&ctrl->ctl_lock, flags); + guard(spinlock_irqsave)(&ctrl->ctl_lock); if (ctrl->ready_buf) { int idx = (ctrl->ready_buf - 1) * SPDIF_QSUB_SIZE; memcpy(&ucontrol->value.bytes.data[0], &ctrl->qsub[idx], SPDIF_QSUB_SIZE); ret = 0; } - spin_unlock_irqrestore(&ctrl->ctl_lock, flags); return ret; } diff --git a/sound/soc/fsl/fsl_ssi.c b/sound/soc/fsl/fsl_ssi.c index b2e1da1781ae..dc022976c982 100644 --- a/sound/soc/fsl/fsl_ssi.c +++ b/sound/soc/fsl/fsl_ssi.c @@ -1218,13 +1218,13 @@ static void fsl_ssi_ac97_write(struct snd_ac97 *ac97, unsigned short reg, if (reg > 0x7f) return; - mutex_lock(&fsl_ac97_data->ac97_reg_lock); + guard(mutex)(&fsl_ac97_data->ac97_reg_lock); ret = clk_prepare_enable(fsl_ac97_data->clk); if (ret) { pr_err("ac97 write clk_prepare_enable failed: %d\n", ret); - goto ret_unlock; + return; } lreg = reg << 12; @@ -1238,9 +1238,6 @@ static void fsl_ssi_ac97_write(struct snd_ac97 *ac97, unsigned short reg, udelay(100); clk_disable_unprepare(fsl_ac97_data->clk); - -ret_unlock: - mutex_unlock(&fsl_ac97_data->ac97_reg_lock); } static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97, @@ -1252,12 +1249,12 @@ static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97, unsigned int lreg; int ret; - mutex_lock(&fsl_ac97_data->ac97_reg_lock); + guard(mutex)(&fsl_ac97_data->ac97_reg_lock); ret = clk_prepare_enable(fsl_ac97_data->clk); if (ret) { pr_err("ac97 read clk_prepare_enable failed: %d\n", ret); - goto ret_unlock; + return val; } lreg = (reg & 0x7f) << 12; @@ -1272,8 +1269,6 @@ static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97, clk_disable_unprepare(fsl_ac97_data->clk); -ret_unlock: - mutex_unlock(&fsl_ac97_data->ac97_reg_lock); return val; } diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index 6677d3bf36ec..41d100500534 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -797,10 +797,9 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, { struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai); bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; - unsigned long lock_flags; int ret = 0; - spin_lock_irqsave(&xcvr->lock, lock_flags); + guard(spinlock_irqsave)(&xcvr->lock); switch (cmd) { case SNDRV_PCM_TRIGGER_START: @@ -812,7 +811,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_EXT_CTRL_DPTH_RESET(tx)); if (ret < 0) { dev_err(dai->dev, "Failed to set DPATH RESET: %d\n", ret); - goto release_lock; + return ret; } if (tx) { @@ -824,7 +823,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_ISR_CMDC_TX_EN); if (ret < 0) { dev_err(dai->dev, "err updating isr %d\n", ret); - goto release_lock; + return ret; } fallthrough; case FSL_XCVR_MODE_SPDIF: @@ -833,7 +832,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_TX_DPTH_CTRL_STRT_DATA_TX); if (ret < 0) { dev_err(dai->dev, "Failed to start DATA_TX: %d\n", ret); - goto release_lock; + return ret; } break; } @@ -844,14 +843,14 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_EXT_CTRL_DMA_DIS(tx), 0); if (ret < 0) { dev_err(dai->dev, "Failed to enable DMA: %d\n", ret); - goto release_lock; + return ret; } ret = regmap_update_bits(xcvr->regmap, FSL_XCVR_EXT_IER0, FSL_XCVR_IRQ_EARC_ALL, FSL_XCVR_IRQ_EARC_ALL); if (ret < 0) { dev_err(dai->dev, "Error while setting IER0: %d\n", ret); - goto release_lock; + return ret; } /* clear DPATH RESET */ @@ -860,7 +859,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, 0); if (ret < 0) { dev_err(dai->dev, "Failed to clear DPATH RESET: %d\n", ret); - goto release_lock; + return ret; } break; @@ -873,14 +872,14 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_EXT_CTRL_DMA_DIS(tx)); if (ret < 0) { dev_err(dai->dev, "Failed to disable DMA: %d\n", ret); - goto release_lock; + return ret; } ret = regmap_update_bits(xcvr->regmap, FSL_XCVR_EXT_IER0, FSL_XCVR_IRQ_EARC_ALL, 0); if (ret < 0) { dev_err(dai->dev, "Failed to clear IER0: %d\n", ret); - goto release_lock; + return ret; } if (tx) { @@ -891,7 +890,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, FSL_XCVR_TX_DPTH_CTRL_STRT_DATA_TX); if (ret < 0) { dev_err(dai->dev, "Failed to stop DATA_TX: %d\n", ret); - goto release_lock; + return ret; } if (xcvr->soc_data->spdif_only) break; @@ -905,7 +904,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, if (ret < 0) { dev_err(dai->dev, "Err updating ISR %d\n", ret); - goto release_lock; + return ret; } break; } @@ -916,8 +915,6 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd, break; } -release_lock: - spin_unlock_irqrestore(&xcvr->lock, lock_flags); return ret; } @@ -1448,11 +1445,10 @@ static void reset_rx_work(struct work_struct *work) { struct fsl_xcvr *xcvr = container_of(work, struct fsl_xcvr, work_rst); struct device *dev = &xcvr->pdev->dev; - unsigned long lock_flags; u32 ext_ctrl; dev_dbg(dev, "reset rx path\n"); - spin_lock_irqsave(&xcvr->lock, lock_flags); + guard(spinlock_irqsave)(&xcvr->lock); regmap_read(xcvr->regmap, FSL_XCVR_EXT_CTRL, &ext_ctrl); if (!(ext_ctrl & FSL_XCVR_EXT_CTRL_DMA_RD_DIS)) { @@ -1469,7 +1465,6 @@ static void reset_rx_work(struct work_struct *work) FSL_XCVR_EXT_CTRL_RX_DPTH_RESET, 0); } - spin_unlock_irqrestore(&xcvr->lock, lock_flags); } static irqreturn_t irq0_isr(int irq, void *devid) diff --git a/sound/soc/fsl/imx-audio-rpmsg.c b/sound/soc/fsl/imx-audio-rpmsg.c index 38aafb8954c7..b55dfbdb4502 100644 --- a/sound/soc/fsl/imx-audio-rpmsg.c +++ b/sound/soc/fsl/imx-audio-rpmsg.c @@ -22,7 +22,6 @@ static int imx_audio_rpmsg_cb(struct rpmsg_device *rpdev, void *data, int len, struct rpmsg_r_msg *r_msg = (struct rpmsg_r_msg *)data; struct rpmsg_info *info; struct rpmsg_msg *msg; - unsigned long flags; if (!rpmsg->rpmsg_pdev) return 0; @@ -37,21 +36,21 @@ static int imx_audio_rpmsg_cb(struct rpmsg_device *rpdev, void *data, int len, /* TYPE C is notification from M core */ switch (r_msg->header.cmd) { case TX_PERIOD_DONE: - spin_lock_irqsave(&info->lock[TX], flags); - msg = &info->msg[TX_PERIOD_DONE + MSG_TYPE_A_NUM]; - msg->r_msg.param.buffer_tail = - r_msg->param.buffer_tail; - msg->r_msg.param.buffer_tail %= info->num_period[TX]; - spin_unlock_irqrestore(&info->lock[TX], flags); + scoped_guard(spinlock_irqsave, &info->lock[TX]) { + msg = &info->msg[TX_PERIOD_DONE + MSG_TYPE_A_NUM]; + msg->r_msg.param.buffer_tail = + r_msg->param.buffer_tail; + msg->r_msg.param.buffer_tail %= info->num_period[TX]; + } info->callback[TX](info->callback_param[TX]); break; case RX_PERIOD_DONE: - spin_lock_irqsave(&info->lock[RX], flags); - msg = &info->msg[RX_PERIOD_DONE + MSG_TYPE_A_NUM]; - msg->r_msg.param.buffer_tail = - r_msg->param.buffer_tail; - msg->r_msg.param.buffer_tail %= info->num_period[1]; - spin_unlock_irqrestore(&info->lock[RX], flags); + scoped_guard(spinlock_irqsave, &info->lock[RX]) { + msg = &info->msg[RX_PERIOD_DONE + MSG_TYPE_A_NUM]; + msg->r_msg.param.buffer_tail = + r_msg->param.buffer_tail; + msg->r_msg.param.buffer_tail %= info->num_period[1]; + } info->callback[RX](info->callback_param[RX]); break; default: diff --git a/sound/soc/fsl/imx-pcm-rpmsg.c b/sound/soc/fsl/imx-pcm-rpmsg.c index 2a4813c6cda9..ee741f3d79bd 100644 --- a/sound/soc/fsl/imx-pcm-rpmsg.c +++ b/sound/soc/fsl/imx-pcm-rpmsg.c @@ -39,10 +39,9 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg, struct rpmsg_device *rpdev = info->rpdev; int ret = 0; - mutex_lock(&info->msg_lock); + guard(mutex)(&info->msg_lock); if (!rpdev) { dev_err(info->dev, "rpmsg channel not ready\n"); - mutex_unlock(&info->msg_lock); return -EINVAL; } @@ -55,15 +54,12 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg, sizeof(struct rpmsg_s_msg)); if (ret) { dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret); - mutex_unlock(&info->msg_lock); return ret; } /* No receive msg for TYPE_C command */ - if (msg->s_msg.header.type == MSG_TYPE_C) { - mutex_unlock(&info->msg_lock); + if (msg->s_msg.header.type == MSG_TYPE_C) return 0; - } /* wait response from rpmsg */ ret = wait_for_completion_timeout(&info->cmd_complete, @@ -71,7 +67,6 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg, if (!ret) { dev_err(&rpdev->dev, "rpmsg_send cmd %d timeout!\n", msg->s_msg.header.cmd); - mutex_unlock(&info->msg_lock); return -ETIMEDOUT; } @@ -100,8 +95,6 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg, dev_dbg(&rpdev->dev, "cmd:%d, resp %d\n", msg->s_msg.header.cmd, info->r_msg.param.resp); - mutex_unlock(&info->msg_lock); - return 0; } @@ -109,14 +102,13 @@ static int imx_rpmsg_insert_workqueue(struct snd_pcm_substream *substream, struct rpmsg_msg *msg, struct rpmsg_info *info) { - unsigned long flags; int ret = 0; /* * Queue the work to workqueue. * If the queue is full, drop the message. */ - spin_lock_irqsave(&info->wq_lock, flags); + guard(spinlock_irqsave)(&info->wq_lock); if (info->work_write_index != info->work_read_index) { int index = info->work_write_index; @@ -130,7 +122,6 @@ static int imx_rpmsg_insert_workqueue(struct snd_pcm_substream *substream, info->msg_drop_count[substream->stream]++; ret = -EPIPE; } - spin_unlock_irqrestore(&info->wq_lock, flags); return ret; } @@ -523,7 +514,6 @@ static int imx_rpmsg_pcm_ack(struct snd_soc_component *component, snd_pcm_sframes_t avail; struct timer_list *timer; struct rpmsg_msg *msg; - unsigned long flags; int buffer_tail = 0; int written_num; @@ -553,11 +543,11 @@ static int imx_rpmsg_pcm_ack(struct snd_soc_component *component, msg->s_msg.param.buffer_tail = buffer_tail; /* The notification message is updated to latest */ - spin_lock_irqsave(&info->lock[substream->stream], flags); - memcpy(&info->notify[substream->stream], msg, - sizeof(struct rpmsg_s_msg)); - info->notify_updated[substream->stream] = true; - spin_unlock_irqrestore(&info->lock[substream->stream], flags); + scoped_guard(spinlock_irqsave, &info->lock[substream->stream]) { + memcpy(&info->notify[substream->stream], msg, + sizeof(struct rpmsg_s_msg)); + info->notify_updated[substream->stream] = true; + } if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) avail = snd_pcm_playback_hw_avail(runtime); @@ -641,7 +631,7 @@ static void imx_rpmsg_pcm_work(struct work_struct *work) bool is_notification = false; struct rpmsg_info *info; struct rpmsg_msg msg; - unsigned long flags; + bool updated; work_of_rpmsg = container_of(work, struct work_of_rpmsg, work); info = work_of_rpmsg->info; @@ -652,25 +642,26 @@ static void imx_rpmsg_pcm_work(struct work_struct *work) * enough data in M core side, need to let M core know * data is updated immediately. */ - spin_lock_irqsave(&info->lock[TX], flags); - if (info->notify_updated[TX]) { - memcpy(&msg, &info->notify[TX], sizeof(struct rpmsg_s_msg)); - info->notify_updated[TX] = false; - spin_unlock_irqrestore(&info->lock[TX], flags); - info->send_message(&msg, info); - } else { - spin_unlock_irqrestore(&info->lock[TX], flags); + scoped_guard(spinlock_irqsave, &info->lock[TX]) { + updated = info->notify_updated[TX]; + if (updated) { + memcpy(&msg, &info->notify[TX], sizeof(struct rpmsg_s_msg)); + info->notify_updated[TX] = false; + } } - - spin_lock_irqsave(&info->lock[RX], flags); - if (info->notify_updated[RX]) { - memcpy(&msg, &info->notify[RX], sizeof(struct rpmsg_s_msg)); - info->notify_updated[RX] = false; - spin_unlock_irqrestore(&info->lock[RX], flags); + if (updated) info->send_message(&msg, info); - } else { - spin_unlock_irqrestore(&info->lock[RX], flags); + + scoped_guard(spinlock_irqsave, &info->lock[RX]) { + updated = info->notify_updated[RX]; + if (updated) { + memcpy(&msg, &info->notify[RX], sizeof(struct rpmsg_s_msg)); + info->notify_updated[RX] = false; + } } + if (updated) + info->send_message(&msg, info); + /* Skip the notification message for it has been processed above */ if (work_of_rpmsg->msg.s_msg.header.type == MSG_TYPE_C && @@ -682,10 +673,10 @@ static void imx_rpmsg_pcm_work(struct work_struct *work) info->send_message(&work_of_rpmsg->msg, info); /* update read index */ - spin_lock_irqsave(&info->wq_lock, flags); - info->work_read_index++; - info->work_read_index %= WORK_MAX_NUM; - spin_unlock_irqrestore(&info->wq_lock, flags); + scoped_guard(spinlock_irqsave, &info->wq_lock) { + info->work_read_index++; + info->work_read_index %= WORK_MAX_NUM; + } } static int imx_rpmsg_pcm_probe(struct platform_device *pdev) diff --git a/sound/soc/fsl/mpc5200_dma.c b/sound/soc/fsl/mpc5200_dma.c index 56e2cf2f727b..bfedb2dea0b3 100644 --- a/sound/soc/fsl/mpc5200_dma.c +++ b/sound/soc/fsl/mpc5200_dma.c @@ -77,18 +77,20 @@ static irqreturn_t psc_dma_bcom_irq(int irq, void *_psc_dma_stream) { struct psc_dma_stream *s = _psc_dma_stream; - spin_lock(&s->psc_dma->lock); - /* For each finished period, dequeue the completed period buffer - * and enqueue a new one in it's place. */ - while (bcom_buffer_done(s->bcom_task)) { - bcom_retrieve_buffer(s->bcom_task, NULL, NULL); + scoped_guard(spinlock, &s->psc_dma->lock) { + /* + * For each finished period, dequeue the completed period buffer + * and enqueue a new one in its place + */ + while (bcom_buffer_done(s->bcom_task)) { + bcom_retrieve_buffer(s->bcom_task, NULL, NULL); - s->period_current = (s->period_current+1) % s->runtime->periods; - s->period_count++; + s->period_current = (s->period_current+1) % s->runtime->periods; + s->period_count++; - psc_dma_bcom_enqueue_next_buffer(s); + psc_dma_bcom_enqueue_next_buffer(s); + } } - spin_unlock(&s->psc_dma->lock); /* If the stream is active, then also inform the PCM middle layer * of the period finished event. */ @@ -116,7 +118,6 @@ static int psc_dma_trigger(struct snd_soc_component *component, struct psc_dma_stream *s = to_psc_dma_stream(substream, psc_dma); struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs; u16 imr; - unsigned long flags; int i; switch (cmd) { @@ -135,19 +136,18 @@ static int psc_dma_trigger(struct snd_soc_component *component, /* Fill up the bestcomm bd queue and enable DMA. * This will begin filling the PSC's fifo. */ - spin_lock_irqsave(&psc_dma->lock, flags); - - if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE) - bcom_gen_bd_rx_reset(s->bcom_task); - else - bcom_gen_bd_tx_reset(s->bcom_task); + scoped_guard(spinlock_irqsave, &psc_dma->lock) { + if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE) + bcom_gen_bd_rx_reset(s->bcom_task); + else + bcom_gen_bd_tx_reset(s->bcom_task); - for (i = 0; i < runtime->periods; i++) - if (!bcom_queue_full(s->bcom_task)) - psc_dma_bcom_enqueue_next_buffer(s); + for (i = 0; i < runtime->periods; i++) + if (!bcom_queue_full(s->bcom_task)) + psc_dma_bcom_enqueue_next_buffer(s); - bcom_enable(s->bcom_task); - spin_unlock_irqrestore(&psc_dma->lock, flags); + bcom_enable(s->bcom_task); + } out_8(®s->command, MPC52xx_PSC_RST_ERR_STAT); @@ -158,13 +158,13 @@ static int psc_dma_trigger(struct snd_soc_component *component, substream->pstr->stream, s->period_count); s->active = 0; - spin_lock_irqsave(&psc_dma->lock, flags); - bcom_disable(s->bcom_task); - if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE) - bcom_gen_bd_rx_reset(s->bcom_task); - else - bcom_gen_bd_tx_reset(s->bcom_task); - spin_unlock_irqrestore(&psc_dma->lock, flags); + scoped_guard(spinlock_irqsave, &psc_dma->lock) { + bcom_disable(s->bcom_task); + if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE) + bcom_gen_bd_rx_reset(s->bcom_task); + else + bcom_gen_bd_tx_reset(s->bcom_task); + } break; diff --git a/sound/soc/fsl/mpc5200_psc_ac97.c b/sound/soc/fsl/mpc5200_psc_ac97.c index 8554fb690772..2aefd6414ace 100644 --- a/sound/soc/fsl/mpc5200_psc_ac97.c +++ b/sound/soc/fsl/mpc5200_psc_ac97.c @@ -5,7 +5,6 @@ // Copyright (C) 2009 Jon Smirl, Digispeaker // Author: Jon Smirl -#include #include #include #include @@ -31,14 +30,13 @@ static unsigned short psc_ac97_read(struct snd_ac97 *ac97, unsigned short reg) int status; unsigned int val; - mutex_lock(&psc_dma->mutex); + guard(mutex)(&psc_dma->mutex); /* Wait for command send status zero = ready */ status = spin_event_timeout(!(in_be16(&psc_dma->psc_regs->sr_csr.status) & MPC52xx_PSC_SR_CMDSEND), 100, 0); if (status == 0) { pr_err("timeout on ac97 bus (rdy)\n"); - mutex_unlock(&psc_dma->mutex); return -ENODEV; } @@ -54,19 +52,16 @@ static unsigned short psc_ac97_read(struct snd_ac97 *ac97, unsigned short reg) if (status == 0) { pr_err("timeout on ac97 read (val) %x\n", in_be16(&psc_dma->psc_regs->sr_csr.status)); - mutex_unlock(&psc_dma->mutex); return -ENODEV; } /* Get the data */ val = in_be32(&psc_dma->psc_regs->ac97_data); if (((val >> 24) & 0x7f) != reg) { pr_err("reg echo error on ac97 read\n"); - mutex_unlock(&psc_dma->mutex); return -ENODEV; } val = (val >> 8) & 0xffff; - mutex_unlock(&psc_dma->mutex); return (unsigned short) val; } @@ -75,52 +70,46 @@ static void psc_ac97_write(struct snd_ac97 *ac97, { int status; - mutex_lock(&psc_dma->mutex); + guard(mutex)(&psc_dma->mutex); /* Wait for command status zero = ready */ status = spin_event_timeout(!(in_be16(&psc_dma->psc_regs->sr_csr.status) & MPC52xx_PSC_SR_CMDSEND), 100, 0); if (status == 0) { pr_err("timeout on ac97 bus (write)\n"); - goto out; + return; } /* Write data */ out_be32(&psc_dma->psc_regs->ac97_cmd, ((reg & 0x7f) << 24) | (val << 8)); - - out: - mutex_unlock(&psc_dma->mutex); } static void psc_ac97_warm_reset(struct snd_ac97 *ac97) { struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs; - mutex_lock(&psc_dma->mutex); + guard(mutex)(&psc_dma->mutex); out_be32(®s->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_AWR); udelay(3); out_be32(®s->sicr, psc_dma->sicr); - - mutex_unlock(&psc_dma->mutex); } static void psc_ac97_cold_reset(struct snd_ac97 *ac97) { struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs; - mutex_lock(&psc_dma->mutex); - dev_dbg(psc_dma->dev, "cold reset\n"); + scoped_guard(mutex, &psc_dma->mutex) { + dev_dbg(psc_dma->dev, "cold reset\n"); - mpc5200_psc_ac97_gpio_reset(psc_dma->id); + mpc5200_psc_ac97_gpio_reset(psc_dma->id); - /* Notify the PSC that a reset has occurred */ - out_be32(®s->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_ACRB); + /* Notify the PSC that a reset has occurred */ + out_be32(®s->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_ACRB); - /* Re-enable RX and TX */ - out_8(®s->command, MPC52xx_PSC_TX_ENABLE | MPC52xx_PSC_RX_ENABLE); - - mutex_unlock(&psc_dma->mutex); + /* Re-enable RX and TX */ + out_8(®s->command, MPC52xx_PSC_TX_ENABLE | MPC52xx_PSC_RX_ENABLE); + } usleep_range(1000, 2000); psc_ac97_warm_reset(ac97); diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c index 9ad44eeed6ad..7831136f4f12 100644 --- a/sound/soc/fsl/mpc5200_psc_i2s.c +++ b/sound/soc/fsl/mpc5200_psc_i2s.c @@ -170,6 +170,7 @@ static int psc_i2s_of_probe(struct platform_device *op) psc_i2s_dai, ARRAY_SIZE(psc_i2s_dai)); if (rc != 0) { pr_err("Failed to register DAI\n"); + mpc5200_audio_dma_destroy(op); return rc; } diff --git a/sound/soc/generic/audio-graph-card2-custom-sample.c b/sound/soc/generic/audio-graph-card2-custom-sample.c index 7151d426bee9..14b212675240 100644 --- a/sound/soc/generic/audio-graph-card2-custom-sample.c +++ b/sound/soc/generic/audio-graph-card2-custom-sample.c @@ -6,7 +6,6 @@ // Copyright (C) 2020 Kuninori Morimoto // #include -#include #include #include #include diff --git a/sound/soc/generic/audio-graph-card2.c b/sound/soc/generic/audio-graph-card2.c index 6894bb936cfd..0202ed0ee78e 100644 --- a/sound/soc/generic/audio-graph-card2.c +++ b/sound/soc/generic/audio-graph-card2.c @@ -778,18 +778,6 @@ static void graph_link_init(struct simple_util_priv *priv, graph_parse_daifmt(ports_cpu, &daifmt); graph_parse_daifmt(ports_codec, &daifmt); graph_parse_daifmt(lnk, &daifmt); - if (daifmt) { - struct device *dev = simple_priv_to_dev(priv); - - /* - * Recommend to use Auto Select by using .auto_selectable_formats. - * linux/sound/soc/renesas/rcar/core.c can be good sample for it. - * - * One note is that Audio Graph Card2 still keeps compatible to set - * DAI format via DT. - */ - dev_warn_once(dev, "use .auto_selectable_formats on each corresponding CPU/Codec driver"); - } graph_util_parse_link_direction(lnk, &playback_only, &capture_only); graph_util_parse_link_direction(ports_cpu, &playback_only, &capture_only); diff --git a/sound/soc/intel/atom/sst-atom-controls.c b/sound/soc/intel/atom/sst-atom-controls.c index 3629ceaaac17..82df398237da 100644 --- a/sound/soc/intel/atom/sst-atom-controls.c +++ b/sound/soc/intel/atom/sst-atom-controls.c @@ -14,6 +14,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -73,14 +74,10 @@ static int sst_fill_and_send_cmd(struct sst_data *drv, u8 ipc_msg, u8 block, u8 task_id, u8 pipe_id, void *cmd_data, u16 len) { - int ret; + guard(mutex)(&drv->lock); - mutex_lock(&drv->lock); - ret = sst_fill_and_send_cmd_unlocked(drv, ipc_msg, block, - task_id, pipe_id, cmd_data, len); - mutex_unlock(&drv->lock); - - return ret; + return sst_fill_and_send_cmd_unlocked(drv, ipc_msg, block, + task_id, pipe_id, cmd_data, len); } /* @@ -167,7 +164,7 @@ static int sst_slot_get(struct snd_kcontrol *kcontrol, unsigned int val, mux; u8 *map = is_tx ? sst_ssp_rx_map : sst_ssp_tx_map; - mutex_lock(&drv->lock); + guard(mutex)(&drv->lock); val = 1 << ctl_no; /* search which slot/channel has this bit set - there should be only one */ for (mux = e->max; mux > 0; mux--) @@ -175,7 +172,6 @@ static int sst_slot_get(struct snd_kcontrol *kcontrol, break; ucontrol->value.enumerated.item[0] = mux; - mutex_unlock(&drv->lock); dev_dbg(c->dev, "%s - %s map = %#x\n", is_tx ? "tx channel" : "rx slot", @@ -235,7 +231,7 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol, if (mux > e->max - 1) return -EINVAL; - mutex_lock(&drv->lock); + guard(mutex)(&drv->lock); /* first clear all registers of this bit */ for (i = 0; i < e->max; i++) map[i] &= ~val; @@ -244,7 +240,6 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol, /* kctl set to 'none' and we reset the bits so send IPC */ ret = sst_check_and_send_slot_map(drv, kcontrol); - mutex_unlock(&drv->lock); return ret; } @@ -258,7 +253,6 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol, ret = sst_check_and_send_slot_map(drv, kcontrol); - mutex_unlock(&drv->lock); return ret; } @@ -354,13 +348,12 @@ static int sst_algo_control_set(struct snd_kcontrol *kcontrol, struct sst_algo_control *bc = (void *)kcontrol->private_value; dev_dbg(cmpnt->dev, "control_name=%s\n", kcontrol->id.name); - mutex_lock(&drv->lock); + guard(mutex)(&drv->lock); switch (bc->type) { case SST_ALGO_PARAMS: memcpy(bc->params, ucontrol->value.bytes.data, bc->max); break; default: - mutex_unlock(&drv->lock); dev_err(cmpnt->dev, "Invalid Input- algo type:%d\n", bc->type); return -EINVAL; @@ -368,7 +361,6 @@ static int sst_algo_control_set(struct snd_kcontrol *kcontrol, /*if pipe is enabled, need to send the algo params from here*/ if (bc->w && bc->w->power) ret = sst_send_algo_cmd(drv, bc); - mutex_unlock(&drv->lock); return ret; } @@ -475,7 +467,7 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol, struct sst_gain_mixer_control *mc = (void *)kcontrol->private_value; struct sst_gain_value *gv = mc->gain_val; - mutex_lock(&drv->lock); + guard(mutex)(&drv->lock); switch (mc->type) { case SST_GAIN_TLV: @@ -497,7 +489,6 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol, break; default: - mutex_unlock(&drv->lock); dev_err(cmpnt->dev, "Invalid Input- gain type:%d\n", mc->type); return -EINVAL; @@ -506,7 +497,6 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol, if (mc->w && mc->w->power) ret = sst_send_gain_cmd(drv, gv, mc->task_id, mc->pipe_id | mc->instance_id, mc->module_id, 0); - mutex_unlock(&drv->lock); return ret; } @@ -521,10 +511,9 @@ static int sst_send_pipe_module_params(struct snd_soc_dapm_widget *w, struct sst_data *drv = snd_soc_component_get_drvdata(c); struct sst_ids *ids = w->priv; - mutex_lock(&drv->lock); + guard(mutex)(&drv->lock); sst_find_and_send_pipe_algo(drv, w->name, ids); sst_set_pipe_gain(ids, drv, 0); - mutex_unlock(&drv->lock); return 0; } @@ -761,27 +750,29 @@ int sst_handle_vb_timer(struct snd_soc_dai *dai, bool enable) return ret; } - mutex_lock(&drv->lock); - if (enable) - timer_usage++; - else - timer_usage--; - - /* - * Send the command only if this call is the first enable or last - * disable - */ - if ((enable && (timer_usage == 1)) || - (!enable && (timer_usage == 0))) { - ret = sst_fill_and_send_cmd_unlocked(drv, SST_IPC_IA_CMD, - SST_FLAG_BLOCKED, SST_TASK_SBA, 0, &cmd, - sizeof(cmd.header) + cmd.header.length); - if (ret && enable) { + scoped_guard(mutex, &drv->lock) { + if (enable) + timer_usage++; + else timer_usage--; - enable = false; + + /* + * Send the command only if this call is the first enable or last + * disable + */ + if ((enable && timer_usage == 1) || + (!enable && timer_usage == 0)) { + ret = sst_fill_and_send_cmd_unlocked(drv, SST_IPC_IA_CMD, + SST_FLAG_BLOCKED, + SST_TASK_SBA, 0, &cmd, + sizeof(cmd.header) + + cmd.header.length); + if (ret && enable) { + timer_usage--; + enable = false; + } } } - mutex_unlock(&drv->lock); if (!enable) sst->ops->power(sst->dev, false); diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c index f074af2499c8..9ee4d9926e06 100644 --- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c +++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c @@ -11,6 +11,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -32,7 +33,7 @@ int sst_register_dsp(struct sst_device *dev) return -EINVAL; if (!try_module_get(dev->dev->driver->owner)) return -ENODEV; - mutex_lock(&sst_lock); + guard(mutex)(&sst_lock); if (sst) { dev_err(dev->dev, "we already have a device %s\n", sst->name); module_put(dev->dev->driver->owner); @@ -41,7 +42,7 @@ int sst_register_dsp(struct sst_device *dev) } dev_dbg(dev->dev, "registering device %s\n", dev->name); sst = dev; - mutex_unlock(&sst_lock); + return 0; } EXPORT_SYMBOL_GPL(sst_register_dsp); @@ -53,17 +54,15 @@ int sst_unregister_dsp(struct sst_device *dev) if (dev != sst) return -EINVAL; - mutex_lock(&sst_lock); + guard(mutex)(&sst_lock); - if (!sst) { - mutex_unlock(&sst_lock); + if (!sst) return -EIO; - } module_put(sst->dev->driver->owner); dev_dbg(dev->dev, "unreg %s\n", sst->name); sst = NULL; - mutex_unlock(&sst_lock); + return 0; } EXPORT_SYMBOL_GPL(sst_unregister_dsp); @@ -103,21 +102,14 @@ static int sst_media_digital_mute(struct snd_soc_dai *dai, int mute, int stream) void sst_set_stream_status(struct sst_runtime_stream *stream, int state) { - unsigned long flags; - spin_lock_irqsave(&stream->status_lock, flags); + guard(spinlock_irqsave)(&stream->status_lock); stream->stream_status = state; - spin_unlock_irqrestore(&stream->status_lock, flags); } static inline int sst_get_stream_status(struct sst_runtime_stream *stream) { - int state; - unsigned long flags; - - spin_lock_irqsave(&stream->status_lock, flags); - state = stream->stream_status; - spin_unlock_irqrestore(&stream->status_lock, flags); - return state; + guard(spinlock_irqsave)(&stream->status_lock); + return stream->stream_status; } static void sst_fill_alloc_params(struct snd_pcm_substream *substream, @@ -304,7 +296,7 @@ static int sst_media_open(struct snd_pcm_substream *substream, { int ret_val = 0; struct snd_pcm_runtime *runtime = substream->runtime; - struct sst_runtime_stream *stream; + struct sst_runtime_stream *stream __free(kfree) = NULL; stream = kzalloc_obj(*stream); if (!stream) @@ -312,15 +304,14 @@ static int sst_media_open(struct snd_pcm_substream *substream, spin_lock_init(&stream->status_lock); /* get the sst ops */ - mutex_lock(&sst_lock); - if (!sst || - !try_module_get(sst->dev->driver->owner)) { - dev_err(dai->dev, "no device available to run\n"); - ret_val = -ENODEV; - goto out_ops; + scoped_guard(mutex, &sst_lock) { + if (!sst || + !try_module_get(sst->dev->driver->owner)) { + dev_err(dai->dev, "no device available to run\n"); + return -ENODEV; + } + stream->ops = sst->ops; } - stream->ops = sst->ops; - mutex_unlock(&sst_lock); stream->stream_info.str_id = 0; @@ -330,7 +321,7 @@ static int sst_media_open(struct snd_pcm_substream *substream, ret_val = power_up_sst(stream); if (ret_val < 0) - goto out_power_up; + return ret_val; /* * Make sure the period to be multiple of 1ms to align the @@ -347,12 +338,14 @@ static int sst_media_open(struct snd_pcm_substream *substream, snd_pcm_hw_constraint_step(substream->runtime, 0, SNDRV_PCM_HW_PARAM_PERIODS, 2); - return snd_pcm_hw_constraint_integer(runtime, - SNDRV_PCM_HW_PARAM_PERIODS); -out_ops: - mutex_unlock(&sst_lock); -out_power_up: - kfree(stream); + ret_val = snd_pcm_hw_constraint_integer(runtime, + SNDRV_PCM_HW_PARAM_PERIODS); + + if (ret_val < 0) + return ret_val; + + stream = NULL; + return ret_val; } diff --git a/sound/soc/intel/atom/sst/sst_ipc.c b/sound/soc/intel/atom/sst/sst_ipc.c index 0d5e71e8a5b5..6c19ab63aa4f 100644 --- a/sound/soc/intel/atom/sst/sst_ipc.c +++ b/sound/soc/intel/atom/sst/sst_ipc.c @@ -11,6 +11,7 @@ * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#include #include #include #include @@ -180,9 +181,8 @@ void intel_sst_clear_intr_mrfld(struct intel_sst_drv *sst_drv_ctx) union interrupt_reg_mrfld isr; union interrupt_reg_mrfld imr; union ipc_header_mrfld clear_ipc; - unsigned long irq_flags; - spin_lock_irqsave(&sst_drv_ctx->ipc_spin_lock, irq_flags); + guard(spinlock_irqsave)(&sst_drv_ctx->ipc_spin_lock); imr.full = sst_shim_read64(sst_drv_ctx->shim, SST_IMRX); isr.full = sst_shim_read64(sst_drv_ctx->shim, SST_ISRX); @@ -200,7 +200,6 @@ void intel_sst_clear_intr_mrfld(struct intel_sst_drv *sst_drv_ctx) /* un mask busy interrupt */ imr.part.busy_interrupt = 0; sst_shim_write64(sst_drv_ctx->shim, SST_IMRX, imr.full); - spin_unlock_irqrestore(&sst_drv_ctx->ipc_spin_lock, irq_flags); } diff --git a/sound/soc/intel/atom/sst/sst_pci.c b/sound/soc/intel/atom/sst/sst_pci.c index 22ae2d22f121..44bb11c69490 100644 --- a/sound/soc/intel/atom/sst/sst_pci.c +++ b/sound/soc/intel/atom/sst/sst_pci.c @@ -130,13 +130,15 @@ static int intel_sst_probe(struct pci_dev *pci, sst_drv_ctx->pci = pci_dev_get(pci); ret = sst_platform_get_resources(sst_drv_ctx); if (ret < 0) - goto do_free_drv_ctx; + goto do_put_pci; pci_set_drvdata(pci, sst_drv_ctx); sst_configure_runtime_pm(sst_drv_ctx); return ret; +do_put_pci: + pci_dev_put(sst_drv_ctx->pci); do_free_drv_ctx: sst_context_cleanup(sst_drv_ctx); dev_err(sst_drv_ctx->dev, "Probe failed with %d\n", ret); diff --git a/sound/soc/intel/atom/sst/sst_pvt.c b/sound/soc/intel/atom/sst/sst_pvt.c index 67b1ab14239f..0b0cfd70efbc 100644 --- a/sound/soc/intel/atom/sst/sst_pvt.c +++ b/sound/soc/intel/atom/sst/sst_pvt.c @@ -11,6 +11,7 @@ * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#include #include #include #include @@ -64,9 +65,8 @@ u64 sst_shim_read64(void __iomem *addr, int offset) void sst_set_fw_state_locked( struct intel_sst_drv *sst_drv_ctx, int sst_state) { - mutex_lock(&sst_drv_ctx->sst_lock); + guard(mutex)(&sst_drv_ctx->sst_lock); sst_drv_ctx->sst_state = sst_state; - mutex_unlock(&sst_drv_ctx->sst_lock); } /* @@ -302,18 +302,17 @@ int sst_assign_pvt_id(struct intel_sst_drv *drv) { int local; - spin_lock(&drv->block_lock); + guard(spinlock)(&drv->block_lock); /* find first zero index from lsb */ local = ffz(drv->pvt_id); dev_dbg(drv->dev, "pvt_id assigned --> %d\n", local); if (local >= SST_MAX_BLOCKS){ - spin_unlock(&drv->block_lock); dev_err(drv->dev, "PVT _ID error: no free id blocks "); return -EINVAL; } /* toggle the index */ change_bit(local, &drv->pvt_id); - spin_unlock(&drv->block_lock); + return local; } diff --git a/sound/soc/intel/avs/apl.c b/sound/soc/intel/avs/apl.c index b922eeaba843..cf600e1c986e 100644 --- a/sound/soc/intel/avs/apl.c +++ b/sound/soc/intel/avs/apl.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -190,7 +191,7 @@ static bool avs_apl_lp_streaming(struct avs_dev *adev) { struct avs_path *path; - spin_lock(&adev->path_list_lock); + guard(spinlock)(&adev->path_list_lock); /* Any gateway without buffer allocated in LP area disqualifies D0IX. */ list_for_each_entry(path, &adev->path_list, node) { struct avs_path_pipeline *ppl; @@ -210,14 +211,11 @@ static bool avs_apl_lp_streaming(struct avs_dev *adev) if (cfg->copier.dma_type == INVALID_OBJECT_ID) continue; - if (!mod->gtw_attrs.lp_buffer_alloc) { - spin_unlock(&adev->path_list_lock); + if (!mod->gtw_attrs.lp_buffer_alloc) return false; - } } } } - spin_unlock(&adev->path_list_lock); return true; } diff --git a/sound/soc/intel/avs/control.c b/sound/soc/intel/avs/control.c index a8f05de338e0..370069247a7d 100644 --- a/sound/soc/intel/avs/control.c +++ b/sound/soc/intel/avs/control.c @@ -27,7 +27,7 @@ static struct avs_path_module *avs_get_volume_module(struct avs_dev *adev, u32 i struct avs_path_pipeline *ppl; struct avs_path_module *mod; - spin_lock(&adev->path_list_lock); + guard(spinlock)(&adev->path_list_lock); list_for_each_entry(path, &adev->path_list, node) { list_for_each_entry(ppl, &path->ppl_list, node) { list_for_each_entry(mod, &ppl->mod_list, node) { @@ -35,14 +35,11 @@ static struct avs_path_module *avs_get_volume_module(struct avs_dev *adev, u32 i if ((guid_equal(type, &AVS_PEAKVOL_MOD_UUID) || guid_equal(type, &AVS_GAIN_MOD_UUID)) && - mod->template->ctl_id == id) { - spin_unlock(&adev->path_list_lock); + mod->template->ctl_id == id) return mod; - } } } } - spin_unlock(&adev->path_list_lock); return NULL; } diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index 1a53856c2ffb..2afe59646896 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -14,6 +14,7 @@ // foundation of this driver // +#include #include #include #include @@ -273,7 +274,7 @@ static irqreturn_t avs_hda_interrupt(struct hdac_bus *bus) if (snd_hdac_bus_handle_stream_irq(bus, status, hdac_update_stream)) ret = IRQ_HANDLED; - spin_lock_irq(&bus->reg_lock); + guard(spinlock_irq)(&bus->reg_lock); /* Clear RIRB interrupt. */ status = snd_hdac_chip_readb(bus, RIRBSTS); if (status & RIRB_INT_MASK) { @@ -283,7 +284,6 @@ static irqreturn_t avs_hda_interrupt(struct hdac_bus *bus) ret = IRQ_HANDLED; } - spin_unlock_irq(&bus->reg_lock); return ret; } diff --git a/sound/soc/intel/avs/debug.h b/sound/soc/intel/avs/debug.h index 94fe8729a5c1..c47fc4e8b02b 100644 --- a/sound/soc/intel/avs/debug.h +++ b/sound/soc/intel/avs/debug.h @@ -9,6 +9,7 @@ #ifndef __SOUND_SOC_INTEL_AVS_DEBUG_H #define __SOUND_SOC_INTEL_AVS_DEBUG_H +#include #include "messages.h" #include "registers.h" @@ -26,14 +27,9 @@ struct avs_dev; static inline int avs_log_buffer_status_locked(struct avs_dev *adev, union avs_notify_msg *msg) { - unsigned long flags; - int ret; + guard(spinlock_irqsave)(&adev->trace_lock); - spin_lock_irqsave(&adev->trace_lock, flags); - ret = avs_dsp_op(adev, log_buffer_status, msg); - spin_unlock_irqrestore(&adev->trace_lock, flags); - - return ret; + return avs_dsp_op(adev, log_buffer_status, msg); } struct avs_apl_log_buffer_layout { diff --git a/sound/soc/intel/avs/debugfs.c b/sound/soc/intel/avs/debugfs.c index 701c247227bf..9ab503da3b75 100644 --- a/sound/soc/intel/avs/debugfs.c +++ b/sound/soc/intel/avs/debugfs.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -251,24 +252,22 @@ static int strace_release(struct inode *inode, struct file *file) union avs_notify_msg msg = AVS_NOTIFICATION(LOG_BUFFER_STATUS); struct avs_dev *adev = file->private_data; unsigned long resource_mask; - unsigned long flags, i; + unsigned long i; u32 num_cores; resource_mask = adev->logged_resources; num_cores = adev->hw_cfg.dsp_cores; - spin_lock_irqsave(&adev->trace_lock, flags); + scoped_guard(spinlock_irqsave, &adev->trace_lock) { + /* Gather any remaining logs. */ + for_each_set_bit(i, &resource_mask, num_cores) { + msg.log.core = i; + avs_dsp_op(adev, log_buffer_status, &msg); + } - /* Gather any remaining logs. */ - for_each_set_bit(i, &resource_mask, num_cores) { - msg.log.core = i; - avs_dsp_op(adev, log_buffer_status, &msg); + kfifo_free(&adev->trace_fifo); } - kfifo_free(&adev->trace_fifo); - - spin_unlock_irqrestore(&adev->trace_lock, flags); - module_put(adev->dev->driver->owner); return 0; } diff --git a/sound/soc/intel/avs/ipc.c b/sound/soc/intel/avs/ipc.c index c0feb9edd7f6..71e7997e52c2 100644 --- a/sound/soc/intel/avs/ipc.c +++ b/sound/soc/intel/avs/ipc.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -99,39 +100,39 @@ static void avs_dsp_recovery(struct avs_dev *adev) unsigned int core_mask; int ret; - mutex_lock(&adev->comp_list_mutex); - /* disconnect all running streams */ - list_for_each_entry(acomp, &adev->comp_list, node) { - struct snd_soc_pcm_runtime *rtd; - struct snd_soc_card *card; + scoped_guard(mutex, &adev->comp_list_mutex) { + /* disconnect all running streams */ + list_for_each_entry(acomp, &adev->comp_list, node) { + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_card *card; - card = acomp->base.card; - if (!card) - continue; - - for_each_card_rtds(card, rtd) { - struct snd_pcm *pcm; - int dir; - - pcm = rtd->pcm; - if (!pcm || rtd->dai_link->no_pcm) + card = acomp->base.card; + if (!card) continue; - for_each_pcm_streams(dir) { - struct snd_pcm_substream *substream; + for_each_card_rtds(card, rtd) { + struct snd_pcm *pcm; + int dir; - substream = pcm->streams[dir].substream; - if (!substream || !substream->runtime) + pcm = rtd->pcm; + if (!pcm || rtd->dai_link->no_pcm) continue; - /* No need for _irq() as we are in nonatomic context. */ - snd_pcm_stream_lock(substream); - snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); - snd_pcm_stream_unlock(substream); + for_each_pcm_streams(dir) { + struct snd_pcm_substream *substream; + + substream = pcm->streams[dir].substream; + if (!substream || !substream->runtime) + continue; + + /* No need for _irq() as we are in nonatomic context. */ + snd_pcm_stream_lock(substream); + snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); + snd_pcm_stream_unlock(substream); + } } } } - mutex_unlock(&adev->comp_list_mutex); /* forcibly shutdown all cores */ core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0); @@ -397,7 +398,7 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request if (!ipc->ready) return -EPERM; - mutex_lock(&ipc->msg_mutex); + guard(mutex)(&ipc->msg_mutex); spin_lock(&ipc->rx_lock); avs_ipc_msg_init(ipc, reply); @@ -412,7 +413,7 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request /* Same treatment as on exception, just stack_dump=0. */ avs_dsp_exception_caught(adev, &msg); } - goto exit; + return ret; } ret = ipc->rx.rsp.status; @@ -436,8 +437,6 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request memcpy(reply->data, ipc->rx.data, reply->size); } -exit: - mutex_unlock(&ipc->msg_mutex); return ret; } @@ -501,7 +500,7 @@ static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *req struct avs_ipc *ipc = adev->ipc; int ret; - mutex_lock(&ipc->msg_mutex); + guard(mutex)(&ipc->msg_mutex); spin_lock(&ipc->rx_lock); avs_ipc_msg_init(ipc, NULL); @@ -522,8 +521,6 @@ static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *req dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n", name, request->glb.primary, request->glb.ext.val, ret); - mutex_unlock(&ipc->msg_mutex); - return ret; } diff --git a/sound/soc/intel/avs/loader.c b/sound/soc/intel/avs/loader.c index 353e343b1d28..bebdc79ec88e 100644 --- a/sound/soc/intel/avs/loader.c +++ b/sound/soc/intel/avs/loader.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -630,15 +631,15 @@ static int avs_load_firmware(struct avs_dev *adev, bool purge) if (ret) goto reenable_gating; - mutex_lock(&adev->comp_list_mutex); - list_for_each_entry(acomp, &adev->comp_list, node) { - struct avs_tplg *tplg = acomp->tplg; + scoped_guard(mutex, &adev->comp_list_mutex) { + list_for_each_entry(acomp, &adev->comp_list, node) { + struct avs_tplg *tplg = acomp->tplg; - ret = avs_dsp_load_libraries(adev, tplg->libs, tplg->num_libs); - if (ret < 0) - break; + ret = avs_dsp_load_libraries(adev, tplg->libs, tplg->num_libs); + if (ret < 0) + break; + } } - mutex_unlock(&adev->comp_list_mutex); reenable_gating: avs_hda_l1sen_enable(adev, true); diff --git a/sound/soc/intel/avs/path.c b/sound/soc/intel/avs/path.c index 2291f9728a54..213d6ecdd7cc 100644 --- a/sound/soc/intel/avs/path.c +++ b/sound/soc/intel/avs/path.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -69,16 +70,13 @@ avs_path_find_path(struct avs_dev *adev, const char *name, u32 template_id) if (!template) return NULL; - spin_lock(&adev->path_list_lock); + guard(spinlock)(&adev->path_list_lock); /* Only one variant of given path template may be instantiated at a time. */ list_for_each_entry(path, &adev->path_list, node) { - if (path->template->owner == template) { - spin_unlock(&adev->path_list_lock); + if (path->template->owner == template) return path; - } } - spin_unlock(&adev->path_list_lock); return NULL; } @@ -1305,7 +1303,7 @@ void avs_path_free(struct avs_path *path) struct avs_path *cpath, *csave; struct avs_dev *adev = path->owner; - mutex_lock(&adev->path_mutex); + guard(mutex)(&adev->path_mutex); /* Free all condpaths this path spawned. */ list_for_each_entry_safe(cpath, csave, &path->source_list, source_node) @@ -1314,8 +1312,6 @@ void avs_path_free(struct avs_path *path) avs_condpath_free(path->owner, cpath); avs_path_free_unlocked(path); - - mutex_unlock(&adev->path_mutex); } struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id, @@ -1334,13 +1330,13 @@ struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id, } /* Serialize path and its components creation. */ - mutex_lock(&adev->path_mutex); + guard(mutex)(&adev->path_mutex); /* Satisfy needs of avs_path_find_tplg(). */ - mutex_lock(&adev->comp_list_mutex); + guard(mutex)(&adev->comp_list_mutex); path = avs_path_create_unlocked(adev, dma_id, variant); if (IS_ERR(path)) - goto exit; + return path; ret = avs_condpaths_walk_all(adev, path); if (ret) { @@ -1348,10 +1344,6 @@ struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id, path = ERR_PTR(ret); } -exit: - mutex_unlock(&adev->comp_list_mutex); - mutex_unlock(&adev->path_mutex); - return path; } @@ -1496,15 +1488,13 @@ static void avs_condpaths_pause(struct avs_dev *adev, struct avs_path *path) { struct avs_path *cpath; - mutex_lock(&adev->path_mutex); + guard(mutex)(&adev->path_mutex); /* If either source or sink stops, so do the attached conditional paths. */ list_for_each_entry(cpath, &path->source_list, source_node) avs_condpath_pause(adev, cpath); list_for_each_entry(cpath, &path->sink_list, sink_node) avs_condpath_pause(adev, cpath); - - mutex_unlock(&adev->path_mutex); } int avs_path_pause(struct avs_path *path) @@ -1560,7 +1550,7 @@ static void avs_condpaths_run(struct avs_dev *adev, struct avs_path *path, int t { struct avs_path *cpath; - mutex_lock(&adev->path_mutex); + guard(mutex)(&adev->path_mutex); /* Run conditional paths only if source and sink are both running. */ list_for_each_entry(cpath, &path->source_list, source_node) @@ -1572,8 +1562,6 @@ static void avs_condpaths_run(struct avs_dev *adev, struct avs_path *path, int t if (cpath->source->state == AVS_PPL_STATE_RUNNING && cpath->sink->state == AVS_PPL_STATE_RUNNING) avs_condpath_run(adev, cpath, trigger); - - mutex_unlock(&adev->path_mutex); } int avs_path_run(struct avs_path *path, int trigger) diff --git a/sound/soc/intel/avs/utils.c b/sound/soc/intel/avs/utils.c index ee36725ac731..ea14ec173855 100644 --- a/sound/soc/intel/avs/utils.c +++ b/sound/soc/intel/avs/utils.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -48,13 +49,12 @@ int avs_get_module_entry(struct avs_dev *adev, const guid_t *uuid, struct avs_mo { int idx; - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); idx = avs_module_entry_index(adev, uuid); if (idx >= 0) memcpy(entry, &adev->mods_info->entries[idx], sizeof(*entry)); - mutex_unlock(&adev->modres_mutex); return (idx < 0) ? idx : 0; } @@ -62,13 +62,12 @@ int avs_get_module_id_entry(struct avs_dev *adev, u32 module_id, struct avs_modu { int idx; - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); idx = avs_module_id_entry_index(adev, module_id); if (idx >= 0) memcpy(entry, &adev->mods_info->entries[idx], sizeof(*entry)); - mutex_unlock(&adev->modres_mutex); return (idx < 0) ? idx : 0; } @@ -86,13 +85,12 @@ bool avs_is_module_ida_empty(struct avs_dev *adev, u32 module_id) bool ret = false; int idx; - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); idx = avs_module_id_entry_index(adev, module_id); if (idx >= 0) ret = ida_is_empty(adev->mod_idas[idx]); - mutex_unlock(&adev->modres_mutex); return ret; } @@ -163,68 +161,57 @@ int avs_module_info_init(struct avs_dev *adev, bool purge) if (ret) return AVS_IPC_RET(ret); - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); ret = avs_module_ida_alloc(adev, info, purge); if (ret < 0) { dev_err(adev->dev, "initialize module idas failed: %d\n", ret); - goto exit; + return ret; } /* Refresh current information with newly received table. */ kfree(adev->mods_info); adev->mods_info = info; -exit: - mutex_unlock(&adev->modres_mutex); return ret; } void avs_module_info_free(struct avs_dev *adev) { - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); avs_module_ida_destroy(adev); kfree(adev->mods_info); adev->mods_info = NULL; - - mutex_unlock(&adev->modres_mutex); } int avs_module_id_alloc(struct avs_dev *adev, u16 module_id) { - int ret, idx, max_id; + int idx, max_id; - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); idx = avs_module_id_entry_index(adev, module_id); if (idx == -ENOENT) { dev_err(adev->dev, "invalid module id: %d", module_id); - ret = -EINVAL; - goto exit; + return -EINVAL; } max_id = adev->mods_info->entries[idx].instance_max_count - 1; - ret = ida_alloc_max(adev->mod_idas[idx], max_id, GFP_KERNEL); -exit: - mutex_unlock(&adev->modres_mutex); - return ret; + + return ida_alloc_max(adev->mod_idas[idx], max_id, GFP_KERNEL); } void avs_module_id_free(struct avs_dev *adev, u16 module_id, u8 instance_id) { int idx; - mutex_lock(&adev->modres_mutex); + guard(mutex)(&adev->modres_mutex); idx = avs_module_id_entry_index(adev, module_id); - if (idx == -ENOENT) { + if (idx == -ENOENT) dev_err(adev->dev, "invalid module id: %d", module_id); - goto exit; - } - - ida_free(adev->mod_idas[idx], instance_id); -exit: - mutex_unlock(&adev->modres_mutex); + else + ida_free(adev->mod_idas[idx], instance_id); } /* diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index d43daf9b025d..24226a387cc2 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -1285,7 +1285,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card) goto err_dai; } - ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_confs); + ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_confs); if (ret < 0) goto err_end; diff --git a/sound/soc/jz4740/jz4740-i2s.c b/sound/soc/jz4740/jz4740-i2s.c index 517619531615..d36bbc820618 100644 --- a/sound/soc/jz4740/jz4740-i2s.c +++ b/sound/soc/jz4740/jz4740-i2s.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/loongson/loongson_card.c b/sound/soc/loongson/loongson_card.c index 7910d5d9ac4f..25cd12eab4b1 100644 --- a/sound/soc/loongson/loongson_card.c +++ b/sound/soc/loongson/loongson_card.c @@ -2,24 +2,131 @@ // // Loongson ASoC Audio Machine driver // -// Copyright (C) 2023 Loongson Technology Corporation Limited +// Copyright (C) 2023-2026 Loongson Technology Corporation Limited // Author: Yingkun Meng +// Binbin Zhou // -#include -#include -#include #include +#include +#include #include +#include #include +#include +#include static char codec_name[SND_ACPI_I2C_ID_LEN]; struct loongson_card_data { struct snd_soc_card snd_card; unsigned int mclk_fs; + struct gpio_desc *gpiod_hp_det; + struct gpio_desc *gpiod_hp_ctl; + struct gpio_desc *gpiod_spkr_en; + const struct loongson_card_config *cfg; +}; + +struct loongson_card_config { + unsigned int fmt; + bool add_hp_jack; + bool add_dapm_widgets; + bool add_dapm_routes; +}; + +static const struct loongson_card_config ls2k1000_card_config = { + .fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_IB_NF | SND_SOC_DAIFMT_CBC_CFC, + .add_hp_jack = false, + .add_dapm_widgets = false, + .add_dapm_routes = false, +}; + +static const struct loongson_card_config ls2k0300_forever_pi_card_config = { + .fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBC_CFC, + .add_hp_jack = false, + .add_dapm_widgets = false, + .add_dapm_routes = false, +}; + +static const struct loongson_card_config ls2k0300_dl2k0300b_card_config = { + .fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBC_CFC, + .add_hp_jack = true, + .add_dapm_widgets = true, + .add_dapm_routes = true, +}; + +static int tegra_machine_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *k, int event) +{ + struct snd_soc_card *card = snd_soc_dapm_to_card(w->dapm); + struct loongson_card_data *priv = snd_soc_card_get_drvdata(card); + + if (!snd_soc_dapm_widget_name_cmp(w, "Speaker")) + gpiod_set_value_cansleep(priv->gpiod_spkr_en, + SND_SOC_DAPM_EVENT_ON(event)); + + if (!snd_soc_dapm_widget_name_cmp(w, "Headphone")) + gpiod_set_value_cansleep(priv->gpiod_hp_ctl, + SND_SOC_DAPM_EVENT_ON(event)); + + return 0; +} + +static const struct snd_soc_dapm_widget loongson_aosc_dapm_widgets[] = { + SND_SOC_DAPM_HP("Headphone", tegra_machine_event), + SND_SOC_DAPM_SPK("Speaker", tegra_machine_event), }; +/* Headphones Jack */ + +static struct snd_soc_jack loongson_asoc_hp_jack; + +static struct snd_soc_jack_pin loongson_asoc_hp_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE + }, + { + .pin = "Speaker", + .mask = SND_JACK_HEADPHONE, + .invert = 1 + }, +}; + +static struct snd_soc_jack_gpio loongson_asoc_hp_jack_gpio = { + .name = "Headphones detection", + .report = SND_JACK_HEADPHONE, + .debounce_time = 150, +}; + +static int loongson_asoc_machine_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct loongson_card_data *ls_priv = snd_soc_card_get_drvdata(card); + int ret = 0; + + if (!ls_priv->cfg->add_hp_jack || !ls_priv->gpiod_hp_det) + return 0; + + ret = snd_soc_card_jack_new_pins(card, "Headphones Jack", + SND_JACK_HEADPHONE, + &loongson_asoc_hp_jack, + loongson_asoc_hp_jack_pins, + ARRAY_SIZE(loongson_asoc_hp_jack_pins)); + if (ret) { + dev_err(rtd->dev, "Headphones Jack creation failed: %d\n", ret); + return ret; + } + + loongson_asoc_hp_jack_gpio.desc = ls_priv->gpiod_hp_det; + + ret = snd_soc_jack_add_gpios(&loongson_asoc_hp_jack, 1, &loongson_asoc_hp_jack_gpio); + if (ret) + dev_err(rtd->dev, "Headphone GPIO not added: %d\n", ret); + + return ret; +} + static int loongson_card_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { @@ -45,7 +152,7 @@ static int loongson_card_hw_params(struct snd_pcm_substream *substream, return ret; } - return 0; + return snd_soc_runtime_set_dai_fmt(rtd, ls_card->cfg->fmt); } static const struct snd_soc_ops loongson_ops = { @@ -61,8 +168,7 @@ static struct snd_soc_dai_link loongson_dai_links[] = { { .name = "Loongson Audio Port", .stream_name = "Loongson Audio", - .dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_IB_NF - | SND_SOC_DAIFMT_CBC_CFC, + .init = loongson_asoc_machine_init, SND_SOC_DAILINK_REG(analog), .ops = &loongson_ops, }, @@ -91,7 +197,7 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) const char *codec_dai_name; struct acpi_device *adev; struct device *phy_dev; - int i; + int i, ret; /* fixup platform name based on reference node */ adev = loongson_card_acpi_find_device(card, "cpu"); @@ -108,7 +214,9 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) return -ENOENT; snprintf(codec_name, sizeof(codec_name), "i2c-%s", acpi_dev_name(adev)); - device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + ret = device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + if (ret) + return ret; for (i = 0; i < card->num_links; i++) { loongson_dai_links[i].platforms->name = dev_name(phy_dev); @@ -121,16 +229,35 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) static int loongson_card_parse_of(struct loongson_card_data *data) { - struct device_node *cpu, *codec; struct snd_soc_card *card = &data->snd_card; + struct device_node *cpu, *codec; struct device *dev = card->dev; int ret, i; + data->gpiod_hp_det = devm_gpiod_get_optional(dev, "hp-det", GPIOD_IN); + if (IS_ERR(data->gpiod_hp_det)) + return PTR_ERR(data->gpiod_hp_det); + + data->gpiod_hp_ctl = devm_gpiod_get_optional(dev, "hp-ctl", GPIOD_OUT_LOW); + if (IS_ERR(data->gpiod_hp_ctl)) + return PTR_ERR(data->gpiod_hp_ctl); + + data->gpiod_spkr_en = devm_gpiod_get_optional(dev, "spkr-en", GPIOD_OUT_LOW); + if (IS_ERR(data->gpiod_spkr_en)) + return PTR_ERR(data->gpiod_spkr_en); + + if (data->cfg->add_dapm_routes) { + ret = snd_soc_of_parse_audio_routing(card, "audio-routing"); + if (ret) + return ret; + } + cpu = of_get_child_by_name(dev->of_node, "cpu"); if (!cpu) { dev_err(dev, "platform property missing or invalid\n"); return -EINVAL; } + codec = of_get_child_by_name(dev->of_node, "codec"); if (!codec) { dev_err(dev, "audio-codec property missing or invalid\n"); @@ -175,12 +302,22 @@ static int loongson_asoc_card_probe(struct platform_device *pdev) if (!ls_priv) return -ENOMEM; + ls_priv->cfg = (const struct loongson_card_config *)device_get_match_data(dev); + if (!ls_priv->cfg) + return -EINVAL; + card = &ls_priv->snd_card; card->dev = dev; card->owner = THIS_MODULE; card->dai_link = loongson_dai_links; card->num_links = ARRAY_SIZE(loongson_dai_links); + + if (ls_priv->cfg->add_dapm_widgets) { + card->dapm_widgets = loongson_aosc_dapm_widgets; + card->num_dapm_widgets = ARRAY_SIZE(loongson_aosc_dapm_widgets); + } + snd_soc_card_set_drvdata(card, ls_priv); ret = device_property_read_string(dev, "model", &card->name); @@ -200,7 +337,19 @@ static int loongson_asoc_card_probe(struct platform_device *pdev) } static const struct of_device_id loongson_asoc_dt_ids[] = { - { .compatible = "loongson,ls-audio-card" }, + /* Loongson-2K1000/Loongson-2K2000/LS7A */ + { + .compatible = "loongson,ls-audio-card", + .data = &ls2k1000_card_config + }, + { + .compatible = "loongson,ls2k0300-forever-pi-audio-card", + .data = &ls2k0300_forever_pi_card_config + }, + { + .compatible = "loongson,ls2k0300-dl2k0300b-audio-card", + .data = &ls2k0300_dl2k0300b_card_config + }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, loongson_asoc_dt_ids); diff --git a/sound/soc/loongson/loongson_i2s_plat.c b/sound/soc/loongson/loongson_i2s_plat.c index ac054b6ce632..82d95c6644ef 100644 --- a/sound/soc/loongson/loongson_i2s_plat.c +++ b/sound/soc/loongson/loongson_i2s_plat.c @@ -2,7 +2,7 @@ // // Loongson I2S controller master mode dirver(platform device) // -// Copyright (C) 2023-2024 Loongson Technology Corporation Limited +// Copyright (C) 2023-2026 Loongson Technology Corporation Limited // // Author: Yingkun Meng // Binbin Zhou @@ -21,6 +21,7 @@ #include "loongson_i2s.h" #include "loongson_dma.h" +/* Loongson-2K1000 APBDMA routing */ #define LOONGSON_I2S_RX_DMA_OFFSET 21 #define LOONGSON_I2S_TX_DMA_OFFSET 18 @@ -30,6 +31,11 @@ #define LOONGSON_DMA3_CONF 0x3 #define LOONGSON_DMA4_CONF 0x4 +struct loongson_i2s_plat_config { + int rev_id; + int (*i2s_dma_config)(struct platform_device *pdev); +}; + static int loongson_i2s_apbdma_config(struct platform_device *pdev) { int val; @@ -47,8 +53,18 @@ static int loongson_i2s_apbdma_config(struct platform_device *pdev) return 0; } +static const struct loongson_i2s_plat_config ls2k0300_i2s_plat_config = { + .rev_id = 1, +}; + +static const struct loongson_i2s_plat_config ls2k1000_i2s_plat_config = { + .rev_id = 0, + .i2s_dma_config = loongson_i2s_apbdma_config, +}; + static int loongson_i2s_plat_probe(struct platform_device *pdev) { + const struct loongson_i2s_plat_config *plat_config; struct device *dev = &pdev->dev; struct loongson_i2s *i2s; struct resource *res; @@ -59,12 +75,17 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev) if (!i2s) return -ENOMEM; - ret = loongson_i2s_apbdma_config(pdev); - if (ret) - return ret; + plat_config = device_get_match_data(dev); + if (!plat_config) + return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2s->reg_base = devm_ioremap_resource(&pdev->dev, res); + if (plat_config->i2s_dma_config) { + ret = plat_config->i2s_dma_config(pdev); + if (ret) + return ret; + } + + i2s->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(i2s->reg_base)) return dev_err_probe(dev, PTR_ERR(i2s->reg_base), "devm_ioremap_resource failed\n"); @@ -87,11 +108,17 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev) if (IS_ERR(i2s_clk)) return dev_err_probe(dev, PTR_ERR(i2s_clk), "clock property invalid\n"); i2s->clk_rate = clk_get_rate(i2s_clk); + i2s->rev_id = plat_config->rev_id; dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); dev_set_name(dev, LS_I2S_DRVNAME); dev_set_drvdata(dev, i2s); + if (i2s->rev_id == 1) { + regmap_update_bits(i2s->regmap, LS_I2S_CTRL, I2S_CTRL_RESET, I2S_CTRL_RESET); + fsleep(200); + } + ret = devm_snd_soc_register_component(dev, &loongson_i2s_edma_component, &loongson_i2s_dai, 1); if (ret) @@ -102,7 +129,8 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev) } static const struct of_device_id loongson_i2s_ids[] = { - { .compatible = "loongson,ls2k1000-i2s" }, + { .compatible = "loongson,ls2k0300-i2s", .data = &ls2k0300_i2s_plat_config }, + { .compatible = "loongson,ls2k1000-i2s", .data = &ls2k1000_i2s_plat_config }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, loongson_i2s_ids); diff --git a/sound/soc/mediatek/Kconfig b/sound/soc/mediatek/Kconfig index 4af7bbb58010..224746e7664f 100644 --- a/sound/soc/mediatek/Kconfig +++ b/sound/soc/mediatek/Kconfig @@ -7,7 +7,7 @@ config SND_SOC_MEDIATEK config SND_SOC_MT2701 tristate "ASoC support for Mediatek MT2701 chip" - depends on ARCH_MEDIATEK + depends on ARCH_MEDIATEK || COMPILE_TEST select SND_SOC_MEDIATEK help This adds ASoC driver for Mediatek MT2701 boards diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650.c b/sound/soc/mediatek/mt8173/mt8173-rt5650.c index 3d6d7bc05b87..8c5096482520 100644 --- a/sound/soc/mediatek/mt8173/mt8173-rt5650.c +++ b/sound/soc/mediatek/mt8173/mt8173-rt5650.c @@ -315,7 +315,7 @@ static int mt8173_rt5650_dev_probe(struct platform_device *pdev) &mt8173_rt5650_priv.pll_from); if (ret) { dev_err(&pdev->dev, - "%s snd_soc_register_card fail %d\n", + "%s device_property_read_u32() fail %d\n", __func__, ret); } } diff --git a/sound/soc/mediatek/mt8365/mt8365-mt6357.c b/sound/soc/mediatek/mt8365/mt8365-mt6357.c index 10f9ef73c130..4339f45ee115 100644 --- a/sound/soc/mediatek/mt8365/mt8365-mt6357.c +++ b/sound/soc/mediatek/mt8365/mt8365-mt6357.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -71,7 +70,8 @@ static const struct snd_soc_dapm_route mt8365_mt6357_routes[] = { static int mt8365_mt6357_int_adda_startup(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(rtd->card); + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv; int ret = 0; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { @@ -102,7 +102,8 @@ static int mt8365_mt6357_int_adda_startup(struct snd_pcm_substream *substream) static void mt8365_mt6357_int_adda_shutdown(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(rtd->card); + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv; int ret = 0; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { @@ -246,7 +247,8 @@ static struct snd_soc_dai_link mt8365_mt6357_dais[] = { static int mt8365_mt6357_gpio_probe(struct snd_soc_card *card) { - struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(card); + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card); + struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv; struct device *dev = card->dev; int ret, i; diff --git a/sound/soc/meson/Makefile b/sound/soc/meson/Makefile index 24078e4396b0..f9ec0ebb01f0 100644 --- a/sound/soc/meson/Makefile +++ b/sound/soc/meson/Makefile @@ -4,6 +4,8 @@ snd-soc-meson-aiu-y := aiu.o snd-soc-meson-aiu-y += aiu-acodec-ctrl.o snd-soc-meson-aiu-y += aiu-codec-ctrl.o snd-soc-meson-aiu-y += aiu-encoder-i2s.o +snd-soc-meson-aiu-y += gx-formatter.o +snd-soc-meson-aiu-y += aiu-formatter-i2s.o snd-soc-meson-aiu-y += aiu-encoder-spdif.o snd-soc-meson-aiu-y += aiu-fifo.o snd-soc-meson-aiu-y += aiu-fifo-i2s.o diff --git a/sound/soc/meson/aiu-encoder-i2s.c b/sound/soc/meson/aiu-encoder-i2s.c index 3b4061508c18..83b579e98f1c 100644 --- a/sound/soc/meson/aiu-encoder-i2s.c +++ b/sound/soc/meson/aiu-encoder-i2s.c @@ -10,14 +10,11 @@ #include #include "aiu.h" +#include "gx-formatter.h" +#include "gx-interface.h" -#define AIU_I2S_SOURCE_DESC_MODE_8CH BIT(0) -#define AIU_I2S_SOURCE_DESC_MODE_24BIT BIT(5) -#define AIU_I2S_SOURCE_DESC_MODE_32BIT BIT(9) #define AIU_I2S_SOURCE_DESC_MODE_SPLIT BIT(11) -#define AIU_RST_SOFT_I2S_FAST BIT(0) -#define AIU_I2S_DAC_CFG_MSB_FIRST BIT(2) #define AIU_CLK_CTRL_I2S_DIV_EN BIT(0) #define AIU_CLK_CTRL_I2S_DIV GENMASK(3, 2) #define AIU_CLK_CTRL_AOCLK_INVERT BIT(6) @@ -35,49 +32,6 @@ static void aiu_encoder_i2s_divider_enable(struct snd_soc_component *component, enable ? AIU_CLK_CTRL_I2S_DIV_EN : 0); } -static int aiu_encoder_i2s_setup_desc(struct snd_soc_component *component, - struct snd_pcm_hw_params *params) -{ - /* Always operate in split (classic interleaved) mode */ - unsigned int desc = AIU_I2S_SOURCE_DESC_MODE_SPLIT; - - /* Reset required to update the pipeline */ - snd_soc_component_write(component, AIU_RST_SOFT, AIU_RST_SOFT_I2S_FAST); - snd_soc_component_read(component, AIU_I2S_SYNC); - - switch (params_physical_width(params)) { - case 16: /* Nothing to do */ - break; - - case 32: - desc |= (AIU_I2S_SOURCE_DESC_MODE_24BIT | - AIU_I2S_SOURCE_DESC_MODE_32BIT); - break; - - default: - return -EINVAL; - } - - switch (params_channels(params)) { - case 2: /* Nothing to do */ - break; - case 8: - desc |= AIU_I2S_SOURCE_DESC_MODE_8CH; - break; - default: - return -EINVAL; - } - - snd_soc_component_update_bits(component, AIU_I2S_SOURCE_DESC, - AIU_I2S_SOURCE_DESC_MODE_8CH | - AIU_I2S_SOURCE_DESC_MODE_24BIT | - AIU_I2S_SOURCE_DESC_MODE_32BIT | - AIU_I2S_SOURCE_DESC_MODE_SPLIT, - desc); - - return 0; -} - static int aiu_encoder_i2s_set_legacy_div(struct snd_soc_component *component, struct snd_pcm_hw_params *params, unsigned int bs) @@ -112,6 +66,9 @@ static int aiu_encoder_i2s_set_more_div(struct snd_soc_component *component, struct snd_pcm_hw_params *params, unsigned int bs) { + struct aiu *aiu = snd_soc_component_get_drvdata(component); + struct gx_iface *iface = &aiu->i2s.iface; + /* * NOTE: this HW is odd. * In most configuration, the i2s divider is 'mclk / blck'. @@ -126,6 +83,18 @@ static int aiu_encoder_i2s_set_more_div(struct snd_soc_component *component, return -EINVAL; } bs += bs / 2; + iface->bs_quirk = true; + } else { + /* + * If the bs quirk is currently applied for one stream and another + * ones tries to setup a configuration for which the quirk is + * not required, then fail. + */ + if (iface->bs_quirk) { + dev_err(component->dev, + "bclk requirements are incompatible with active stream\n"); + return -EINVAL; + } } /* Use CLK_MORE for mclk to bclk divider */ @@ -145,21 +114,17 @@ static int aiu_encoder_i2s_set_clocks(struct snd_soc_component *component, struct snd_pcm_hw_params *params) { struct aiu *aiu = snd_soc_component_get_drvdata(component); + struct gx_iface *iface = &aiu->i2s.iface; unsigned int srate = params_rate(params); unsigned int fs, bs; int ret; /* Get the oversampling factor */ - fs = DIV_ROUND_CLOSEST(clk_get_rate(aiu->i2s.clks[MCLK].clk), srate); + fs = DIV_ROUND_CLOSEST(iface->mclk_rate, srate); - if (fs % 64) + if ((fs % 64) || (fs == 0)) return -EINVAL; - /* Send data MSB first */ - snd_soc_component_update_bits(component, AIU_I2S_DAC_CFG, - AIU_I2S_DAC_CFG_MSB_FIRST, - AIU_I2S_DAC_CFG_MSB_FIRST); - /* Set bclk to lrlck ratio */ snd_soc_component_update_bits(component, AIU_CODEC_DAC_LRCLK_CTRL, AIU_CODEC_DAC_LRCLK_CTRL_DIV, @@ -188,24 +153,53 @@ static int aiu_encoder_i2s_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) { + struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream); + struct gx_iface *iface = ts->iface; struct snd_soc_component *component = dai->component; int ret; - /* Disable the clock while changing the settings */ - aiu_encoder_i2s_divider_enable(component, false); - - ret = aiu_encoder_i2s_setup_desc(component, params); - if (ret) { - dev_err(dai->dev, "setting i2s desc failed\n"); - return ret; + /* + * Enforce interface wide rate symmetry only if there is more than + * 1 stream active. + */ + if (snd_soc_dai_active(dai) > 1) { + if (iface->rate && iface->rate != params_rate(params)) { + dev_err(dai->dev, "can't set iface rate (%d != %d)\n", + iface->rate, params_rate(params)); + return -EINVAL; + } } ret = aiu_encoder_i2s_set_clocks(component, params); if (ret) { - dev_err(dai->dev, "setting i2s clocks failed\n"); + dev_err(dai->dev, "setting i2s clocks failed: %d\n", ret); return ret; } + iface->rate = params_rate(params); + ts->physical_width = params_physical_width(params); + ts->width = params_width(params); + ts->channels = params_channels(params); + + return 0; +} + +static int aiu_encoder_i2s_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream); + struct snd_soc_component *component = dai->component; + int ret; + + if (ts->clk_enabled) + return 0; + + ret = clk_prepare_enable(ts->iface->mclk); + if (ret) + return ret; + + ts->clk_enabled = true; + aiu_encoder_i2s_divider_enable(component, true); return 0; @@ -214,9 +208,24 @@ static int aiu_encoder_i2s_hw_params(struct snd_pcm_substream *substream, static int aiu_encoder_i2s_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { + struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream); + struct gx_iface *iface = ts->iface; struct snd_soc_component *component = dai->component; - aiu_encoder_i2s_divider_enable(component, false); + /* + * If this is the last substream being closed then disable the i2s + * clock divider and clear 'iface->rate'. + */ + if (snd_soc_dai_active(dai) <= 1) { + aiu_encoder_i2s_divider_enable(component, 0); + iface->rate = 0; + iface->bs_quirk = false; + } + + if (ts->clk_enabled) { + clk_disable_unprepare(ts->iface->mclk); + ts->clk_enabled = false; + } return 0; } @@ -224,6 +233,8 @@ static int aiu_encoder_i2s_hw_free(struct snd_pcm_substream *substream, static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; + struct aiu *aiu = snd_soc_component_get_drvdata(component); + struct gx_iface *iface = &aiu->i2s.iface; unsigned int inv = fmt & SND_SOC_DAIFMT_INV_MASK; unsigned int val = 0; unsigned int skew; @@ -255,9 +266,12 @@ static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) skew = 0; break; default: + dev_err(dai->dev, "unsupported dai format\n"); return -EINVAL; } + iface->fmt = fmt; + val |= FIELD_PREP(AIU_CLK_CTRL_LRCLK_SKEW, skew); snd_soc_component_update_bits(component, AIU_CLK_CTRL, AIU_CLK_CTRL_LRCLK_INVERT | @@ -272,6 +286,7 @@ static int aiu_encoder_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int freq, int dir) { struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); + struct gx_iface *iface = &aiu->i2s.iface; int ret; if (WARN_ON(clk_id != 0)) @@ -280,11 +295,15 @@ static int aiu_encoder_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, if (dir == SND_SOC_CLOCK_IN) return 0; - ret = clk_set_rate(aiu->i2s.clks[MCLK].clk, freq); - if (ret) - dev_err(dai->dev, "Failed to set sysclk to %uHz", freq); + ret = clk_set_rate(iface->mclk, freq); + if (ret) { + dev_err(dai->dev, "Failed to set sysclk to %uHz: %d", freq, ret); + return ret; + } - return ret; + iface->mclk_rate = freq; + + return 0; } static const unsigned int hw_channels[] = {2, 8}; @@ -305,15 +324,54 @@ static int aiu_encoder_i2s_startup(struct snd_pcm_substream *substream, SNDRV_PCM_HW_PARAM_CHANNELS, &hw_channel_constraints); if (ret) { - dev_err(dai->dev, "adding channels constraints failed\n"); + dev_err(dai->dev, "adding channels constraints failed: %d\n", ret); return ret; } - ret = clk_bulk_prepare_enable(aiu->i2s.clk_num, aiu->i2s.clks); - if (ret) - dev_err(dai->dev, "failed to enable i2s clocks\n"); + /* + * Enable only clocks which are required for the interface internal + * logic. MCLK is enabled/disabled from the formatter and the I2S + * divider is enabled/disabled in "hw_params"/"hw_free", respectively. + */ + ret = clk_prepare_enable(aiu->i2s.clks[PCLK].clk); + if (ret) { + dev_err(dai->dev, "failed to enable PCLK: %d\n", ret); + return ret; + } + ret = clk_prepare_enable(aiu->i2s.clks[MIXER].clk); + if (ret) { + dev_err(dai->dev, "failed to enable MIXER: %d\n", ret); + clk_disable_unprepare(aiu->i2s.clks[PCLK].clk); + return ret; + } + ret = clk_prepare_enable(aiu->i2s.clks[AOCLK].clk); + if (ret) { + dev_err(dai->dev, "failed to enable AOCLK: %d\n", ret); + clk_disable_unprepare(aiu->i2s.clks[MIXER].clk); + clk_disable_unprepare(aiu->i2s.clks[PCLK].clk); + return ret; + } - return ret; + /* + * We're always operating in split mode for the playback stream. + * + * This setting arguably belong to the 'aiu-formatter', but it's kept + * here for backward compatibility reason. At reset the I2S encoder + * operates in normal mode which would only support 8ch, but by default + * only 2ch are enabled. If a playback stream is started without + * changing to split mode, then the I2S encoder doesn't consume audio + * samples and the playback fails. + * Moving this to 'aiu-formatter' would cause the split mode to be set + * only when the formatter is enabled, which doesn't happen at boot as + * the default value for "HDMI CTRL SRC" is "DISABLED". + */ + ret = snd_soc_component_update_bits(dai->component, AIU_I2S_SOURCE_DESC, + AIU_I2S_SOURCE_DESC_MODE_SPLIT, + AIU_I2S_SOURCE_DESC_MODE_SPLIT); + if (ret < 0) + dev_err(dai->dev, "failed to update AIU_I2S_SOURCE_DESC: %d", ret); + + return 0; } static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream, @@ -321,14 +379,89 @@ static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream, { struct aiu *aiu = snd_soc_component_get_drvdata(dai->component); - clk_bulk_disable_unprepare(aiu->i2s.clk_num, aiu->i2s.clks); + clk_disable_unprepare(aiu->i2s.clks[AOCLK].clk); + clk_disable_unprepare(aiu->i2s.clks[MIXER].clk); + clk_disable_unprepare(aiu->i2s.clks[PCLK].clk); +} + +static int aiu_encoder_i2s_trigger(struct snd_pcm_substream *substream, + int cmd, + struct snd_soc_dai *dai) +{ + struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream); + int ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + ret = gx_stream_start(ts); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + gx_stream_stop(ts); + ret = 0; + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int aiu_encoder_i2s_remove_dai(struct snd_soc_dai *dai) +{ + int stream; + + for_each_pcm_streams(stream) { + struct gx_stream *ts; + + ts = snd_soc_dai_dma_data_get(dai, stream); + if (ts) + gx_stream_free(ts); + + snd_soc_dai_dma_data_set(dai, stream, NULL); + } + + return 0; +} + +static int aiu_encoder_i2s_probe_dai(struct snd_soc_dai *dai) +{ + struct aiu *aiu = snd_soc_dai_get_drvdata(dai); + struct gx_iface *iface = &aiu->i2s.iface; + int stream; + + for_each_pcm_streams(stream) { + struct gx_stream *ts; + + if (!snd_soc_dai_get_widget(dai, stream)) + continue; + + ts = gx_stream_alloc(iface); + if (!ts) { + aiu_encoder_i2s_remove_dai(dai); + return -ENOMEM; + } + snd_soc_dai_dma_data_set(dai, stream, ts); + } + + iface->mclk = aiu->i2s.clks[MCLK].clk; + iface->mclk_rate = clk_get_rate(iface->mclk); + + return 0; } const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops = { + .probe = aiu_encoder_i2s_probe_dai, + .remove = aiu_encoder_i2s_remove_dai, .hw_params = aiu_encoder_i2s_hw_params, + .prepare = aiu_encoder_i2s_prepare, .hw_free = aiu_encoder_i2s_hw_free, .set_fmt = aiu_encoder_i2s_set_fmt, .set_sysclk = aiu_encoder_i2s_set_sysclk, .startup = aiu_encoder_i2s_startup, .shutdown = aiu_encoder_i2s_shutdown, + .trigger = aiu_encoder_i2s_trigger, }; diff --git a/sound/soc/meson/aiu-formatter-i2s.c b/sound/soc/meson/aiu-formatter-i2s.c new file mode 100644 index 000000000000..b4604734fe88 --- /dev/null +++ b/sound/soc/meson/aiu-formatter-i2s.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2026 BayLibre, SAS. +// Author: Valerio Setti + +#include +#include +#include + +#include "aiu.h" +#include "gx-formatter.h" + +#define AIU_I2S_SOURCE_DESC_MODE_8CH BIT(0) +#define AIU_I2S_SOURCE_DESC_MODE_24BIT BIT(5) +#define AIU_I2S_SOURCE_DESC_MODE_32BIT BIT(9) +#define AIU_RST_SOFT_I2S_FAST BIT(0) + +#define AIU_I2S_DAC_CFG_MSB_FIRST BIT(2) + +static struct snd_soc_dai * +aiu_formatter_i2s_get_be(struct snd_soc_dapm_widget *w) +{ + struct snd_soc_dapm_path *p; + struct snd_soc_dai *be; + + snd_soc_dapm_widget_for_each_sink_path(w, p) { + if (!p->connect) + continue; + + if (p->sink->id == snd_soc_dapm_dai_in) + return (struct snd_soc_dai *)p->sink->priv; + + be = aiu_formatter_i2s_get_be(p->sink); + if (be) + return be; + } + + return NULL; +} + +static struct gx_stream * +aiu_formatter_i2s_get_stream(struct snd_soc_dapm_widget *w) +{ + struct snd_soc_dai *be = aiu_formatter_i2s_get_be(w); + + if (!be) + return NULL; + + return snd_soc_dai_dma_data_get_playback(be); +} + +static int aiu_formatter_i2s_prepare(struct regmap *map, + const struct gx_formatter_hw *quirks, + struct gx_stream *ts) +{ + /* Always operate in split (classic interleaved) mode */ + unsigned int desc = 0; + unsigned int tmp; + + /* Reset required to update the pipeline */ + regmap_write(map, AIU_RST_SOFT, AIU_RST_SOFT_I2S_FAST); + regmap_read(map, AIU_I2S_SYNC, &tmp); + + switch (ts->physical_width) { + case 16: /* Nothing to do */ + break; + + case 32: + desc |= (AIU_I2S_SOURCE_DESC_MODE_24BIT | + AIU_I2S_SOURCE_DESC_MODE_32BIT); + break; + + default: + return -EINVAL; + } + + switch (ts->channels) { + case 2: /* Nothing to do */ + break; + case 8: + desc |= AIU_I2S_SOURCE_DESC_MODE_8CH; + break; + default: + return -EINVAL; + } + + regmap_update_bits(map, AIU_I2S_SOURCE_DESC, + AIU_I2S_SOURCE_DESC_MODE_8CH | + AIU_I2S_SOURCE_DESC_MODE_24BIT | + AIU_I2S_SOURCE_DESC_MODE_32BIT, + desc); + + /* Send data MSB first */ + regmap_update_bits(map, AIU_I2S_DAC_CFG, + AIU_I2S_DAC_CFG_MSB_FIRST, + AIU_I2S_DAC_CFG_MSB_FIRST); + + return 0; +} + +const struct gx_formatter_ops aiu_formatter_i2s_ops = { + .get_stream = aiu_formatter_i2s_get_stream, + .prepare = aiu_formatter_i2s_prepare, +}; diff --git a/sound/soc/meson/aiu.c b/sound/soc/meson/aiu.c index f2890111c1d2..64ace4d25d92 100644 --- a/sound/soc/meson/aiu.c +++ b/sound/soc/meson/aiu.c @@ -29,13 +29,22 @@ static SOC_ENUM_SINGLE_DECL(aiu_spdif_encode_sel_enum, AIU_I2S_MISC, static const struct snd_kcontrol_new aiu_spdif_encode_mux = SOC_DAPM_ENUM("SPDIF Buffer Src", aiu_spdif_encode_sel_enum); -static const struct snd_soc_dapm_widget aiu_cpu_dapm_widgets[] = { - SND_SOC_DAPM_MUX("SPDIF SRC SEL", SND_SOC_NOPM, 0, 0, - &aiu_spdif_encode_mux), +#define AIU_WIDGET_SPDIF_SRC_SEL 0 +#define AIU_WIDGET_I2S_FORMATTER 1 + +static struct snd_soc_dapm_widget aiu_cpu_dapm_widgets[] = { + [AIU_WIDGET_SPDIF_SRC_SEL] = + SND_SOC_DAPM_MUX("SPDIF SRC SEL", SND_SOC_NOPM, 0, 0, + &aiu_spdif_encode_mux), + [AIU_WIDGET_I2S_FORMATTER] = + SND_SOC_DAPM_PGA_E("I2S Formatter", SND_SOC_NOPM, 0, 0, NULL, 0, + gx_formatter_event, + (SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_PRE_PMD)), }; static const struct snd_soc_dapm_route aiu_cpu_dapm_routes[] = { - { "I2S Encoder Playback", NULL, "I2S FIFO Playback" }, + { "I2S Formatter", NULL, "I2S FIFO Playback" }, + { "I2S Encoder Playback", NULL, "I2S Formatter" }, { "SPDIF SRC SEL", "SPDIF", "SPDIF FIFO Playback" }, { "SPDIF SRC SEL", "I2S", "I2S FIFO Playback" }, { "SPDIF Encoder Playback", NULL, "SPDIF SRC SEL" }, @@ -172,6 +181,11 @@ static const struct regmap_config aiu_regmap_cfg = { .max_register = 0x2ac, }; +const struct gx_formatter_driver aiu_formatter_i2s_drv = { + .regmap_cfg = &aiu_regmap_cfg, + .ops = &aiu_formatter_i2s_ops, +}; + static int aiu_clk_bulk_get(struct device *dev, const char * const *ids, unsigned int num, @@ -282,6 +296,14 @@ static int aiu_probe(struct platform_device *pdev) if (ret) return ret; + /* Allocate the aiu-formatter into its widget */ + ret = gx_formatter_create(dev, &aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER], + &aiu_formatter_i2s_drv, map); + if (ret) { + dev_err(dev, "Failed to allocate aiu formatter\n"); + goto err; + } + /* Register the cpu component of the aiu */ ret = snd_soc_register_component(dev, &aiu_cpu_component, aiu_cpu_dai_drv, @@ -310,12 +332,14 @@ static int aiu_probe(struct platform_device *pdev) return 0; err: + gx_formatter_free(&aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER]); snd_soc_unregister_component(dev); return ret; } static void aiu_remove(struct platform_device *pdev) { + gx_formatter_free(&aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER]); snd_soc_unregister_component(&pdev->dev); } diff --git a/sound/soc/meson/aiu.h b/sound/soc/meson/aiu.h index 0f94c8bf6081..7d0b98c1f351 100644 --- a/sound/soc/meson/aiu.h +++ b/sound/soc/meson/aiu.h @@ -7,6 +7,8 @@ #ifndef _MESON_AIU_H #define _MESON_AIU_H +#include "gx-formatter.h" + struct clk; struct clk_bulk_data; struct device; @@ -25,6 +27,7 @@ struct aiu_interface { struct clk_bulk_data *clks; unsigned int clk_num; int irq; + struct gx_iface iface; }; struct aiu_platform_data { @@ -58,6 +61,7 @@ extern const struct snd_soc_dai_ops aiu_fifo_i2s_dai_ops; extern const struct snd_soc_dai_ops aiu_fifo_spdif_dai_ops; extern const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops; extern const struct snd_soc_dai_ops aiu_encoder_spdif_dai_ops; +extern const struct gx_formatter_ops aiu_formatter_i2s_ops; #define AIU_IEC958_BPF 0x000 #define AIU_958_MISC 0x010 diff --git a/sound/soc/meson/g12a-toacodec.c b/sound/soc/meson/g12a-toacodec.c index a95375b53f0a..21941ee552c5 100644 --- a/sound/soc/meson/g12a-toacodec.c +++ b/sound/soc/meson/g12a-toacodec.c @@ -312,7 +312,7 @@ static int g12a_toacodec_probe(struct platform_device *pdev) ret = device_reset(dev); if (ret) - return ret; + return dev_err_probe(dev, ret, "failed to reset device\n"); regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) diff --git a/sound/soc/meson/g12a-tohdmitx.c b/sound/soc/meson/g12a-tohdmitx.c index d541ca4acfaf..967109ca2b57 100644 --- a/sound/soc/meson/g12a-tohdmitx.c +++ b/sound/soc/meson/g12a-tohdmitx.c @@ -251,7 +251,7 @@ static int g12a_tohdmitx_probe(struct platform_device *pdev) ret = device_reset(dev); if (ret) - return ret; + return dev_err_probe(dev, ret, "failed to reset device\n"); regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) diff --git a/sound/soc/meson/gx-formatter.c b/sound/soc/meson/gx-formatter.c new file mode 100644 index 000000000000..311e63affb23 --- /dev/null +++ b/sound/soc/meson/gx-formatter.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +// +// Copyright (c) 2026 BayLibre, SAS. +// Author: Valerio Setti + +#include +#include +#include +#include + +#include "gx-formatter.h" + +struct gx_formatter { + struct list_head list; + struct gx_stream *stream; + const struct gx_formatter_driver *drv; + bool enabled; + struct regmap *map; +}; + +static int gx_formatter_enable(struct gx_formatter *formatter) +{ + int ret; + + /* Do nothing if the formatter is already enabled */ + if (formatter->enabled) + return 0; + + /* Setup the stream parameter in the formatter */ + if (formatter->drv->ops->prepare) { + ret = formatter->drv->ops->prepare(formatter->map, + formatter->drv->quirks, + formatter->stream); + if (ret) + return ret; + } + + /* Finally, actually enable the formatter */ + if (formatter->drv->ops->enable) + formatter->drv->ops->enable(formatter->map); + + formatter->enabled = true; + + return 0; +} + +static void gx_formatter_disable(struct gx_formatter *formatter) +{ + /* Do nothing if the formatter is already disabled */ + if (!formatter->enabled) + return; + + if (formatter->drv->ops->disable) + formatter->drv->ops->disable(formatter->map); + + formatter->enabled = false; +} + +static int gx_formatter_attach(struct gx_formatter *formatter) +{ + struct gx_stream *ts = formatter->stream; + int ret = 0; + + mutex_lock(&ts->lock); + + /* Catch up if the stream is already running when we attach */ + if (ts->ready) { + ret = gx_formatter_enable(formatter); + if (ret) { + pr_err("failed to enable formatter\n"); + goto out; + } + } + + list_add_tail(&formatter->list, &ts->formatter_list); +out: + mutex_unlock(&ts->lock); + return ret; +} + +static void gx_formatter_detach(struct gx_formatter *formatter) +{ + struct gx_stream *ts = formatter->stream; + + if (!ts) + return; + + mutex_lock(&ts->lock); + list_del(&formatter->list); + mutex_unlock(&ts->lock); + + gx_formatter_disable(formatter); +} + +static int gx_formatter_power_up(struct gx_formatter *formatter, + struct snd_soc_dapm_widget *w) +{ + struct gx_stream *ts = formatter->drv->ops->get_stream(w); + int ret; + + /* + * If we don't get a stream at this stage, it would mean that the + * widget is powering up but is not attached to any backend DAI. + * It should not happen, ever ! + */ + if (WARN_ON(!ts)) + return -ENODEV; + + formatter->stream = ts; + INIT_LIST_HEAD(&formatter->list); + ret = gx_formatter_attach(formatter); + if (ret) + return ret; + + return 0; +} + +static void gx_formatter_power_down(struct gx_formatter *formatter) +{ + gx_formatter_detach(formatter); + formatter->stream = NULL; +} + +int gx_formatter_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *control, + int event) +{ + struct snd_soc_component *c; + struct gx_formatter *formatter; + int ret = 0; + + c = snd_soc_dapm_to_component(w->dapm); + + if (w->priv) + formatter = w->priv; + else + formatter = snd_soc_component_get_drvdata(c); + + switch (event) { + case SND_SOC_DAPM_PRE_PMU: + ret = gx_formatter_power_up(formatter, w); + break; + + case SND_SOC_DAPM_PRE_PMD: + gx_formatter_power_down(formatter); + break; + + default: + dev_err(c->dev, "Unexpected event %d\n", event); + return -EINVAL; + } + + return ret; +} +EXPORT_SYMBOL_GPL(gx_formatter_event); + +int gx_formatter_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct gx_formatter_driver *drv; + struct gx_formatter *formatter; + void __iomem *regs; + + drv = of_device_get_match_data(dev); + if (!drv) { + dev_err(dev, "failed to match device\n"); + return -ENODEV; + } + + formatter = devm_kzalloc(dev, sizeof(*formatter), GFP_KERNEL); + if (!formatter) + return -ENOMEM; + platform_set_drvdata(pdev, formatter); + formatter->drv = drv; + + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + formatter->map = devm_regmap_init_mmio(dev, regs, drv->regmap_cfg); + if (IS_ERR(formatter->map)) { + dev_err(dev, "failed to init regmap: %ld\n", + PTR_ERR(formatter->map)); + return PTR_ERR(formatter->map); + } + + return devm_snd_soc_register_component(dev, drv->component_drv, + NULL, 0); +} +EXPORT_SYMBOL_GPL(gx_formatter_probe); + +int gx_formatter_create(struct device *dev, + struct snd_soc_dapm_widget *w, + const struct gx_formatter_driver *drv, + struct regmap *regmap) +{ + struct gx_formatter *formatter; + + formatter = devm_kzalloc(dev, sizeof(*formatter), GFP_KERNEL); + if (!formatter) + return -ENOMEM; + + formatter->drv = drv; + formatter->map = regmap; + + w->priv = formatter; + + return 0; +} +EXPORT_SYMBOL_GPL(gx_formatter_create); + +int gx_stream_start(struct gx_stream *ts) +{ + struct gx_formatter *formatter; + int ret = 0; + + mutex_lock(&ts->lock); + + /* Start all the formatters attached to the stream */ + list_for_each_entry(formatter, &ts->formatter_list, list) { + ret = gx_formatter_enable(formatter); + if (ret) { + pr_err("failed to enable formatter\n"); + goto out; + } + } + + ts->ready = true; + +out: + mutex_unlock(&ts->lock); + return ret; +} +EXPORT_SYMBOL_GPL(gx_stream_start); + +void gx_stream_stop(struct gx_stream *ts) +{ + struct gx_formatter *formatter; + + mutex_lock(&ts->lock); + ts->ready = false; + + /* Stop all the formatters attached to the stream */ + list_for_each_entry(formatter, &ts->formatter_list, list) { + gx_formatter_disable(formatter); + } + + mutex_unlock(&ts->lock); +} +EXPORT_SYMBOL_GPL(gx_stream_stop); + +struct gx_stream *gx_stream_alloc(struct gx_iface *iface) +{ + struct gx_stream *ts; + + ts = kzalloc(sizeof(*ts), GFP_KERNEL); + if (ts) { + INIT_LIST_HEAD(&ts->formatter_list); + mutex_init(&ts->lock); + ts->iface = iface; + } + + return ts; +} +EXPORT_SYMBOL_GPL(gx_stream_alloc); + +void gx_stream_free(struct gx_stream *ts) +{ + /* + * If the list is not empty, it would mean that one of the formatter + * widget is still powered and attached to the interface while we + * are removing the TDM DAI. It should not be possible + */ + WARN_ON(!list_empty(&ts->formatter_list)); + mutex_destroy(&ts->lock); + kfree(ts); +} +EXPORT_SYMBOL_GPL(gx_stream_free); + +MODULE_DESCRIPTION("Amlogic GX formatter driver"); +MODULE_AUTHOR("Valerio Setti "); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/meson/gx-formatter.h b/sound/soc/meson/gx-formatter.h new file mode 100644 index 000000000000..b90b1814d79b --- /dev/null +++ b/sound/soc/meson/gx-formatter.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright (c) 2026 Baylibre SAS. + * Author: Valerio Setti + */ + +#ifndef _MESON_GX_FORMATTER_H +#define _MESON_GX_FORMATTER_H + +#include "gx-interface.h" + +struct platform_device; +struct regmap; +struct snd_soc_dapm_widget; +struct snd_kcontrol; + +struct gx_formatter_hw { + unsigned int skew_offset; +}; + +struct gx_formatter_ops { + struct gx_stream *(*get_stream)(struct snd_soc_dapm_widget *w); + void (*enable)(struct regmap *map); + void (*disable)(struct regmap *map); + int (*prepare)(struct regmap *map, + const struct gx_formatter_hw *quirks, + struct gx_stream *ts); +}; + +struct gx_formatter_driver { + const struct snd_soc_component_driver *component_drv; + const struct regmap_config *regmap_cfg; + const struct gx_formatter_ops *ops; + const struct gx_formatter_hw *quirks; +}; + +int gx_formatter_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *control, + int event); +int gx_formatter_probe(struct platform_device *pdev); + +int gx_formatter_create(struct device *dev, + struct snd_soc_dapm_widget *w, + const struct gx_formatter_driver *drv, + struct regmap *regmap); + +/* + * Formatter data is already freed when the associated device is removed, + * so we just need to remove the pointer from the widget. + */ +static inline void gx_formatter_free(struct snd_soc_dapm_widget *w) +{ + w->priv = NULL; +} + +#endif /* _MESON_GX_FORMATTER_H */ diff --git a/sound/soc/meson/gx-interface.h b/sound/soc/meson/gx-interface.h new file mode 100644 index 000000000000..65c46dcce32a --- /dev/null +++ b/sound/soc/meson/gx-interface.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright (c) 2026 Baylibre SAS. + * Author: Valerio Setti + */ + +#ifndef _MESON_GX_INTERFACE_H +#define _MESON_GX_INTERFACE_H + +#include +#include +#include +#include +#include + +struct gx_iface { + struct clk *mclk; + unsigned long mclk_rate; + + /* format is common to all the DAIs of the iface */ + unsigned int fmt; + + /* For component wide symmetry */ + int rate; + + /* Only for GX platform */ + int bs_quirk; +}; + +struct gx_stream { + struct gx_iface *iface; + struct list_head formatter_list; + struct mutex lock; + unsigned int channels; + unsigned int width; + unsigned int physical_width; + bool ready; + + /* For continuous clock tracking */ + bool clk_enabled; +}; + +struct gx_stream *gx_stream_alloc(struct gx_iface *iface); +void gx_stream_free(struct gx_stream *ts); +int gx_stream_start(struct gx_stream *ts); +void gx_stream_stop(struct gx_stream *ts); + +#endif /* _MESON_GX_INTERFACE_H */ diff --git a/sound/soc/meson/t9015.c b/sound/soc/meson/t9015.c index da1a93946d67..f0b55aee5241 100644 --- a/sound/soc/meson/t9015.c +++ b/sound/soc/meson/t9015.c @@ -265,10 +265,8 @@ static int t9015_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->avdd), "failed to AVDD\n"); ret = device_reset(dev); - if (ret) { - dev_err(dev, "reset failed\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to reset device\n"); regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) { diff --git a/sound/soc/qcom/apq8096.c b/sound/soc/qcom/apq8096.c index 4f6594cc723c..cfd6438dbcb3 100644 --- a/sound/soc/qcom/apq8096.c +++ b/sound/soc/qcom/apq8096.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2018, Linaro Limited -#include #include #include #include diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 2e5b25b8d00f..641d6d243229 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -587,6 +587,10 @@ static int graph_callback(const struct gpr_resp_pkt *data, void *priv, int op) token = hdr->token & APM_WRITE_TOKEN_MASK; done = data->payload; + if (!graph->rx_data.buf) { + mutex_unlock(&graph->lock); + break; + } phys = graph->rx_data.buf[token].phys; mutex_unlock(&graph->lock); /* token numbering starts at 0 */ @@ -609,6 +613,10 @@ static int graph_callback(const struct gpr_resp_pkt *data, void *priv, int op) client_event = APM_CLIENT_EVENT_DATA_READ_DONE; mutex_lock(&graph->lock); rd_done = data->payload; + if (!graph->tx_data.buf) { + mutex_unlock(&graph->lock); + break; + } phys = graph->tx_data.buf[hdr->token].phys; mutex_unlock(&graph->lock); /* token numbering starts at 0 */ diff --git a/sound/soc/qcom/sc7280.c b/sound/soc/qcom/sc7280.c index abdd58c1d0a4..d3d8a6e83268 100644 --- a/sound/soc/qcom/sc7280.c +++ b/sound/soc/qcom/sc7280.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/qcom/storm.c b/sound/soc/qcom/storm.c index c8d5ac43a176..1e0eda8c24c4 100644 --- a/sound/soc/qcom/storm.c +++ b/sound/soc/qcom/storm.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/renesas/Kconfig b/sound/soc/renesas/Kconfig index 11c2027c88a7..6520217e7407 100644 --- a/sound/soc/renesas/Kconfig +++ b/sound/soc/renesas/Kconfig @@ -56,6 +56,7 @@ config SND_SOC_MSIOF config SND_SOC_RZ tristate "RZ/G2L series SSIF-2 support" depends on ARCH_RZG2L || COMPILE_TEST + select SND_SOC_GENERIC_DMAENGINE_PCM help This option enables RZ/G2L SSIF-2 sound support. diff --git a/sound/soc/renesas/fsi.c b/sound/soc/renesas/fsi.c index ae86014c3819..6be6587e1095 100644 --- a/sound/soc/renesas/fsi.c +++ b/sound/soc/renesas/fsi.c @@ -1992,7 +1992,7 @@ static int fsi_probe(struct platform_device *pdev) const struct fsi_core *core; struct fsi_priv *fsi; struct resource *res; - unsigned int irq; + int irq; int ret; memset(&info, 0, sizeof(info)); @@ -2007,12 +2007,15 @@ static int fsi_probe(struct platform_device *pdev) } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - irq = platform_get_irq(pdev, 0); - if (!res || (int)irq <= 0) { + if (!res) { dev_err(&pdev->dev, "Not enough FSI platform resources.\n"); return -ENODEV; } + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL); if (!master) return -ENOMEM; diff --git a/sound/soc/renesas/rcar/adg.c b/sound/soc/renesas/rcar/adg.c index 5479cefb6dbe..53efd1be5139 100644 --- a/sound/soc/renesas/rcar/adg.c +++ b/sound/soc/renesas/rcar/adg.c @@ -45,6 +45,7 @@ struct rsnd_adg { struct rsnd_mod mod; int clkin_rate[CLKINMAX]; bool ssi_clk_prepared; + bool clk_enabled; int clkin_size; int clkout_size; u32 ckr; @@ -463,6 +464,22 @@ int rsnd_adg_clk_control(struct rsnd_priv *priv, int enable) struct clk *clk; int ret = 0, i; + /* + * rsnd_adg_clk_enable() and rsnd_adg_clk_disable() can be called + * redundantly, for example when system suspend follows a resume + * whose enable failed. Make this function idempotent so that the + * "adg" clock, which has no clkin_rate[] style guard, is never + * disabled twice. + */ + if (enable) { + if (adg->clk_enabled) + return 0; + } else { + if (!adg->clk_enabled) + return 0; + adg->clk_enabled = false; + } + if (enable) { ret = clk_prepare_enable(adg->adg); if (ret < 0) @@ -520,12 +537,22 @@ int rsnd_adg_clk_control(struct rsnd_priv *priv, int enable) * rsnd_adg_clk_enable() might return error (_disable() will not). * We need to rollback in such case */ - if (ret < 0) + if (ret < 0) { + /* + * Mark as enabled so that the rollback below is not + * short-circuited by the idempotency guard. It clears + * the flag again on its way through. + */ + adg->clk_enabled = true; rsnd_adg_clk_disable(priv); + return ret; + } /* disable adg */ if (!enable) clk_disable_unprepare(adg->adg); + else + adg->clk_enabled = true; return ret; } diff --git a/sound/soc/renesas/rcar/src.c b/sound/soc/renesas/rcar/src.c index ac806bdc96d9..2cdb39e898af 100644 --- a/sound/soc/renesas/rcar/src.c +++ b/sound/soc/renesas/rcar/src.c @@ -850,6 +850,7 @@ void rsnd_src_suspend(struct rsnd_priv *priv) clk_disable_unprepare(src_ctrl->scu_x2); clk_disable_unprepare(src_ctrl->scu); + clk_disable_unprepare(src_ctrl->scu_supply); } void rsnd_src_resume(struct rsnd_priv *priv) @@ -861,6 +862,7 @@ void rsnd_src_resume(struct rsnd_priv *priv) if (!src_ctrl) return; + clk_prepare_enable(src_ctrl->scu_supply); clk_prepare_enable(src_ctrl->scu); clk_prepare_enable(src_ctrl->scu_x2); diff --git a/sound/soc/renesas/rz-ssi.c b/sound/soc/renesas/rz-ssi.c index 71e434cfe07b..9fe8a639c47c 100644 --- a/sound/soc/renesas/rz-ssi.c +++ b/sound/soc/renesas/rz-ssi.c @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -87,8 +89,6 @@ struct rz_ssi_stream { struct rz_ssi_priv *priv; struct snd_pcm_substream *substream; int fifo_sample_size; /* sample capacity of SSI FIFO */ - int dma_buffer_pos; /* The address for the next DMA descriptor */ - int completed_dma_buf_pos; /* The address of the last completed DMA descriptor. */ int period_counter; /* for keeping track of periods transferred */ int buffer_pos; /* current frame position in the buffer */ int running; /* 0=stopped, 1=running */ @@ -96,8 +96,6 @@ struct rz_ssi_stream { int uerr_num; int oerr_num; - struct dma_chan *dma_ch; - int (*transfer)(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm); }; @@ -108,7 +106,6 @@ struct rz_ssi_priv { struct clk *sfr_clk; struct clk *clk; - phys_addr_t phys; int irq_int; int irq_tx; int irq_rx; @@ -148,9 +145,10 @@ struct rz_ssi_priv { unsigned int sample_width; unsigned int sample_bits; } hw_params_cache; -}; -static void rz_ssi_dma_complete(void *data); + struct snd_dmaengine_dai_dma_data dma_dais[SNDRV_PCM_STREAM_LAST + 1]; + struct dma_chan *dmas[SNDRV_PCM_STREAM_LAST + 1]; +}; static void rz_ssi_reg_writel(struct rz_ssi_priv *priv, uint reg, u32 data) { @@ -172,11 +170,6 @@ static void rz_ssi_reg_mask_setl(struct rz_ssi_priv *priv, uint reg, writel(val, (priv->base + reg)); } -static inline bool rz_ssi_stream_is_play(struct snd_pcm_substream *substream) -{ - return substream->stream == SNDRV_PCM_STREAM_PLAYBACK; -} - static inline struct rz_ssi_stream * rz_ssi_stream_get(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream) { @@ -185,7 +178,7 @@ rz_ssi_stream_get(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream) static inline bool rz_ssi_is_dma_enabled(struct rz_ssi_priv *ssi) { - return (ssi->playback.dma_ch && (ssi->dma_rt || ssi->capture.dma_ch)); + return !ssi->playback.transfer && !ssi->capture.transfer; } static void rz_ssi_set_substream(struct rz_ssi_stream *strm, @@ -215,8 +208,6 @@ static void rz_ssi_stream_init(struct rz_ssi_stream *strm, struct snd_pcm_substream *substream) { rz_ssi_set_substream(strm, substream); - strm->dma_buffer_pos = 0; - strm->completed_dma_buf_pos = 0; strm->period_counter = 0; strm->buffer_pos = 0; @@ -242,12 +233,13 @@ static void rz_ssi_stream_quit(struct rz_ssi_priv *ssi, dev_info(dev, "underrun = %d\n", strm->uerr_num); } -static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate, - unsigned int channels) +static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, struct snd_pcm_substream *substream, + unsigned int rate, unsigned int channels) { static u8 ckdv[] = { 1, 2, 4, 8, 16, 32, 64, 128, 6, 12, 24, 48, 96 }; unsigned int channel_bits = 32; /* System Word Length */ unsigned long bclk_rate = rate * channels * channel_bits; + struct snd_dmaengine_dai_dma_data *dma_dai; unsigned int div; unsigned int i; u32 ssicr = 0; @@ -290,6 +282,8 @@ static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate, return -EINVAL; } + dma_dai = &ssi->dma_dais[substream->stream]; + /* * DWL: Data Word Length = {16, 24, 32} bits * SWL: System Word Length = 32 bits @@ -298,12 +292,15 @@ static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate, switch (ssi->hw_params_cache.sample_width) { case 16: ssicr |= SSICR_DWL(1); + dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; break; case 24: ssicr |= SSICR_DWL(5) | SSICR_PDTA; + dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; break; case 32: ssicr |= SSICR_DWL(6); + dma_dai->addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; break; default: dev_err(ssi->dev, "Not support %u data width", @@ -344,7 +341,7 @@ static void rz_ssi_set_idle(struct rz_ssi_priv *ssi) static int rz_ssi_start(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) { - bool is_play = rz_ssi_stream_is_play(strm->substream); + bool is_play = strm->substream->stream == SNDRV_PCM_STREAM_PLAYBACK; bool is_full_duplex; u32 ssicr, ssifcr; @@ -423,14 +420,6 @@ static int rz_ssi_stop(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) /* Disable TX/RX */ rz_ssi_reg_mask_setl(ssi, SSICR, SSICR_TEN | SSICR_REN, 0); - /* Cancel all remaining DMA transactions */ - if (rz_ssi_is_dma_enabled(ssi)) { - if (ssi->playback.dma_ch) - dmaengine_terminate_async(ssi->playback.dma_ch); - if (ssi->capture.dma_ch) - dmaengine_terminate_async(ssi->capture.dma_ch); - } - rz_ssi_set_idle(ssi); return 0; @@ -458,10 +447,6 @@ static void rz_ssi_pointer_update(struct rz_ssi_stream *strm, int frames) snd_pcm_period_elapsed(strm->substream); strm->period_counter = current_period; } - - strm->completed_dma_buf_pos += runtime->period_size; - if (strm->completed_dma_buf_pos >= runtime->buffer_size) - strm->completed_dma_buf_pos = 0; } static int rz_ssi_pio_recv(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) @@ -606,12 +591,6 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data) if (irq == ssi->irq_int) { /* error or idle */ bool is_stopped = !!(ssisr & (SSISR_RUIRQ | SSISR_ROIRQ | SSISR_TUIRQ | SSISR_TOIRQ)); - int i, count; - - if (rz_ssi_is_dma_enabled(ssi)) - count = 4; - else - count = 1; if (ssi->capture.substream && is_stopped) { if (ssisr & SSISR_RUIRQ) @@ -631,19 +610,41 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data) rz_ssi_stop(ssi, strm_playback); } + if (!rz_ssi_is_stream_running(&ssi->playback) && + !rz_ssi_is_stream_running(&ssi->capture) && + rz_ssi_is_dma_enabled(ssi) && is_stopped) { + if (ssi->playback.substream && + ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]) + dmaengine_pause(ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]); + if (ssi->capture.substream && + ssi->dmas[SNDRV_PCM_STREAM_CAPTURE] && + /* Avoid calling pause twice in case of half duplex. */ + ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK] != + ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]) + dmaengine_pause(ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]); + } + /* Clear all flags */ rz_ssi_reg_mask_setl(ssi, SSISR, SSISR_TOIRQ | SSISR_TUIRQ | SSISR_ROIRQ | SSISR_RUIRQ, 0); /* Add/remove more data */ if (ssi->capture.substream && is_stopped) { - for (i = 0; i < count; i++) + if (rz_ssi_is_dma_enabled(ssi)) { + if (ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]) + dmaengine_resume(ssi->dmas[SNDRV_PCM_STREAM_CAPTURE]); + } else { strm_capture->transfer(ssi, strm_capture); + } } if (ssi->playback.substream && is_stopped) { - for (i = 0; i < count; i++) + if (rz_ssi_is_dma_enabled(ssi)) { + if (ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]) + dmaengine_resume(ssi->dmas[SNDRV_PCM_STREAM_PLAYBACK]); + } else { strm_playback->transfer(ssi, strm_playback); + } } /* Resume */ @@ -679,153 +680,11 @@ static irqreturn_t rz_ssi_interrupt(int irq, void *data) return IRQ_HANDLED; } -static int rz_ssi_dma_slave_config(struct rz_ssi_priv *ssi, - struct dma_chan *dma_ch, bool is_play) -{ - struct dma_slave_config cfg; - - memset(&cfg, 0, sizeof(cfg)); - - cfg.direction = is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; - cfg.dst_addr = ssi->phys + SSIFTDR; - cfg.src_addr = ssi->phys + SSIFRDR; - if (ssi->hw_params_cache.sample_width == 16) { - cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; - cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; - } else { - cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - } - - return dmaengine_slave_config(dma_ch, &cfg); -} - -static int rz_ssi_dma_transfer(struct rz_ssi_priv *ssi, - struct rz_ssi_stream *strm) -{ - struct snd_pcm_substream *substream = strm->substream; - struct dma_async_tx_descriptor *desc; - struct snd_pcm_runtime *runtime; - enum dma_transfer_direction dir; - u32 dma_paddr, dma_size; - int amount; - - if (!rz_ssi_stream_is_valid(ssi, strm)) - return -EINVAL; - - runtime = substream->runtime; - if (runtime->state == SNDRV_PCM_STATE_DRAINING) - /* - * Stream is ending, so do not queue up any more DMA - * transfers otherwise we play partial sound clips - * because we can't shut off the DMA quick enough. - */ - return 0; - - dir = rz_ssi_stream_is_play(substream) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; - - /* Always transfer 1 period */ - amount = runtime->period_size; - - /* DMA physical address and size */ - dma_paddr = runtime->dma_addr + frames_to_bytes(runtime, - strm->dma_buffer_pos); - dma_size = frames_to_bytes(runtime, amount); - desc = dmaengine_prep_slave_single(strm->dma_ch, dma_paddr, dma_size, - dir, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - if (!desc) { - dev_err(ssi->dev, "dmaengine_prep_slave_single() fail\n"); - return -ENOMEM; - } - - desc->callback = rz_ssi_dma_complete; - desc->callback_param = strm; - - if (dmaengine_submit(desc) < 0) { - dev_err(ssi->dev, "dmaengine_submit() fail\n"); - return -EIO; - } - - /* Update DMA pointer */ - strm->dma_buffer_pos += amount; - if (strm->dma_buffer_pos >= runtime->buffer_size) - strm->dma_buffer_pos = 0; - - /* Start DMA */ - dma_async_issue_pending(strm->dma_ch); - - return 0; -} - -static void rz_ssi_dma_complete(void *data) -{ - struct rz_ssi_stream *strm = (struct rz_ssi_stream *)data; - - if (!strm->running || !strm->substream || !strm->substream->runtime) - return; - - /* Note that next DMA transaction has probably already started */ - rz_ssi_pointer_update(strm, strm->substream->runtime->period_size); - - /* Queue up another DMA transaction */ - rz_ssi_dma_transfer(strm->priv, strm); -} - -static void rz_ssi_release_dma_channels(struct rz_ssi_priv *ssi) -{ - if (ssi->playback.dma_ch) { - dma_release_channel(ssi->playback.dma_ch); - ssi->playback.dma_ch = NULL; - if (ssi->dma_rt) - ssi->dma_rt = false; - } - - if (ssi->capture.dma_ch) { - dma_release_channel(ssi->capture.dma_ch); - ssi->capture.dma_ch = NULL; - } -} - -static int rz_ssi_dma_request(struct rz_ssi_priv *ssi, struct device *dev) -{ - ssi->playback.dma_ch = dma_request_chan(dev, "tx"); - if (IS_ERR(ssi->playback.dma_ch)) - ssi->playback.dma_ch = NULL; - - ssi->capture.dma_ch = dma_request_chan(dev, "rx"); - if (IS_ERR(ssi->capture.dma_ch)) - ssi->capture.dma_ch = NULL; - - if (!ssi->playback.dma_ch && !ssi->capture.dma_ch) { - ssi->playback.dma_ch = dma_request_chan(dev, "rt"); - if (IS_ERR(ssi->playback.dma_ch)) { - ssi->playback.dma_ch = NULL; - goto no_dma; - } - - ssi->dma_rt = true; - } - - if (!rz_ssi_is_dma_enabled(ssi)) - goto no_dma; - - return 0; - -no_dma: - rz_ssi_release_dma_channels(ssi); - - return -ENODEV; -} - static int rz_ssi_trigger_resume(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm) { struct snd_pcm_substream *substream = strm->substream; - struct snd_pcm_runtime *runtime = substream->runtime; int ret; - strm->dma_buffer_pos = strm->completed_dma_buf_pos + runtime->period_size; - if (rz_ssi_is_stream_running(&ssi->playback) || rz_ssi_is_stream_running(&ssi->capture)) return 0; @@ -834,7 +693,7 @@ static int rz_ssi_trigger_resume(struct rz_ssi_priv *ssi, struct rz_ssi_stream * if (ret) return ret; - return rz_ssi_clk_setup(ssi, ssi->hw_params_cache.rate, + return rz_ssi_clk_setup(ssi, substream, ssi->hw_params_cache.rate, ssi->hw_params_cache.channels); } @@ -843,10 +702,11 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd, { struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); struct rz_ssi_stream *strm = rz_ssi_stream_get(ssi, substream); - int ret = 0, i, num_transfer = 1; + int ret = 0; switch (cmd) { case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: ret = rz_ssi_trigger_resume(ssi, strm); if (ret) return ret; @@ -857,28 +717,7 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd, if (cmd == SNDRV_PCM_TRIGGER_START) rz_ssi_stream_init(strm, substream); - if (rz_ssi_is_dma_enabled(ssi)) { - bool is_playback = rz_ssi_stream_is_play(substream); - - if (ssi->dma_rt) - ret = rz_ssi_dma_slave_config(ssi, ssi->playback.dma_ch, - is_playback); - else - ret = rz_ssi_dma_slave_config(ssi, strm->dma_ch, - is_playback); - - /* Fallback to pio */ - if (ret < 0) { - ssi->playback.transfer = rz_ssi_pio_send; - ssi->capture.transfer = rz_ssi_pio_recv; - rz_ssi_release_dma_channels(ssi); - } else { - /* For DMA, queue up multiple DMA descriptors */ - num_transfer = 4; - } - } - - for (i = 0; i < num_transfer; i++) { + if (!rz_ssi_is_dma_enabled(ssi)) { ret = strm->transfer(ssi, strm); if (ret) return ret; @@ -888,6 +727,7 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd, break; case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: rz_ssi_stop(ssi, strm); break; @@ -973,6 +813,8 @@ static void rz_ssi_shutdown(struct snd_pcm_substream *substream, ssi->dup.tx_active = false; else ssi->dup.rx_active = false; + + ssi->dmas[substream->stream] = NULL; } static bool rz_ssi_is_valid_hw_params(struct rz_ssi_priv *ssi, unsigned int rate, @@ -1024,6 +866,12 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream, return -EINVAL; } + /* Save the DMA channels for recovery. */ + if (rz_ssi_is_dma_enabled(ssi)) + ssi->dmas[substream->stream] = snd_dmaengine_pcm_get_chan(substream); + else + ssi->dmas[substream->stream] = NULL; + if (rz_ssi_is_stream_running(&ssi->playback) || rz_ssi_is_stream_running(&ssi->capture)) { if (rz_ssi_is_valid_hw_params(ssi, rate, channels, sample_width, sample_bits)) @@ -1039,10 +887,21 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream, if (ret) return ret; - return rz_ssi_clk_setup(ssi, rate, channels); + return rz_ssi_clk_setup(ssi, substream, rate, channels); +} + +static int rz_ssi_dai_probe(struct snd_soc_dai *dai) +{ + struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); + + snd_soc_dai_init_dma_data(dai, &ssi->dma_dais[SNDRV_PCM_STREAM_PLAYBACK], + &ssi->dma_dais[SNDRV_PCM_STREAM_CAPTURE]); + + return 0; } static const struct snd_soc_dai_ops rz_ssi_dai_ops = { + .probe = rz_ssi_dai_probe, .startup = rz_ssi_startup, .shutdown = rz_ssi_shutdown, .trigger = rz_ssi_dai_trigger, @@ -1054,10 +913,11 @@ static const struct snd_pcm_hardware rz_ssi_pcm_hardware = { .info = SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_RESUME, - .buffer_bytes_max = PREALLOC_BUFFER, + SNDRV_PCM_INFO_RESUME | + SNDRV_PCM_INFO_PAUSE, + .buffer_bytes_max = 192 * 1024, .period_bytes_min = 32, - .period_bytes_max = 8192, + .period_bytes_max = 48 * 1024, .channels_min = SSI_CHAN_MIN, .channels_max = SSI_CHAN_MAX, .periods_min = 1, @@ -1065,8 +925,8 @@ static const struct snd_pcm_hardware rz_ssi_pcm_hardware = { .fifo_size = 32 * 2, }; -static int rz_ssi_pcm_open(struct snd_soc_component *component, - struct snd_pcm_substream *substream) +static int rz_ssi_pcm_open_pio(struct snd_soc_component *component, + struct snd_pcm_substream *substream) { snd_soc_set_runtime_hwparams(substream, &rz_ssi_pcm_hardware); @@ -1074,6 +934,13 @@ static int rz_ssi_pcm_open(struct snd_soc_component *component, SNDRV_PCM_HW_PARAM_PERIODS); } +static int rz_ssi_pcm_open_dma(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + return snd_pcm_hw_constraint_integer(substream->runtime, + SNDRV_PCM_HW_PARAM_PERIODS); +} + static snd_pcm_uframes_t rz_ssi_pcm_pointer(struct snd_soc_component *component, struct snd_pcm_substream *substream) { @@ -1090,7 +957,8 @@ static int rz_ssi_pcm_new(struct snd_soc_component *component, { snd_pcm_set_managed_buffer_all(rtd->pcm, SNDRV_DMA_TYPE_DEV, rtd->card->snd_card->dev, - PREALLOC_BUFFER, PREALLOC_BUFFER_MAX); + rz_ssi_pcm_hardware.buffer_bytes_max, + rz_ssi_pcm_hardware.buffer_bytes_max); return 0; } @@ -1113,16 +981,30 @@ static struct snd_soc_dai_driver rz_ssi_soc_dai[] = { }, }; -static const struct snd_soc_component_driver rz_ssi_soc_component = { +static const struct snd_soc_component_driver rz_ssi_soc_component_pio = { .name = "rz-ssi", - .open = rz_ssi_pcm_open, + .open = rz_ssi_pcm_open_pio, .pointer = rz_ssi_pcm_pointer, .pcm_new = rz_ssi_pcm_new, .legacy_dai_naming = 1, }; +static const struct snd_soc_component_driver rz_ssi_soc_component_dma = { + .name = "rz-ssi", + .open = rz_ssi_pcm_open_dma, + .legacy_dai_naming = 1, +}; + +static const struct snd_dmaengine_pcm_config rz_ssi_dmaengine_pcm_conf = { + .pcm_hardware = &rz_ssi_pcm_hardware, + .prealloc_buffer_size = 192 * 1024, + .prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config, +}; + static int rz_ssi_probe(struct platform_device *pdev) { + const struct snd_soc_component_driver *component_driver; + struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct rz_ssi_priv *ssi; struct clk *audio_clk; @@ -1138,7 +1020,6 @@ static int rz_ssi_probe(struct platform_device *pdev) if (IS_ERR(ssi->base)) return PTR_ERR(ssi->base); - ssi->phys = res->start; ssi->clk = devm_clk_get(dev, "ssi"); if (IS_ERR(ssi->clk)) return PTR_ERR(ssi->clk); @@ -1162,16 +1043,43 @@ static int rz_ssi_probe(struct platform_device *pdev) ssi->audio_mck = ssi->audio_clk_1 ? ssi->audio_clk_1 : ssi->audio_clk_2; - /* Detect DMA support */ - ret = rz_ssi_dma_request(ssi, dev); - if (ret < 0) { + ssi->dma_dais[SNDRV_PCM_STREAM_PLAYBACK].addr = (dma_addr_t)res->start + SSIFTDR; + ssi->dma_dais[SNDRV_PCM_STREAM_CAPTURE].addr = (dma_addr_t)res->start + SSIFRDR; + + if (of_property_present(np, "dma-names")) { + struct snd_dmaengine_pcm_config *config; + unsigned int flags = 0; + + config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL); + if (!config) + return -ENOMEM; + + config->pcm_hardware = rz_ssi_dmaengine_pcm_conf.pcm_hardware; + config->prealloc_buffer_size = rz_ssi_dmaengine_pcm_conf.prealloc_buffer_size; + config->prepare_slave_config = rz_ssi_dmaengine_pcm_conf.prepare_slave_config; + + if (of_property_match_string(np, "dma-names", "rt") == 0) { + flags = SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX; + config->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = "rt"; + } else { + config->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = "tx"; + config->chan_names[SNDRV_PCM_STREAM_CAPTURE] = "rx"; + } + ret = devm_snd_dmaengine_pcm_register(&pdev->dev, config, flags); + } else { + ret = -ENODEV; + } + + if (ret == -EPROBE_DEFER) { + return ret; + } else if (ret) { dev_warn(dev, "DMA not available, using PIO\n"); ssi->playback.transfer = rz_ssi_pio_send; ssi->capture.transfer = rz_ssi_pio_recv; + component_driver = &rz_ssi_soc_component_pio; } else { - dev_info(dev, "DMA enabled"); - ssi->playback.transfer = rz_ssi_dma_transfer; - ssi->capture.transfer = rz_ssi_dma_transfer; + dev_info(dev, "DMA enabled\n"); + component_driver = &rz_ssi_soc_component_dma; } ssi->playback.priv = ssi; @@ -1182,17 +1090,13 @@ static int rz_ssi_probe(struct platform_device *pdev) /* Error Interrupt */ ssi->irq_int = platform_get_irq_byname(pdev, "int_req"); - if (ssi->irq_int < 0) { - ret = ssi->irq_int; - goto err_release_dma_chs; - } + if (ssi->irq_int < 0) + return ssi->irq_int; ret = devm_request_irq(dev, ssi->irq_int, rz_ssi_interrupt, 0, dev_name(dev), ssi); - if (ret < 0) { - dev_err_probe(dev, ret, "irq request error (int_req)\n"); - goto err_release_dma_chs; - } + if (ret < 0) + return dev_err_probe(dev, ret, "irq request error (int_req)\n"); if (!rz_ssi_is_dma_enabled(ssi)) { /* Tx and Rx interrupts (pio only) */ @@ -1233,43 +1137,19 @@ static int rz_ssi_probe(struct platform_device *pdev) } ssi->rstc = devm_reset_control_get_exclusive(dev, NULL); - if (IS_ERR(ssi->rstc)) { - ret = PTR_ERR(ssi->rstc); - goto err_release_dma_chs; - } + if (IS_ERR(ssi->rstc)) + return dev_err_probe(dev, PTR_ERR(ssi->rstc), "Failed to get reset\n"); /* Default 0 for power saving. Can be overridden via sysfs. */ pm_runtime_set_autosuspend_delay(dev, 0); pm_runtime_use_autosuspend(dev); ret = devm_pm_runtime_enable(dev); - if (ret < 0) { - dev_err(dev, "Failed to enable runtime PM!\n"); - goto err_release_dma_chs; - } - - ret = devm_snd_soc_register_component(dev, &rz_ssi_soc_component, - rz_ssi_soc_dai, - ARRAY_SIZE(rz_ssi_soc_dai)); - if (ret < 0) { - dev_err(dev, "failed to register snd component\n"); - goto err_release_dma_chs; - } - - return 0; - -err_release_dma_chs: - rz_ssi_release_dma_channels(ssi); - - return ret; -} - -static void rz_ssi_remove(struct platform_device *pdev) -{ - struct rz_ssi_priv *ssi = dev_get_drvdata(&pdev->dev); - - rz_ssi_release_dma_channels(ssi); + if (ret < 0) + return dev_err_probe(dev, ret, "Failed to enable runtime PM!\n"); - reset_control_assert(ssi->rstc); + return devm_snd_soc_register_component(dev, component_driver, + rz_ssi_soc_dai, + ARRAY_SIZE(rz_ssi_soc_dai)); } static const struct of_device_id rz_ssi_of_match[] = { @@ -1304,7 +1184,6 @@ static struct platform_driver rz_ssi_driver = { .pm = pm_ptr(&rz_ssi_pm_ops), }, .probe = rz_ssi_probe, - .remove = rz_ssi_remove, }; module_platform_driver(rz_ssi_driver); diff --git a/sound/soc/rockchip/rockchip_sai.c b/sound/soc/rockchip/rockchip_sai.c index a195e96fed0a..585e89f61f0d 100644 --- a/sound/soc/rockchip/rockchip_sai.c +++ b/sound/soc/rockchip/rockchip_sai.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include + #include "rockchip_sai.h" #define DRV_NAME "rockchip-sai" @@ -215,12 +217,14 @@ static void rockchip_sai_xfer_clk_stop_and_wait(struct rk_sai_dev *sai, unsigned static int rockchip_sai_runtime_suspend(struct device *dev) { struct rk_sai_dev *sai = dev_get_drvdata(dev); + unsigned long flags; rockchip_sai_fsync_lost_detect(sai, 0); rockchip_sai_fsync_err_detect(sai, 0); - scoped_guard(spinlock_irqsave, &sai->xfer_lock) - rockchip_sai_xfer_clk_stop_and_wait(sai, NULL); + spin_lock_irqsave(&sai->xfer_lock, flags); + rockchip_sai_xfer_clk_stop_and_wait(sai, NULL); + spin_unlock_irqrestore(&sai->xfer_lock, flags); regcache_cache_only(sai->regmap, true); /* @@ -480,6 +484,7 @@ static int rockchip_sai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) struct rk_sai_dev *sai = snd_soc_dai_get_drvdata(dai); unsigned int mask = 0, val = 0; unsigned int clk_gates; + unsigned long flags; int ret = 0; pm_runtime_get_sync(dai->dev); @@ -495,56 +500,56 @@ static int rockchip_sai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) sai->is_master_mode = false; break; default: - pm_runtime_put(dai->dev); - return -EINVAL; + ret = -EINVAL; + goto err_pm_put; } - scoped_guard(spinlock_irqsave, &sai->xfer_lock) { - rockchip_sai_xfer_clk_stop_and_wait(sai, &clk_gates); - if (sai->initialized) { - if (sai->has_capture && sai->has_playback) - rockchip_sai_xfer_stop(sai, -1); - else if (sai->has_capture) - rockchip_sai_xfer_stop(sai, SNDRV_PCM_STREAM_CAPTURE); - else - rockchip_sai_xfer_stop(sai, SNDRV_PCM_STREAM_PLAYBACK); - } else { - rockchip_sai_clear(sai, 0); - sai->initialized = true; - } + spin_lock_irqsave(&sai->xfer_lock, flags); + rockchip_sai_xfer_clk_stop_and_wait(sai, &clk_gates); + if (sai->initialized) { + if (sai->has_capture && sai->has_playback) + rockchip_sai_xfer_stop(sai, -1); + else if (sai->has_capture) + rockchip_sai_xfer_stop(sai, SNDRV_PCM_STREAM_CAPTURE); + else + rockchip_sai_xfer_stop(sai, SNDRV_PCM_STREAM_PLAYBACK); + } else { + rockchip_sai_clear(sai, 0); + sai->initialized = true; + } - regmap_update_bits(sai->regmap, SAI_CKR, mask, val); - - mask = SAI_CKR_CKP_MASK | SAI_CKR_FSP_MASK; - switch (fmt & SND_SOC_DAIFMT_INV_MASK) { - case SND_SOC_DAIFMT_NB_NF: - val = SAI_CKR_CKP_NORMAL | SAI_CKR_FSP_NORMAL; - break; - case SND_SOC_DAIFMT_NB_IF: - val = SAI_CKR_CKP_NORMAL | SAI_CKR_FSP_INVERTED; - break; - case SND_SOC_DAIFMT_IB_NF: - val = SAI_CKR_CKP_INVERTED | SAI_CKR_FSP_NORMAL; - break; - case SND_SOC_DAIFMT_IB_IF: - val = SAI_CKR_CKP_INVERTED | SAI_CKR_FSP_INVERTED; - break; - default: - ret = -EINVAL; - break; - } + regmap_update_bits(sai->regmap, SAI_CKR, mask, val); - if (ret == 0) { - regmap_update_bits(sai->regmap, SAI_CKR, mask, val); - rockchip_sai_fmt_create(sai, fmt); - } - - if (clk_gates) - regmap_update_bits(sai->regmap, SAI_XFER, - SAI_XFER_CLK_MASK | SAI_XFER_FSS_MASK, - clk_gates); + mask = SAI_CKR_CKP_MASK | SAI_CKR_FSP_MASK; + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_NB_NF: + val = SAI_CKR_CKP_NORMAL | SAI_CKR_FSP_NORMAL; + break; + case SND_SOC_DAIFMT_NB_IF: + val = SAI_CKR_CKP_NORMAL | SAI_CKR_FSP_INVERTED; + break; + case SND_SOC_DAIFMT_IB_NF: + val = SAI_CKR_CKP_INVERTED | SAI_CKR_FSP_NORMAL; + break; + case SND_SOC_DAIFMT_IB_IF: + val = SAI_CKR_CKP_INVERTED | SAI_CKR_FSP_INVERTED; + break; + default: + ret = -EINVAL; + goto err_xfer_unlock; } + regmap_update_bits(sai->regmap, SAI_CKR, mask, val); + + rockchip_sai_fmt_create(sai, fmt); + +err_xfer_unlock: + if (clk_gates) + regmap_update_bits(sai->regmap, SAI_XFER, + SAI_XFER_CLK_MASK | SAI_XFER_FSS_MASK, + clk_gates); + spin_unlock_irqrestore(&sai->xfer_lock, flags); +err_pm_put: pm_runtime_put(dai->dev); return ret; @@ -560,6 +565,7 @@ static int rockchip_sai_hw_params(struct snd_pcm_substream *substream, unsigned int ch_per_lane, slot_width; unsigned int val, fscr, reg; unsigned int lanes, req_lanes; + unsigned long flags; int ret = 0; if (!rockchip_sai_stream_valid(substream, dai)) @@ -586,8 +592,8 @@ static int rockchip_sai_hw_params(struct snd_pcm_substream *substream, dev_err(sai->dev, "not enough lanes (%d) for requested number of %s channels (%d)\n", lanes, reg == SAI_TXCR ? "playback" : "capture", params_channels(params)); - pm_runtime_put(sai->dev); - return -EINVAL; + ret = -EINVAL; + goto err_pm_put; } else { lanes = req_lanes; } @@ -613,88 +619,84 @@ static int rockchip_sai_hw_params(struct snd_pcm_substream *substream, val = SAI_XCR_VDW(32); break; default: - pm_runtime_put(sai->dev); - return -EINVAL; + ret = -EINVAL; + goto err_pm_put; } val |= SAI_XCR_CSR(lanes); - scoped_guard(spinlock_irqsave, &sai->xfer_lock) { + spin_lock_irqsave(&sai->xfer_lock, flags); + + regmap_update_bits(sai->regmap, reg, SAI_XCR_VDW_MASK | SAI_XCR_CSR_MASK, val); - regmap_update_bits(sai->regmap, reg, SAI_XCR_VDW_MASK | SAI_XCR_CSR_MASK, val); + if (!sai->is_tdm) + regmap_update_bits(sai->regmap, reg, SAI_XCR_SBW_MASK, + SAI_XCR_SBW(params_physical_width(params))); - if (!sai->is_tdm) - regmap_update_bits(sai->regmap, reg, SAI_XCR_SBW_MASK, - SAI_XCR_SBW(params_physical_width(params))); + regmap_read(sai->regmap, reg, &val); - regmap_read(sai->regmap, reg, &val); + slot_width = SAI_XCR_SBW_V(val); + ch_per_lane = params_channels(params) / lanes; - slot_width = SAI_XCR_SBW_V(val); - ch_per_lane = params_channels(params) / lanes; + regmap_update_bits(sai->regmap, reg, SAI_XCR_SNB_MASK, + SAI_XCR_SNB(ch_per_lane)); - regmap_update_bits(sai->regmap, reg, SAI_XCR_SNB_MASK, - SAI_XCR_SNB(ch_per_lane)); + fscr = SAI_FSCR_FW(sai->fw_ratio * slot_width * ch_per_lane); - fscr = SAI_FSCR_FW(sai->fw_ratio * slot_width * ch_per_lane); + switch (sai->fpw) { + case FPW_ONE_BCLK_WIDTH: + fscr |= SAI_FSCR_FPW(1); + break; + case FPW_ONE_SLOT_WIDTH: + fscr |= SAI_FSCR_FPW(slot_width); + break; + case FPW_HALF_FRAME_WIDTH: + fscr |= SAI_FSCR_FPW(sai->fw_ratio * slot_width * ch_per_lane / 2); + break; + default: + dev_err(sai->dev, "Invalid Frame Pulse Width %d\n", sai->fpw); + ret = -EINVAL; + goto err_xfer_unlock; + } - switch (sai->fpw) { - case FPW_ONE_BCLK_WIDTH: - fscr |= SAI_FSCR_FPW(1); - break; - case FPW_ONE_SLOT_WIDTH: - fscr |= SAI_FSCR_FPW(slot_width); - break; - case FPW_HALF_FRAME_WIDTH: - fscr |= SAI_FSCR_FPW(sai->fw_ratio * slot_width * ch_per_lane / 2); - break; - default: - dev_err(sai->dev, "Invalid Frame Pulse Width %d\n", sai->fpw); + regmap_update_bits(sai->regmap, SAI_FSCR, + SAI_FSCR_FW_MASK | SAI_FSCR_FPW_MASK, fscr); + + if (sai->is_master_mode) { + bclk_rate = sai->fw_ratio * slot_width * ch_per_lane * params_rate(params); + ret = clk_set_rate(sai->mclk, sai->mclk_rate); + if (ret) { + dev_err(sai->dev, "Failed to set mclk to %u: %pe\n", + sai->mclk_rate, ERR_PTR(ret)); + goto err_xfer_unlock; + } + + mclk_rate = clk_get_rate(sai->mclk); + if (mclk_rate < bclk_rate) { + dev_err(sai->dev, "Mismatch mclk: %u, at least %u\n", + mclk_rate, bclk_rate); ret = -EINVAL; - break; + goto err_xfer_unlock; } - if (ret == 0) { - regmap_update_bits(sai->regmap, SAI_FSCR, - SAI_FSCR_FW_MASK | SAI_FSCR_FPW_MASK, fscr); - - if (sai->is_master_mode) { - bclk_rate = sai->fw_ratio * slot_width * - ch_per_lane * params_rate(params); - ret = clk_set_rate(sai->mclk, sai->mclk_rate); - if (ret) - dev_err(sai->dev, "Failed to set mclk to %u: %pe\n", - sai->mclk_rate, ERR_PTR(ret)); - else { - mclk_rate = clk_get_rate(sai->mclk); - if (mclk_rate < bclk_rate) { - dev_err(sai->dev, "Mismatch mclk: %u, at least %u\n", - mclk_rate, bclk_rate); - ret = -EINVAL; - } else { - - div_bclk = DIV_ROUND_CLOSEST(mclk_rate, bclk_rate); - mclk_req_rate = bclk_rate * div_bclk; - - if (mclk_rate < - mclk_req_rate - CLK_SHIFT_RATE_HZ_MAX || - mclk_rate > - mclk_req_rate + CLK_SHIFT_RATE_HZ_MAX) { - dev_err(sai->dev, - "Mismatch mclk: %u, expected %u (+/- %dHz)\n", - mclk_rate, mclk_req_rate, - CLK_SHIFT_RATE_HZ_MAX); - ret = -EINVAL; - } else - regmap_update_bits(sai->regmap, - SAI_CKR, - SAI_CKR_MDIV_MASK, - SAI_CKR_MDIV(div_bclk)); - } - } - } + div_bclk = DIV_ROUND_CLOSEST(mclk_rate, bclk_rate); + mclk_req_rate = bclk_rate * div_bclk; + + if (mclk_rate < mclk_req_rate - CLK_SHIFT_RATE_HZ_MAX || + mclk_rate > mclk_req_rate + CLK_SHIFT_RATE_HZ_MAX) { + dev_err(sai->dev, "Mismatch mclk: %u, expected %u (+/- %dHz)\n", + mclk_rate, mclk_req_rate, CLK_SHIFT_RATE_HZ_MAX); + ret = -EINVAL; + goto err_xfer_unlock; } + + regmap_update_bits(sai->regmap, SAI_CKR, SAI_CKR_MDIV_MASK, + SAI_CKR_MDIV(div_bclk)); } +err_xfer_unlock: + spin_unlock_irqrestore(&sai->xfer_lock, flags); +err_pm_put: pm_runtime_put(sai->dev); return ret; @@ -704,6 +706,7 @@ static int rockchip_sai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct rk_sai_dev *sai = snd_soc_dai_get_drvdata(dai); + unsigned long flags; if (!rockchip_sai_stream_valid(substream, dai)) return 0; @@ -724,12 +727,13 @@ static int rockchip_sai_prepare(struct snd_pcm_substream *substream, * udelay falls short. */ udelay(20); - scoped_guard(spinlock_irqsave, &sai->xfer_lock) - regmap_update_bits(sai->regmap, SAI_XFER, - SAI_XFER_CLK_MASK | - SAI_XFER_FSS_MASK, - SAI_XFER_CLK_EN | - SAI_XFER_FSS_EN); + spin_lock_irqsave(&sai->xfer_lock, flags); + regmap_update_bits(sai->regmap, SAI_XFER, + SAI_XFER_CLK_MASK | + SAI_XFER_FSS_MASK, + SAI_XFER_CLK_EN | + SAI_XFER_FSS_EN); + spin_unlock_irqrestore(&sai->xfer_lock, flags); } rockchip_sai_fsync_lost_detect(sai, 1); @@ -912,6 +916,7 @@ static int rockchip_sai_set_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct rk_sai_dev *sai = snd_soc_dai_get_drvdata(dai); + unsigned long flags; unsigned int clk_gates; int sw = slot_width; @@ -927,16 +932,16 @@ static int rockchip_sai_set_tdm_slot(struct snd_soc_dai *dai, return -EINVAL; pm_runtime_get_sync(dai->dev); - scoped_guard(spinlock_irqsave, &sai->xfer_lock) { - rockchip_sai_xfer_clk_stop_and_wait(sai, &clk_gates); - regmap_update_bits(sai->regmap, SAI_TXCR, SAI_XCR_SBW_MASK, - SAI_XCR_SBW(sw)); - regmap_update_bits(sai->regmap, SAI_RXCR, SAI_XCR_SBW_MASK, - SAI_XCR_SBW(sw)); - regmap_update_bits(sai->regmap, SAI_XFER, - SAI_XFER_CLK_MASK | SAI_XFER_FSS_MASK, - clk_gates); - } + spin_lock_irqsave(&sai->xfer_lock, flags); + rockchip_sai_xfer_clk_stop_and_wait(sai, &clk_gates); + regmap_update_bits(sai->regmap, SAI_TXCR, SAI_XCR_SBW_MASK, + SAI_XCR_SBW(sw)); + regmap_update_bits(sai->regmap, SAI_RXCR, SAI_XCR_SBW_MASK, + SAI_XCR_SBW(sw)); + regmap_update_bits(sai->regmap, SAI_XFER, + SAI_XFER_CLK_MASK | SAI_XFER_FSS_MASK, + clk_gates); + spin_unlock_irqrestore(&sai->xfer_lock, flags); pm_runtime_put(dai->dev); return 0; diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c index 48ccc1d1854b..6db91b73f25c 100644 --- a/sound/soc/samsung/aries_wm8994.c +++ b/sound/soc/samsung/aries_wm8994.c @@ -658,6 +658,7 @@ static int aries_audio_probe(struct platform_device *pdev) goto out; } + of_node_get(aries_dai[0].cpus->of_node); aries_dai[0].platforms->of_node = aries_dai[0].cpus->of_node; /* Set CPU of_node for BT DAI */ diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index f80f697a5d55..56d11741dabd 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -512,7 +513,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, u32 mod, mask, val = 0; int ret = 0; - pm_runtime_get_sync(dai->dev); + guard(pm_runtime_active)(dai->dev); scoped_guard(spinlock_irqsave, &priv->lock) mod = readl(priv->addr + I2SMOD); @@ -537,8 +538,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, && (mod & cdcon_mask))))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } if (dir == SND_SOC_CLOCK_IN) @@ -566,7 +566,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, } else { priv->rclk_srcrate = clk_get_rate(priv->op_clk); - goto done; + return 0; } } @@ -580,14 +580,14 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, if (WARN_ON(IS_ERR(priv->op_clk))) { ret = PTR_ERR(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } ret = clk_prepare_enable(priv->op_clk); if (ret) { clk_put(priv->op_clk); priv->op_clk = NULL; - goto err; + return ret; } priv->rclk_srcrate = clk_get_rate(priv->op_clk); @@ -595,11 +595,10 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, || (clk_id && !(mod & rsrc_mask))) { dev_err(&i2s->pdev->dev, "%s:%d Other DAI busy\n", __func__, __LINE__); - ret = -EAGAIN; - goto err; + return -EAGAIN; } else { /* Call can't be on the active DAI */ - goto done; + return 0; } if (clk_id == 1) @@ -607,8 +606,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, break; default: dev_err(&i2s->pdev->dev, "We don't serve that!\n"); - ret = -EINVAL; - goto err; + return -EINVAL; } scoped_guard(spinlock_irqsave, &priv->lock) { @@ -616,13 +614,8 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs, mod = (mod & ~mask) | val; writel(mod, priv->addr + I2SMOD); } -done: - pm_runtime_put(dai->dev); return 0; -err: - pm_runtime_put(dai->dev); - return ret; } static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) @@ -1448,10 +1441,10 @@ static int samsung_i2s_probe(struct platform_device *pdev) regs_base = res->start; priv->clk = devm_clk_get(&pdev->dev, "iis"); - if (IS_ERR(priv->clk)) { - dev_err(&pdev->dev, "Failed to get iis clock\n"); - return PTR_ERR(priv->clk); - } + if (IS_ERR(priv->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk), + "Failed to get iis clock\n"); + ret = clk_prepare_enable(priv->clk); if (ret != 0) { diff --git a/sound/soc/samsung/spdif.c b/sound/soc/samsung/spdif.c index 7fc46d55c522..53eaabaf8956 100644 --- a/sound/soc/samsung/spdif.c +++ b/sound/soc/samsung/spdif.c @@ -380,8 +380,8 @@ static int spdif_probe(struct platform_device *pdev) spdif->pclk = devm_clk_get(&pdev->dev, "spdif"); if (IS_ERR(spdif->pclk)) { - dev_err(&pdev->dev, "failed to get peri-clock\n"); - ret = -ENOENT; + ret = dev_err_probe(&pdev->dev, PTR_ERR(spdif->pclk), + "failed to get peri-clock\n"); goto err0; } ret = clk_prepare_enable(spdif->pclk); @@ -390,8 +390,8 @@ static int spdif_probe(struct platform_device *pdev) spdif->sclk = devm_clk_get(&pdev->dev, "sclk_spdif"); if (IS_ERR(spdif->sclk)) { - dev_err(&pdev->dev, "failed to get internal source clock\n"); - ret = -ENOENT; + ret = dev_err_probe(&pdev->dev, PTR_ERR(spdif->sclk), + "failed to get internal source clock\n"); goto err1; } ret = clk_prepare_enable(spdif->sclk); diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index e76afa396b0a..b4dedba719dc 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -160,6 +160,9 @@ static int ge_put_enum_double(struct snd_kcontrol *kcontrol, unsigned int reg = e->reg; int ret; + if (item[0] >= e->items) + return -EINVAL; + reg &= ~SDW_SDCA_CTL_CSEL(0x3F); reg |= SDW_SDCA_CTL_CSEL(SDCA_CTL_GE_DETECTED_MODE); diff --git a/sound/soc/sdca/sdca_class.c b/sound/soc/sdca/sdca_class.c index 6937a91ddfb9..8d7b007a068f 100644 --- a/sound/soc/sdca/sdca_class.c +++ b/sound/soc/sdca/sdca_class.c @@ -9,7 +9,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c index e0e32a279787..129a437ae397 100644 --- a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c +++ b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c @@ -99,11 +99,9 @@ static const struct snd_soc_dai_link bridge_dai_template = { SND_SOC_DAILINK_REG(asoc_sdw_bridge_dai), }; -int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card, +int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx, int *num_dais, int *num_devs) { - struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card); - if (ctx->mc_quirk & SOC_SDW_SIDECAR_AMPS) { (*num_dais)++; (*num_devs) += ARRAY_SIZE(bridge_cs35l56_name_prefixes); diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c index d8db8fc5313e..dd2cc57059d6 100644 --- a/sound/soc/sdw_utils/soc_sdw_utils.c +++ b/sound/soc/sdw_utils/soc_sdw_utils.c @@ -1976,14 +1976,13 @@ static int is_sdca_endpoint_present(struct device *dev, return ret; } -int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, +int asoc_sdw_parse_sdw_endpoints(struct device *dev, + struct asoc_sdw_mc_private *ctx, struct snd_soc_aux_dev *soc_aux, struct asoc_sdw_dailink *soc_dais, struct asoc_sdw_endpoint *soc_ends, int *num_devs) { - struct device *dev = card->dev; - struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card); struct snd_soc_acpi_mach *mach = dev_get_platdata(dev); struct snd_soc_acpi_mach_params *mach_params = &mach->mach_params; const struct snd_soc_acpi_link_adr *adr_link; @@ -2045,7 +2044,7 @@ int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card, ctx->ignore_internal_dmic |= codec_info->ignore_internal_dmic; if (codec_info->count_sidecar && codec_info->add_sidecar) { - ret = codec_info->count_sidecar(card, &num_dais, num_devs); + ret = codec_info->count_sidecar(ctx, &num_dais, num_devs); if (ret) return ret; diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index 86b6c752a56b..7817beea5b3b 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -1980,19 +1980,15 @@ static void soc_cleanup_card_resources(struct snd_soc_card *card) } } -static void snd_soc_remove_device_links(struct snd_soc_card *card, - struct snd_soc_component *stop_at) +static void snd_soc_remove_device_links(struct snd_soc_card *card) { struct snd_soc_component *component; for_each_card_components(card, component) { - if (card->dev == component->dev) - continue; - - device_link_remove(card->dev, component->dev); - - if (component == stop_at) - return; + if (component->card_device_link) { + device_link_del(component->card_device_link); + component->card_device_link = NULL; + } } } @@ -2001,7 +1997,7 @@ static void snd_soc_unbind_card(struct snd_soc_card *card) if (snd_soc_card_is_instantiated(card)) { card->instantiated = false; - snd_soc_remove_device_links(card, NULL); + snd_soc_remove_device_links(card); soc_cleanup_card_resources(card); } @@ -2012,7 +2008,6 @@ static int snd_soc_bind_card(struct snd_soc_card *card) struct snd_soc_pcm_runtime *rtd; struct snd_soc_component *component; struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card); - struct snd_soc_component *last_devlinked_component = NULL; int ret; snd_soc_card_mutex_lock_root(card); @@ -2141,19 +2136,21 @@ static int snd_soc_bind_card(struct snd_soc_card *card) * Add device_link from card to component so that system_suspend * will be done in the correct order. The card must suspend first * to stop audio activity before the components suspend. + * + * If a driver pair already have a link in the opposite direction + * they must manage their own suspend order. */ for_each_card_components(card, component) { if (card->dev == component->dev) continue; - if (!device_link_add(card->dev, component->dev, DL_FLAG_STATELESS)) { - dev_warn(card->dev, "Failed to create device link to %s\n", + component->card_device_link = device_link_add(card->dev, + component->dev, + DL_FLAG_STATELESS); + if (!component->card_device_link) { + dev_warn(card->dev, "Could not create device link to %s\n", dev_name(component->dev)); - ret = -EINVAL; - goto probe_end; } - - last_devlinked_component = component; } ret = snd_soc_card_late_probe(card); @@ -2185,9 +2182,7 @@ static int snd_soc_bind_card(struct snd_soc_card *card) probe_end: if (ret < 0) { - if (last_devlinked_component) - snd_soc_remove_device_links(card, last_devlinked_component); - + snd_soc_remove_device_links(card); soc_cleanup_card_resources(card); } diff --git a/sound/soc/sof/amd/Kconfig b/sound/soc/sof/amd/Kconfig index 05faf1c6d6fc..7413bfdb0943 100644 --- a/sound/soc/sof/amd/Kconfig +++ b/sound/soc/sof/amd/Kconfig @@ -103,5 +103,15 @@ config SND_SOC_SOF_AMD_ACP70 Select this option for SOF support on AMD ACP7.0/ACP7.1 version based platforms. Say Y if you want to enable SOF on ACP7.0/ACP7.1 based platforms. +config SND_SOC_SOF_AMD_ACP7X + tristate "SOF support for ACP7.B/7.F platforms" + depends on SND_SOC_SOF_PCI + depends on AMD_NODE + select SND_SOC_SOF_AMD_COMMON + help + Select this option for SOF support on AMD ACP7.B and ACP7.F PCI + revision based platforms. + Say Y if you want to enable SOF on ACP7.B/7.F based platforms. + If unsure select "N". endif diff --git a/sound/soc/sof/amd/Makefile b/sound/soc/sof/amd/Makefile index 6ae39fd5a836..bb1c0ddef69d 100644 --- a/sound/soc/sof/amd/Makefile +++ b/sound/soc/sof/amd/Makefile @@ -11,6 +11,7 @@ snd-sof-amd-rembrandt-y := pci-rmb.o rembrandt.o snd-sof-amd-vangogh-y := pci-vangogh.o vangogh.o snd-sof-amd-acp63-y := pci-acp63.o acp63.o snd-sof-amd-acp70-y := pci-acp70.o acp70.o +snd-sof-amd-acp7x-y := pci-acp7x.o acp7x.o obj-$(CONFIG_SND_SOC_SOF_AMD_COMMON) += snd-sof-amd-acp.o obj-$(CONFIG_SND_SOC_SOF_AMD_RENOIR) += snd-sof-amd-renoir.o @@ -18,3 +19,4 @@ obj-$(CONFIG_SND_SOC_SOF_AMD_REMBRANDT) += snd-sof-amd-rembrandt.o obj-$(CONFIG_SND_SOC_SOF_AMD_VANGOGH) += snd-sof-amd-vangogh.o obj-$(CONFIG_SND_SOC_SOF_AMD_ACP63) += snd-sof-amd-acp63.o obj-$(CONFIG_SND_SOC_SOF_AMD_ACP70) += snd-sof-amd-acp70.o +obj-$(CONFIG_SND_SOC_SOF_AMD_ACP7X) += snd-sof-amd-acp7x.o diff --git a/sound/soc/sof/amd/acp-dsp-offset.h b/sound/soc/sof/amd/acp-dsp-offset.h index 08583a91afbc..bea1bd3afa70 100644 --- a/sound/soc/sof/amd/acp-dsp-offset.h +++ b/sound/soc/sof/amd/acp-dsp-offset.h @@ -68,12 +68,14 @@ #define ACP5X_PGFSM_BASE 0x1424 #define ACP6X_PGFSM_BASE 0x1024 #define ACP70_PGFSM_BASE ACP6X_PGFSM_BASE +#define ACP7X_PGFSM_BASE ACP6X_PGFSM_BASE #define PGFSM_CONTROL_OFFSET 0x0 #define PGFSM_STATUS_OFFSET 0x4 #define ACP3X_CLKMUX_SEL 0x1424 #define ACP5X_CLKMUX_SEL 0x142C #define ACP6X_CLKMUX_SEL 0x102C #define ACP70_CLKMUX_SEL ACP6X_CLKMUX_SEL +#define ACP7X_CLKMUX_SEL ACP6X_CLKMUX_SEL /* Registers from ACP_INTR block */ #define ACP3X_EXT_INTR_STAT 0x1808 @@ -86,21 +88,31 @@ #define ACP70_EXTERNAL_INTR_CNTL ACP6X_EXTERNAL_INTR_CNTL #define ACP70_EXT_INTR_STAT ACP6X_EXT_INTR_STAT #define ACP70_EXT_INTR_STAT1 ACP6X_EXT_INTR_STAT1 +#define ACP7X_EXTERNAL_INTR_ENB ACP6X_EXTERNAL_INTR_ENB +#define ACP7X_EXTERNAL_INTR_CNTL 0x1A04 +#define ACP7X_EXT_INTR_STAT 0x1A1C +#define ACP7X_EXTERNAL_INTR_CNTL1 0x1A08 +#define ACP7X_EXT_INTR_STAT1 0x1A20 #define ACP3X_DSP_SW_INTR_BASE 0x1814 #define ACP5X_DSP_SW_INTR_BASE 0x1814 #define ACP6X_DSP_SW_INTR_BASE 0x1808 #define ACP70_DSP_SW_INTR_BASE ACP6X_DSP_SW_INTR_BASE +#define ACP7X_DSP_SW_INTR_BASE 0x1860 #define DSP_SW_INTR_CNTL_OFFSET 0x0 #define DSP_SW_INTR_STAT_OFFSET 0x4 +#define ACP7X_DSP_SW_INTR_STAT (ACP7X_DSP_SW_INTR_BASE + DSP_SW_INTR_STAT_OFFSET) #define DSP_SW_INTR_TRIG_OFFSET 0x8 +#define ACP7X_DSP_SW_INTR_TRIG_OFFSET 0x30 #define ACP3X_ERROR_STATUS 0x18C4 #define ACP6X_ERROR_STATUS 0x1A4C #define ACP70_ERROR_STATUS ACP6X_ERROR_STATUS +#define ACP7X_ERROR_STATUS 0x1A88 #define ACP3X_AXI2DAGB_SEM_0 0x1880 #define ACP5X_AXI2DAGB_SEM_0 0x1884 #define ACP6X_AXI2DAGB_SEM_0 0x1874 #define ACP70_AXI2DAGB_SEM_0 ACP6X_AXI2DAGB_SEM_0 +#define ACP7X_AXI2DAGB_SEM_0 0x18F4 /* ACP common registers to report errors related to I2S & SoundWire interfaces */ #define ACP3X_SW_I2S_ERROR_REASON 0x18C8 @@ -123,6 +135,7 @@ #define ACP_SCRATCH_REG_0 0x10000 #define ACP6X_DSP_FUSION_RUNSTALL 0x0644 #define ACP70_DSP_FUSION_RUNSTALL ACP6X_DSP_FUSION_RUNSTALL +#define ACP7X_DSP_FUSION_RUNSTALL ACP6X_DSP_FUSION_RUNSTALL /* Cache window registers */ #define ACP_DSP0_CACHE_OFFSET0 0x0420 @@ -139,5 +152,9 @@ #define ACP70_SDW1_HOST_WAKE_STAT BIT(25) #define ACP70_SDW0_PME_STAT BIT(26) #define ACP70_SDW1_PME_STAT BIT(27) +#define ACP7X_DSP0_IDMA_ERROR_MASK 0x4B0 +#define ACP7X_IDMA_ERROR_MASK 0x1FF9FF +#define ACP7X_ZSC_DSP_CTRL 0x001014 +#define ACP7X_PME_EN ACP70_PME_EN #endif diff --git a/sound/soc/sof/amd/acp-ipc.c b/sound/soc/sof/amd/acp-ipc.c index 94025bc799ea..494f9cb0c06f 100644 --- a/sound/soc/sof/amd/acp-ipc.c +++ b/sound/soc/sof/amd/acp-ipc.c @@ -32,11 +32,20 @@ static void acpbus_trigger_host_to_dsp_swintr(struct acp_dev_data *adata) struct snd_sof_dev *sdev = adata->dev; const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); u32 swintr_trigger; - + unsigned int swintr_trigger_reg_offset; + + switch (adata->pci_rev) { + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + swintr_trigger_reg_offset = ACP7X_DSP_SW_INTR_TRIG_OFFSET; + break; + default: + swintr_trigger_reg_offset = DSP_SW_INTR_TRIG_OFFSET; + } swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->dsp_intr_base + - DSP_SW_INTR_TRIG_OFFSET); + swintr_trigger_reg_offset); swintr_trigger |= 0x01; - snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_TRIG_OFFSET, + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + swintr_trigger_reg_offset, swintr_trigger); } diff --git a/sound/soc/sof/amd/acp-loader.c b/sound/soc/sof/amd/acp-loader.c index 98324bbade15..f1ec85694323 100644 --- a/sound/soc/sof/amd/acp-loader.c +++ b/sound/soc/sof/amd/acp-loader.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "../ops.h" #include "acp-dsp-offset.h" @@ -173,10 +174,31 @@ int acp_dsp_pre_fw_run(struct snd_sof_dev *sdev) adata = sdev->pdata->hw_pdata; - if (adata->quirks && adata->quirks->signed_fw_image) + if (adata->pci_rev >= ACP7B_PCI_ID) { + if (adata->acp_sof_signed_firmware_image) { + if (adata->fw_bin_size <= ACP_IMAGE_HEADER_SIZE) { + dev_err(sdev->dev, "Invalid signed firmware size %u\n", + adata->fw_bin_size); + return -EINVAL; + } + size_fw = get_unaligned_le32(adata->bin_buf + + ACP_IMAGE_HDR_SIZE_FW_SIGNED_OFF); + if (!size_fw || + size_fw > adata->fw_bin_size - ACP_IMAGE_HEADER_SIZE) { + dev_err(sdev->dev, + "Invalid signed firmware payload size %u (max %u)\n", + size_fw, adata->fw_bin_size - ACP_IMAGE_HEADER_SIZE); + return -EINVAL; + } + size_fw += ACP_IMAGE_HEADER_SIZE; + } else { + size_fw = adata->fw_bin_size; + } + } else if (adata->quirks && adata->quirks->signed_fw_image) { size_fw = adata->fw_bin_size - ACP_FIRMWARE_SIGNATURE; - else + } else { size_fw = adata->fw_bin_size; + } page_count = PAGE_ALIGN(size_fw) >> PAGE_SHIFT; adata->fw_bin_page_count = page_count; @@ -312,9 +334,14 @@ int acp_sof_load_signed_firmware(struct snd_sof_dev *sdev) } kfree(fw_filename); - ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_DRAM, 0, - (void *)adata->fw_dbin->data, - adata->fw_dbin->size); + if (adata->pci_rev >= ACP7B_PCI_ID) + ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_SRAM, 0, + (void *)adata->fw_dbin->data, + adata->fw_dbin->size); + else + ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_DRAM, 0, + (void *)adata->fw_dbin->data, + adata->fw_dbin->size); return ret; } EXPORT_SYMBOL_NS(acp_sof_load_signed_firmware, "SND_SOC_SOF_AMD_COMMON"); diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c index e6af8927baa0..f89ad86260b4 100644 --- a/sound/soc/sof/amd/acp.c +++ b/sound/soc/sof/amd/acp.c @@ -12,6 +12,7 @@ * Hardware interface for generic AMD ACP processor */ +#include #include #include #include @@ -60,6 +61,8 @@ static void init_dma_descriptor(struct acp_dev_data *adata) case ACP70_PCI_ID: case ACP71_PCI_ID: case ACP72_PCI_ID: + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: acp_dma_desc_base_addr = ACP70_DMA_DESC_BASE_ADDR; acp_dma_desc_max_num_dscr = ACP70_DMA_DESC_MAX_NUM_DSCR; break; @@ -101,6 +104,8 @@ static int config_dma_channel(struct acp_dev_data *adata, unsigned int ch, case ACP70_PCI_ID: case ACP71_PCI_ID: case ACP72_PCI_ID: + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: acp_dma_cntl_0 = ACP70_DMA_CNTL_0; acp_dma_ch_rst_sts = ACP70_DMA_CH_RST_STS; acp_dma_dscr_err_sts_0 = ACP70_DMA_ERR_STS_0; @@ -283,7 +288,8 @@ int configure_and_run_sha_dma(struct acp_dev_data *adata, void *image_addr, } } - if (adata->quirks && adata->quirks->signed_fw_image) + if ((adata->quirks && adata->quirks->signed_fw_image) || + adata->acp_sof_signed_firmware_image) snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SHA_DMA_INCLUDE_HDR, ACP_SHA_HEADER); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SHA_DMA_STRT_ADDR, start_addr); @@ -342,6 +348,8 @@ int acp_dma_status(struct acp_dev_data *adata, unsigned char ch) case ACP70_PCI_ID: case ACP71_PCI_ID: case ACP72_PCI_ID: + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: acp_dma_ch_sts = ACP70_DMA_CH_STS; break; default: @@ -534,8 +542,17 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id) snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, ACP_ERROR_IRQ_MASK); snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_sw0_i2s_err_reason, 0); /* ACP_SW1_I2S_ERROR_REASON is newly added register from rmb platform onwards */ - if (adata->pci_rev >= ACP_RMB_PCI_ID) + switch (adata->pci_rev) { + case ACP_RMB_PCI_ID: + case ACP63_PCI_ID: + case ACP70_PCI_ID: + case ACP71_PCI_ID: + case ACP72_PCI_ID: snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SW1_I2S_ERROR_REASON, 0); + break; + default: + break; + } snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_error_stat, 0); irq_flag = 1; } @@ -564,6 +581,35 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id) return IRQ_NONE; } +static irqreturn_t acp7x_irq_handler(int irq, void *dev_id) +{ + struct snd_sof_dev *sdev = dev_id; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int base = desc->dsp_intr_base; + unsigned int val; + unsigned int ext_intr_stat; + int irq_flag = 0; + + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET); + if (val & ACP_DSP_TO_HOST_IRQ) { + snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET, + ACP_DSP_TO_HOST_IRQ); + return IRQ_WAKE_THREAD; + } + + ext_intr_stat = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->ext_intr_stat); + if (ext_intr_stat & ACP_ERROR_IRQ_MASK) { + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, ACP_ERROR_IRQ_MASK); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_error_stat, 0); + irq_flag = 1; + } + + if (irq_flag) + return IRQ_HANDLED; + + return IRQ_NONE; +} + static int acp_power_on(struct snd_sof_dev *sdev) { const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); @@ -571,6 +617,7 @@ static int acp_power_on(struct snd_sof_dev *sdev) unsigned int base = desc->pgfsm_base; unsigned int val; unsigned int acp_pgfsm_status_mask, acp_pgfsm_cntl_mask; + bool use_masked_status = false; int ret; val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET); @@ -595,6 +642,12 @@ static int acp_power_on(struct snd_sof_dev *sdev) acp_pgfsm_status_mask = ACP70_PGFSM_STATUS_MASK; acp_pgfsm_cntl_mask = ACP70_PGFSM_CNTL_POWER_ON_MASK; break; + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + acp_pgfsm_status_mask = ACP7X_PGFSM_STATUS_MASK; + acp_pgfsm_cntl_mask = ACP7X_PGFSM_CNTL_POWER_ON_MASK; + use_masked_status = true; + break; default: return -EINVAL; } @@ -603,8 +656,17 @@ static int acp_power_on(struct snd_sof_dev *sdev) snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + PGFSM_CONTROL_OFFSET, acp_pgfsm_cntl_mask); - ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET, val, - !val, ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US); + if (use_masked_status) + ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, + base + PGFSM_STATUS_OFFSET, val, + !(val & acp_pgfsm_status_mask), + ACP_REG_POLL_INTERVAL, + ACP_REG_POLL_TIMEOUT_US); + else + ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, + base + PGFSM_STATUS_OFFSET, val, + !val, ACP_REG_POLL_INTERVAL, + ACP_REG_POLL_TIMEOUT_US); if (ret < 0) dev_err(sdev->dev, "timeout in ACP_PGFSM_STATUS read\n"); @@ -703,6 +765,13 @@ static int acp_init(struct snd_sof_dev *sdev) snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP70_PME_EN, 1); break; + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_ZSC_DSP_CTRL, 0); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_PME_EN, 1); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_DSP0_IDMA_ERROR_MASK, + ACP7X_IDMA_ERROR_MASK); + break; } return 0; } @@ -749,6 +818,8 @@ int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state) case ACP72_PCI_ID: enable = true; break; + default: + break; } snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, enable); @@ -990,6 +1061,194 @@ void amd_sof_acp_remove(struct snd_sof_dev *sdev) } EXPORT_SYMBOL_NS(amd_sof_acp_remove, "SND_SOC_SOF_AMD_COMMON"); +int amd_sof_acp7x_probe(struct snd_sof_dev *sdev) +{ + struct pci_dev *pci = to_pci_dev(sdev->dev); + struct acp_dev_data *adata; + const struct sof_amd_acp_desc *chip; + const union acpi_object *obj; + struct acpi_device *adev; + unsigned int addr; + unsigned int irqflags; + int ret; + + chip = get_chip_info(sdev->pdata); + if (!chip) { + dev_err(sdev->dev, "no such device supported, chip id:%x\n", pci->device); + return -EIO; + } + adata = devm_kzalloc(sdev->dev, sizeof(struct acp_dev_data), GFP_KERNEL); + if (!adata) + return -ENOMEM; + + adata->dev = sdev; + adata->dmic_dev = platform_device_register_data(sdev->dev, "dmic-codec", + PLATFORM_DEVID_NONE, NULL, 0); + if (IS_ERR(adata->dmic_dev)) { + dev_err(sdev->dev, "failed to register platform for dmic codec\n"); + return PTR_ERR(adata->dmic_dev); + } + + addr = pci_resource_start(pci, ACP_DSP_BAR); + sdev->bar[ACP_DSP_BAR] = devm_ioremap(sdev->dev, addr, pci_resource_len(pci, ACP_DSP_BAR)); + if (!sdev->bar[ACP_DSP_BAR]) { + dev_err(sdev->dev, "ioremap error\n"); + ret = -ENXIO; + goto unregister_dev; + } + + pci_set_master(pci); + adata->addr = addr; + adata->reg_range = chip->reg_end_addr - chip->reg_start_addr; + adata->pci_rev = pci->revision; + mutex_init(&adata->acp_lock); + sdev->pdata->hw_pdata = adata; + + ret = acp_init(sdev); + if (ret < 0) + goto unregister_dev; + + adev = ACPI_COMPANION(&pci->dev); + + sdev->ipc_irq = pci->irq; + irqflags = IRQF_SHARED; + + ret = request_threaded_irq(pci->irq, acp7x_irq_handler, acp_irq_thread, + irqflags, "AudioDSP", sdev); + if (ret < 0) { + dev_err(sdev->dev, "failed to register IRQ %d\n", sdev->ipc_irq); + goto unregister_dev; + } + + if (adev) { + if (!acpi_dev_get_property(adev, "acp-sof-signed-firmware-image", + ACPI_TYPE_INTEGER, &obj)) + adata->acp_sof_signed_firmware_image = obj->integer.value; + } + + sdev->dsp_box.offset = 0; + sdev->dsp_box.size = BOX_SIZE_512; + + sdev->host_box.offset = sdev->dsp_box.offset + sdev->dsp_box.size; + sdev->host_box.size = BOX_SIZE_512; + + sdev->debug_box.offset = sdev->host_box.offset + sdev->host_box.size; + sdev->debug_box.size = BOX_SIZE_1024; + + if (adata->acp_sof_signed_firmware_image) { + adata->fw_code_bin = devm_kasprintf(sdev->dev, GFP_KERNEL, + "sof-%s-code.bin", chip->name); + if (!adata->fw_code_bin) { + ret = -ENOMEM; + goto free_ipc_irq; + } + adata->fw_data_bin = devm_kasprintf(sdev->dev, GFP_KERNEL, + "sof-%s-data.bin", chip->name); + if (!adata->fw_data_bin) { + ret = -ENOMEM; + goto free_ipc_irq; + } + } + + adata->enable_fw_debug = enable_fw_debug; + acp_memory_init(sdev); + acp_dsp_stream_init(sdev); + + return 0; + +free_ipc_irq: + free_irq(sdev->ipc_irq, sdev); +unregister_dev: + platform_device_unregister(adata->dmic_dev); + return ret; +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_probe, "SND_SOC_SOF_AMD_COMMON"); + +void amd_sof_acp7x_remove(struct snd_sof_dev *sdev) +{ + struct acp_dev_data *adata = sdev->pdata->hw_pdata; + + if (sdev->ipc_irq) + free_irq(sdev->ipc_irq, sdev); + + if (adata->dmic_dev) + platform_device_unregister(adata->dmic_dev); + + acp_reset(sdev); +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_remove, "SND_SOC_SOF_AMD_COMMON"); + +int amd_sof_acp7x_suspend(struct snd_sof_dev *sdev, u32 target_state) +{ + struct acp_dev_data *acp_data; + int ret; + bool enable = false; + + acp_data = sdev->pdata->hw_pdata; + + ret = acp_reset(sdev); + if (ret) { + dev_err(sdev->dev, "ACP Reset failed\n"); + return ret; + } + switch (acp_data->pci_rev) { + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + enable = true; + break; + default: + break; + } + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, enable); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_ZSC_DSP_CTRL, 1); + + return 0; +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_suspend, "SND_SOC_SOF_AMD_COMMON"); + +int amd_sof_acp7x_resume(struct snd_sof_dev *sdev) +{ + struct acp_dev_data *acp_data; + int ret; + + acp_data = sdev->pdata->hw_pdata; + + ret = acp_init(sdev); + if (ret) { + dev_err(sdev->dev, "ACP Init failed\n"); + return ret; + } + ret = acp_memory_init(sdev); + if (ret) { + dev_err(sdev->dev, "ACP Memory init failed\n"); + return ret; + } + + switch (acp_data->pci_rev) { + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_PME_EN, 1); + break; + default: + break; + } + + return 0; +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_resume, "SND_SOC_SOF_AMD_COMMON"); + +int amd_sof_acp7x_suspend_runtime(struct snd_sof_dev *sdev) +{ + return amd_sof_acp7x_suspend(sdev, 0); +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_suspend_runtime, "SND_SOC_SOF_AMD_COMMON"); + +int amd_sof_acp7x_resume_runtime(struct snd_sof_dev *sdev) +{ + return amd_sof_acp7x_resume(sdev); +} +EXPORT_SYMBOL_NS(amd_sof_acp7x_resume_runtime, "SND_SOC_SOF_AMD_COMMON"); + MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("AMD ACP sof driver"); MODULE_IMPORT_NS("SOUNDWIRE_AMD_INIT"); diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h index 7bcb76676a98..16d66b2eaa70 100644 --- a/sound/soc/sof/amd/acp.h +++ b/sound/soc/sof/amd/acp.h @@ -76,6 +76,12 @@ #define ACP70_PCI_ID 0x70 #define ACP71_PCI_ID 0x71 #define ACP72_PCI_ID 0x72 +#define ACP7B_PCI_ID 0x7B +#define ACP7F_PCI_ID 0x7F + +#define ACP7X_PGFSM_CNTL_POWER_ON_MASK 0x7F +#define ACP7X_PGFSM_STATUS_MASK 0xFFF +#define ACP7X_SRAM_PTE_OFFSET ACP6X_SRAM_PTE_OFFSET #define HOST_BRIDGE_CZN 0x1630 #define HOST_BRIDGE_VGH 0x1645 @@ -107,6 +113,9 @@ #define PROBE_STATUS_BIT BIT(31) #define ACP_FIRMWARE_SIGNATURE 0x100 +#define ACP_IMAGE_HEADER_SIZE ACP_FIRMWARE_SIGNATURE +#define ACP_IMAGE_HDR_SIZE_FW_SIGNED_OFF 0x14 + #define ACP_ERROR_IRQ_MASK BIT(29) #define ACP_SDW0_IRQ_MASK BIT(21) #define ACP_SDW1_IRQ_MASK BIT(2) @@ -271,6 +280,7 @@ struct acp_dev_data { /* acp70_sdw1_wake_event flag set to true when wake irq asserted for SW1 instance */ bool acp70_sdw1_wake_event; unsigned int pci_rev; + int acp_sof_signed_firmware_image; }; void memcpy_to_scratch(struct snd_sof_dev *sdev, u32 offset, unsigned int *src, size_t bytes); @@ -344,6 +354,16 @@ int sof_acp63_ops_init(struct snd_sof_dev *sdev); extern struct snd_sof_dsp_ops sof_acp70_ops; int sof_acp70_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_acp7x_ops; +int sof_acp7x_ops_init(struct snd_sof_dev *sdev); + +int amd_sof_acp7x_probe(struct snd_sof_dev *sdev); +void amd_sof_acp7x_remove(struct snd_sof_dev *sdev); +int amd_sof_acp7x_suspend(struct snd_sof_dev *sdev, u32 target_state); +int amd_sof_acp7x_resume(struct snd_sof_dev *sdev); +int amd_sof_acp7x_suspend_runtime(struct snd_sof_dev *sdev); +int amd_sof_acp7x_resume_runtime(struct snd_sof_dev *sdev); + struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev); /* Machine configuration */ int snd_amd_acp_find_config(struct pci_dev *pci); diff --git a/sound/soc/sof/amd/acp7x.c b/sound/soc/sof/amd/acp7x.c new file mode 100644 index 000000000000..b6722d11168c --- /dev/null +++ b/sound/soc/sof/amd/acp7x.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2025 Advanced Micro Devices, Inc. +// +// Authors: Vijendar Mukunda + +/* + * Hardware interface for Audio DSP on ACP7.B/7.F platforms + */ + +#include +#include +#include +#include +#include + +#include "../ops.h" +#include "../sof-audio.h" +#include "acp.h" +#include "acp-dsp-offset.h" + +#define I2S_TDM0_INSTANCE 0 +#define I2S_TDM1_INSTANCE 1 +#define I2S_TDM2_INSTANCE 2 +#define PDM0_DMIC_INSTANCE 3 +#define PDM1_DMIC_INSTANCE 4 + +static struct snd_soc_dai_driver acp7x_sof_dai[] = { + [I2S_TDM0_INSTANCE] = { + .id = I2S_TDM0_INSTANCE, + .name = "acp-sof-i2s0", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S HS controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + }, + + [I2S_TDM1_INSTANCE] = { + .id = I2S_TDM1_INSTANCE, + .name = "acp-sof-i2s1", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S BT controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + }, + + [I2S_TDM2_INSTANCE] = { + .id = I2S_TDM2_INSTANCE, + .name = "acp-sof-i2s2", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S SP controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + }, + + [PDM0_DMIC_INSTANCE] = { + .id = PDM0_DMIC_INSTANCE, + .name = "acp-sof-dmic0", + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 4, + .rate_min = 8000, + .rate_max = 48000, + }, + }, + + [PDM1_DMIC_INSTANCE] = { + .id = PDM1_DMIC_INSTANCE, + .name = "acp-sof-dmic1", + .capture = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 4, + .rate_min = 8000, + .rate_max = 96000, + }, + }, +}; + +static int sof_acp7x_post_fw_run_delay(struct snd_sof_dev *sdev) +{ + /* + * Resuming from suspend in some cases may cause the DSP firmware + * to enter an unrecoverable faulty state. Delaying a bit any host + * to DSP transmission right after firmware boot completion seems + * to resolve the issue. + */ + if (!sdev->first_boot) + usleep_range(100, 150); + + return 0; +} + +struct snd_sof_dsp_ops sof_acp7x_ops; +EXPORT_SYMBOL_NS(sof_acp7x_ops, "SND_SOC_SOF_AMD_COMMON"); + +int sof_acp7x_ops_init(struct snd_sof_dev *sdev) +{ + struct acpi_device *adev = ACPI_COMPANION(&to_pci_dev(sdev->dev)->dev); + const union acpi_object *obj; + int acp_sof_signed_firmware_image = 0; + int acp_sof_post_fw_run_delay = 0; + + /* common defaults */ + memcpy(&sof_acp7x_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops)); + + sof_acp7x_ops.drv = acp7x_sof_dai; + sof_acp7x_ops.num_drv = ARRAY_SIZE(acp7x_sof_dai); + sof_acp7x_ops.probe = amd_sof_acp7x_probe; + sof_acp7x_ops.remove = amd_sof_acp7x_remove; + + if (adev) { + if (!acpi_dev_get_property(adev, "acp-sof-signed-firmware-image", + ACPI_TYPE_INTEGER, &obj)) + acp_sof_signed_firmware_image = obj->integer.value; + + if (!acpi_dev_get_property(adev, "acp-sof-post_fw_run_delay", + ACPI_TYPE_INTEGER, &obj)) + acp_sof_post_fw_run_delay = obj->integer.value; + } + + if (acp_sof_signed_firmware_image) + sof_acp7x_ops.load_firmware = acp_sof_load_signed_firmware; + + if (acp_sof_post_fw_run_delay) + sof_acp7x_ops.post_fw_run = sof_acp7x_post_fw_run_delay; + + sof_acp7x_ops.suspend = amd_sof_acp7x_suspend; + sof_acp7x_ops.resume = amd_sof_acp7x_resume; + sof_acp7x_ops.runtime_suspend = amd_sof_acp7x_suspend_runtime; + sof_acp7x_ops.runtime_resume = amd_sof_acp7x_resume_runtime; + + return 0; +} diff --git a/sound/soc/sof/amd/pci-acp7x.c b/sound/soc/sof/amd/pci-acp7x.c new file mode 100644 index 000000000000..532e15313795 --- /dev/null +++ b/sound/soc/sof/amd/pci-acp7x.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Authors: Vijendar Mukunda + +/* + * PCI interface for ACP7.B/7.F devices + */ + +#include +#include +#include +#include + +#include "../ops.h" +#include "../sof-pci-dev.h" +#include "../../amd/mach-config.h" +#include "acp.h" +#include "acp-dsp-offset.h" + +#define ACP7X_FUTURE_REG_ACLK_0 0x18e0 +#define ACP7X_REG_START 0x1240000 +#define ACP7X_REG_END 0x125C000 + +static const struct sof_amd_acp_desc acp7x_chip_info = { + .name = "acp7x", + .pgfsm_base = ACP7X_PGFSM_BASE, + .ext_intr_enb = ACP6X_EXTERNAL_INTR_ENB, + .ext_intr_cntl = ACP7X_EXTERNAL_INTR_CNTL, + .ext_intr_stat = ACP7X_EXT_INTR_STAT, + .ext_intr_stat1 = ACP7X_EXT_INTR_STAT1, + .dsp_intr_base = ACP7X_DSP_SW_INTR_BASE, + .acp_error_stat = ACP7X_ERROR_STATUS, + .sram_pte_offset = ACP7X_SRAM_PTE_OFFSET, + .hw_semaphore_offset = ACP7X_AXI2DAGB_SEM_0, + .fusion_dsp_offset = ACP7X_DSP_FUSION_RUNSTALL, + .probe_reg_offset = ACP7X_FUTURE_REG_ACLK_0, + .reg_start_addr = ACP7X_REG_START, + .reg_end_addr = ACP7X_REG_END, +}; + +static const struct sof_dev_desc acp7x_desc = { + .machines = snd_soc_acpi_amd_acp7x_sof_machines, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &acp7x_chip_info, + .ipc_supported_mask = BIT(SOF_IPC_TYPE_3), + .ipc_default = SOF_IPC_TYPE_3, + .default_fw_path = { + [SOF_IPC_TYPE_3] = "amd/sof", + }, + .default_tplg_path = { + [SOF_IPC_TYPE_3] = "amd/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC_TYPE_3] = "sof-acp7x.ri", + }, + .nocodec_tplg_filename = "sof-acp.tplg", + .ops = &sof_acp7x_ops, + .ops_init = sof_acp7x_ops_init, +}; + +static int acp7x_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) +{ + unsigned int flag; + + switch (pci->revision) { + case ACP7B_PCI_ID: + case ACP7F_PCI_ID: + break; + default: + return -ENODEV; + } + + flag = snd_amd_acp_find_config(pci); + if (flag != FLAG_AMD_SOF && flag != FLAG_AMD_SOF_ONLY_DMIC) + return -ENODEV; + + return sof_pci_probe(pci, pci_id); +} + +static void acp7x_pci_remove(struct pci_dev *pci) +{ + sof_pci_remove(pci); +} + +/* PCI IDs */ +static const struct pci_device_id acp7x_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_PCI_DEV_ID), + .driver_data = (unsigned long)&acp7x_desc}, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, acp7x_pci_ids); + +/* pci_driver definition */ +static struct pci_driver snd_sof_pci_amd_acp7x_driver = { + .name = KBUILD_MODNAME, + .id_table = acp7x_pci_ids, + .probe = acp7x_pci_probe, + .remove = acp7x_pci_remove, + .driver = { + .pm = pm_ptr(&sof_pci_pm), + }, +}; +module_pci_driver(snd_sof_pci_amd_acp7x_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("ACP7X SOF Driver"); +MODULE_IMPORT_NS("SND_SOC_SOF_AMD_COMMON"); +MODULE_IMPORT_NS("SND_SOC_SOF_PCI_DEV"); diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c index 90ef5d99f626..143bf0fe8dd9 100644 --- a/sound/soc/sof/ipc3-pcm.c +++ b/sound/soc/sof/ipc3-pcm.c @@ -421,6 +421,12 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, dev_dbg(component->dev, "AMD_SDW channels_min: %d channels_max: %d\n", channels->min, channels->max); break; + case SOF_DAI_AMD_I2S: + rate->min = private->dai_config->acp_i2s.fsync_rate; + rate->max = private->dai_config->acp_i2s.fsync_rate; + channels->min = private->dai_config->acp_i2s.tdm_slots; + channels->max = private->dai_config->acp_i2s.tdm_slots; + break; default: dev_err(component->dev, "Invalid DAI type %d\n", private->dai_config->type); break; diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c index 4e066bbded91..26d85ca9be26 100644 --- a/sound/soc/sof/ipc3-topology.c +++ b/sound/soc/sof/ipc3-topology.c @@ -281,7 +281,10 @@ static const struct sof_topology_token acpdmic_tokens[] = { offsetof(struct sof_ipc_dai_acpdmic_params, pdm_ch)}, }; -/* ACPI2S */ +/* + * ACPI2S tokens fill struct sof_ipc_dai_acp_params; SOF_DAI_AMD_I2S (ACPTDM + * on ACP7.B/7.F) reuses this tuple group rather than defining a parallel set. + */ static const struct sof_topology_token acpi2s_tokens[] = { {SOF_TKN_AMD_ACPI2S_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, offsetof(struct sof_ipc_dai_acp_params, fsync_rate)}, @@ -289,6 +292,8 @@ static const struct sof_topology_token acpi2s_tokens[] = { offsetof(struct sof_ipc_dai_acp_params, tdm_slots)}, {SOF_TKN_AMD_ACPI2S_TDM_MODE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, offsetof(struct sof_ipc_dai_acp_params, tdm_mode)}, + {SOF_TKN_AMD_ACPI2S_FORMAT, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, + offsetof(struct sof_ipc_dai_acp_params, format)}, }; /* MICFIL PDM */ @@ -1363,6 +1368,35 @@ static int sof_link_acp_sdw_load(struct snd_soc_component *scomp, struct snd_sof return 0; } +static int sof_link_acp_i2s_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink, + struct sof_ipc_dai_config *config, struct snd_sof_dai *dai) +{ + struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs; + struct sof_dai_private_data *private = dai->private; + u32 size = sizeof(*config); + int ret; + + /* handle master/slave and inverted clocks */ + sof_dai_set_format(hw_config, config); + + /* init IPC */ + memset(&config->acp_i2s, 0, sizeof(config->acp_i2s)); + config->hdr.size = size; + + ret = sof_update_ipc_object(scomp, &config->acp_i2s, SOF_ACPI2S_TOKENS, slink->tuples, + slink->num_tuples, size, slink->num_hw_configs); + if (ret < 0) + return ret; + + dai->number_configs = 1; + dai->current_config = 0; + private->dai_config = kmemdup(config, size, GFP_KERNEL); + if (!private->dai_config) + return -ENOMEM; + + return 0; +} + static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink, struct sof_ipc_dai_config *config, struct snd_sof_dai *dai) { @@ -1692,6 +1726,9 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget) case SOF_DAI_AMD_SDW: ret = sof_link_acp_sdw_load(scomp, slink, config, dai); break; + case SOF_DAI_AMD_I2S: + ret = sof_link_acp_i2s_load(scomp, slink, config, dai); + break; default: break; } diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 95ad5266b0c6..8ac7dde32f77 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -1127,6 +1127,7 @@ static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", src->available_fmt.num_input_formats, src->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } @@ -1179,6 +1180,7 @@ static int sof_ipc4_widget_setup_comp_asrc(struct snd_sof_widget *swidget) "Invalid number of formats: input: %d, output: %d\n", asrc->available_fmt.num_input_formats, asrc->available_fmt.num_output_formats); + ret = -EINVAL; goto err; } diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c index 11a95dba3c9c..34744f8ae34a 100644 --- a/sound/soc/sof/nocodec.c +++ b/sound/soc/sof/nocodec.c @@ -21,12 +21,13 @@ static struct snd_soc_card sof_nocodec_card = { static int sof_nocodec_bes_setup(struct device *dev, struct snd_soc_dai_driver *drv, struct snd_soc_dai_link *links, - int link_num, struct snd_soc_card *card) + int link_num) { + struct snd_soc_card *card = &sof_nocodec_card; struct snd_soc_dai_link_component *dlc; int i; - if (!drv || !links || !card) + if (!drv || !links) return -EINVAL; /* set up BE dai_links */ @@ -67,10 +68,10 @@ static int sof_nocodec_bes_setup(struct device *dev, return 0; } -static int sof_nocodec_setup(struct device *dev, - u32 num_dai_drivers, - struct snd_soc_dai_driver *dai_drivers) +static int sof_nocodec_setup(struct device *dev, struct snd_soc_acpi_mach *mach) { + u32 num_dai_drivers = mach->mach_params.num_dai_drivers; + struct snd_soc_dai_driver *dai_drivers = mach->mach_params.dai_drivers; struct snd_soc_dai_link *links; /* create dummy BE dai_links */ @@ -78,7 +79,7 @@ static int sof_nocodec_setup(struct device *dev, if (!links) return -ENOMEM; - return sof_nocodec_bes_setup(dev, dai_drivers, links, num_dai_drivers, &sof_nocodec_card); + return sof_nocodec_bes_setup(dev, dai_drivers, links, num_dai_drivers); } static int sof_nocodec_probe(struct platform_device *pdev) @@ -92,8 +93,7 @@ static int sof_nocodec_probe(struct platform_device *pdev) snd_soc_card_set_topology_name(card, "sof"); - ret = sof_nocodec_setup(card->dev, mach->mach_params.num_dai_drivers, - mach->mach_params.dai_drivers); + ret = sof_nocodec_setup(card->dev, mach); if (ret < 0) return ret; diff --git a/sound/soc/sof/sof-client-ipc-flood-test.c b/sound/soc/sof/sof-client-ipc-flood-test.c index 7b72d1c9c739..2396cc35489a 100644 --- a/sound/soc/sof/sof-client-ipc-flood-test.c +++ b/sound/soc/sof/sof-client-ipc-flood-test.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/sof/sof-client-ipc-kernel-injector.c b/sound/soc/sof/sof-client-ipc-kernel-injector.c index d5984990098a..02d0d97ad1a0 100644 --- a/sound/soc/sof/sof-client-ipc-kernel-injector.c +++ b/sound/soc/sof/sof-client-ipc-kernel-injector.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/sof/sof-client-ipc-msg-injector.c b/sound/soc/sof/sof-client-ipc-msg-injector.c index c28f106de6ba..932ab459c079 100644 --- a/sound/soc/sof/sof-client-ipc-msg-injector.c +++ b/sound/soc/sof/sof-client-ipc-msg-injector.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/sof/sof-client-probes-ipc3.c b/sound/soc/sof/sof-client-probes-ipc3.c index a78ec0954a61..a3e382d6161f 100644 --- a/sound/soc/sof/sof-client-probes-ipc3.c +++ b/sound/soc/sof/sof-client-probes-ipc3.c @@ -107,7 +107,7 @@ static int ipc3_probes_info(struct sof_client_dev *cdev, unsigned int cmd, struct device *dev = &cdev->auxdev.dev; struct sof_ipc_probe_info_params msg = {{{0}}}; struct sof_ipc_probe_info_params *reply; - size_t bytes; + size_t bytes, elem_size, payload_size; int ret; *params = NULL; @@ -128,14 +128,29 @@ static int ipc3_probes_info(struct sof_client_dev *cdev, unsigned int cmd, if (ret < 0 || reply->rhdr.error < 0) goto exit; + payload_size = reply->rhdr.hdr.size; + if (payload_size < offsetof(struct sof_ipc_probe_info_params, dma)) { + ret = -EINVAL; + goto exit; + } + if (!reply->num_elems) goto exit; if (cmd == SOF_IPC_PROBE_DMA_INFO) - bytes = sizeof(reply->dma[0]); + elem_size = sizeof(reply->dma[0]); else - bytes = sizeof(reply->desc[0]); - bytes *= reply->num_elems; + elem_size = sizeof(reply->desc[0]); + + payload_size -= offsetof(struct sof_ipc_probe_info_params, dma); + if (reply->num_elems > payload_size / elem_size) { + dev_err(dev, "%s: invalid probe info element count %u\n", + __func__, reply->num_elems); + ret = -EINVAL; + goto exit; + } + + bytes = reply->num_elems * elem_size; *params = kmemdup(&reply->dma[0], bytes, GFP_KERNEL); if (!*params) { ret = -ENOMEM; diff --git a/sound/soc/sof/sof-client-probes-ipc4.c b/sound/soc/sof/sof-client-probes-ipc4.c index 88397c7dc4c3..2eef32b55395 100644 --- a/sound/soc/sof/sof-client-probes-ipc4.c +++ b/sound/soc/sof/sof-client-probes-ipc4.c @@ -248,10 +248,19 @@ static int ipc4_probes_points_info(struct sof_client_dev *cdev, return ret; } info = msg.data_ptr; + if (msg.data_size < sizeof(*info) || + info->num_elems > (msg.data_size - sizeof(*info)) / + sizeof(info->points[0])) { + dev_err(dev, "%s: invalid probe info element count %u\n", + __func__, info->num_elems); + kfree(msg.data_ptr); + return -EINVAL; + } + *num_desc = info->num_elems; dev_dbg(dev, "%s: got %zu probe points", __func__, *num_desc); - *desc = kzalloc(*num_desc * sizeof(**desc), GFP_KERNEL); + *desc = kcalloc(*num_desc, sizeof(**desc), GFP_KERNEL); if (!*desc) { kfree(msg.data_ptr); return -ENOMEM; diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 42a2d90bb705..6fd69ba11c41 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -309,7 +309,7 @@ static const struct sof_dai_types sof_dais[] = { {"ACPHS_VIRTUAL", SOF_DAI_AMD_HS_VIRTUAL}, {"MICFIL", SOF_DAI_IMX_MICFIL}, {"ACP_SDW", SOF_DAI_AMD_SDW}, - + {"ACPTDM", SOF_DAI_AMD_I2S}, }; static enum sof_ipc_dai_type find_dai(const char *name) @@ -1102,7 +1102,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, full = NULL; partial = NULL; - list_for_each_entry(rtd, &card->rtd_list, list) { + for_each_card_rtds(card, rtd) { /* does stream match DAI link ? */ if (rtd->dai_link->stream_name) { if (!strcmp(rtd->dai_link->stream_name, w->sname)) { @@ -1167,7 +1167,7 @@ static void sof_disconnect_dai_widget(struct snd_soc_component *scomp, else return; - list_for_each_entry(rtd, &card->rtd_list, list) { + for_each_card_rtds(card, rtd) { /* does stream match DAI link ? */ if (!rtd->dai_link->stream_name || !strstr(rtd->dai_link->stream_name, sname)) @@ -1943,8 +1943,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ private->array, le32_to_cpu(private->size)); if (ret < 0) { dev_err(scomp->dev, "Failed tp parse common DAI link tokens\n"); - kfree(slink); - return ret; + goto free_slink; } token_list = tplg_ops ? tplg_ops->token_list : NULL; @@ -1995,6 +1994,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ case SOF_DAI_AMD_HS: case SOF_DAI_AMD_SP_VIRTUAL: case SOF_DAI_AMD_HS_VIRTUAL: + case SOF_DAI_AMD_I2S: token_id = SOF_ACPI2S_TOKENS; num_tuples += token_list[SOF_ACPI2S_TOKENS].count; break; @@ -2013,8 +2013,8 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ /* allocate memory for tuples array */ slink->tuples = kzalloc_objs(*slink->tuples, num_tuples); if (!slink->tuples) { - kfree(slink); - return -ENOMEM; + ret = -ENOMEM; + goto free_slink; } if (token_list[SOF_DAI_LINK_TOKENS].tokens) { @@ -2070,6 +2070,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ err: kfree(slink->tuples); +free_slink: kfree(slink); return ret; diff --git a/sound/soc/sunxi/sun50i-codec-analog.c b/sound/soc/sunxi/sun50i-codec-analog.c index a19f8aaaf1c4..9f5b067a8ccc 100644 --- a/sound/soc/sunxi/sun50i-codec-analog.c +++ b/sound/soc/sunxi/sun50i-codec-analog.c @@ -13,7 +13,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c index eddfebe16616..5c784b1f6846 100644 --- a/sound/soc/sunxi/sun50i-dmic.c +++ b/sound/soc/sunxi/sun50i-dmic.c @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra186_asrc.c b/sound/soc/tegra/tegra186_asrc.c index 7135aa23a7fc..7f360dfaf8b1 100644 --- a/sound/soc/tegra/tegra186_asrc.c +++ b/sound/soc/tegra/tegra186_asrc.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra186_dspk.c b/sound/soc/tegra/tegra186_dspk.c index 7cf7d6dbfc35..0d3807b231f3 100644 --- a/sound/soc/tegra/tegra186_dspk.c +++ b/sound/soc/tegra/tegra186_dspk.c @@ -5,7 +5,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra20_spdif.c b/sound/soc/tegra/tegra20_spdif.c index 38661d9b4a7c..5eefcf149ae3 100644 --- a/sound/soc/tegra/tegra20_spdif.c +++ b/sound/soc/tegra/tegra20_spdif.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_adx.c b/sound/soc/tegra/tegra210_adx.c index 9b662fcee66f..15a94196ee1a 100644 --- a/sound/soc/tegra/tegra210_adx.c +++ b/sound/soc/tegra/tegra210_adx.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_amx.c b/sound/soc/tegra/tegra210_amx.c index 930b080aec0a..cc1f9c158191 100644 --- a/sound/soc/tegra/tegra210_amx.c +++ b/sound/soc/tegra/tegra210_amx.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_dmic.c b/sound/soc/tegra/tegra210_dmic.c index 3e42e2c75eb9..6098ad056ba9 100644 --- a/sound/soc/tegra/tegra210_dmic.c +++ b/sound/soc/tegra/tegra210_dmic.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_i2s.c b/sound/soc/tegra/tegra210_i2s.c index 0259b137547c..ff8c72fc38c5 100644 --- a/sound/soc/tegra/tegra210_i2s.c +++ b/sound/soc/tegra/tegra210_i2s.c @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_mixer.c b/sound/soc/tegra/tegra210_mixer.c index c237ba7531de..a69774578d69 100644 --- a/sound/soc/tegra/tegra210_mixer.c +++ b/sound/soc/tegra/tegra210_mixer.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_mvc.c b/sound/soc/tegra/tegra210_mvc.c index b55f8142c4a4..ac04350107c4 100644 --- a/sound/soc/tegra/tegra210_mvc.c +++ b/sound/soc/tegra/tegra210_mvc.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/tegra/tegra210_ope.c b/sound/soc/tegra/tegra210_ope.c index ad4c400281e8..30a54b1222d9 100644 --- a/sound/soc/tegra/tegra210_ope.c +++ b/sound/soc/tegra/tegra210_ope.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c index c214ae0d7b95..312298e0b004 100644 --- a/sound/soc/ti/j721e-evm.c +++ b/sound/soc/ti/j721e-evm.c @@ -4,6 +4,7 @@ * Author: Peter Ujfalusi */ +#include #include #include #include @@ -263,7 +264,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) int ret = 0; int i; - guard(mutex)(&priv->mutex); + mutex_lock(&priv->mutex); domain->active++; @@ -303,6 +304,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream) out: if (ret) domain->active--; + mutex_unlock(&priv->mutex); return ret; } diff --git a/sound/soc/ti/omap-dmic.c b/sound/soc/ti/omap-dmic.c index 7ca46c57566d..b795b9f66b0e 100644 --- a/sound/soc/ti/omap-dmic.c +++ b/sound/soc/ti/omap-dmic.c @@ -11,7 +11,6 @@ */ #include -#include #include #include #include diff --git a/sound/soc/ti/omap-mcpdm.c b/sound/soc/ti/omap-mcpdm.c index c7d7b502f120..5698d2f26973 100644 --- a/sound/soc/ti/omap-mcpdm.c +++ b/sound/soc/ti/omap-mcpdm.c @@ -11,7 +11,6 @@ */ #include -#include #include #include #include diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index b20aae0caf60..a16e59248480 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -134,14 +134,22 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) struct device *dev = &urb->dev->dev; struct snd_usb_caiaqdev *cdev = urb->context; unsigned char *buf = urb->transfer_buffer; + unsigned int payload_len; + unsigned int copy_len; if (urb->status || !cdev) { dev_warn(dev, "received EP1 urb->status = %i\n", urb->status); return; } + if (urb->actual_length < 1) + return; + + payload_len = urb->actual_length - 1; switch(buf[0]) { case EP1_CMD_GET_DEVICE_INFO: + if (payload_len < sizeof(struct caiaq_device_spec)) + break; memcpy(&cdev->spec, buf+1, sizeof(struct caiaq_device_spec)); cdev->spec.fw_version = le16_to_cpu(cdev->spec.fw_version); dev_dbg(dev, "device spec (firmware %d): audio: %d in, %d out, " @@ -157,18 +165,21 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) wake_up(&cdev->ep1_wait_queue); break; case EP1_CMD_AUDIO_PARAMS: + if (payload_len < 1) + break; cdev->audio_parm_answer = buf[1]; wake_up(&cdev->ep1_wait_queue); break; case EP1_CMD_MIDI_READ: + if (urb->actual_length < 3 || urb->actual_length - 3 < buf[2]) + break; snd_usb_caiaq_midi_handle_input(cdev, buf[1], buf + 3, buf[2]); break; case EP1_CMD_READ_IO: if (cdev->chip.usb_id == USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_AUDIO8DJ)) { - if (urb->actual_length > sizeof(cdev->control_state)) - urb->actual_length = sizeof(cdev->control_state); - memcpy(cdev->control_state, buf + 1, urb->actual_length); + copy_len = min_t(unsigned int, payload_len, sizeof(cdev->control_state)); + memcpy(cdev->control_state, buf + 1, copy_len); wake_up(&cdev->ep1_wait_queue); break; } diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c index 5c70fdf61cc1..8d924330c54c 100644 --- a/sound/usb/caiaq/input.c +++ b/sound/usb/caiaq/input.c @@ -203,6 +203,8 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev, switch (cdev->chip.usb_id) { case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL2): + if (len < 6) + return; snd_caiaq_input_report_abs(cdev, ABS_X, buf, 2); snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 0); snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 1); @@ -210,11 +212,15 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev, case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL3): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER2): + if (len < 6) + return; snd_caiaq_input_report_abs(cdev, ABS_X, buf, 0); snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 1); snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 2); break; case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_TRAKTORKONTROLX1): + if (len < 16) + return; snd_caiaq_input_report_abs(cdev, ABS_HAT0X, buf, 4); snd_caiaq_input_report_abs(cdev, ABS_HAT0Y, buf, 2); snd_caiaq_input_report_abs(cdev, ABS_HAT1X, buf, 6); @@ -237,12 +243,16 @@ static void snd_caiaq_input_read_erp(struct snd_usb_caiaqdev *cdev, switch (cdev->chip.usb_id) { case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_AK1): + if (len < 2) + return; i = decode_erp(buf[0], buf[1]); input_report_abs(input_dev, ABS_X, i); input_sync(input_dev); break; case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER2): + if (len < 16) + return; i = decode_erp(buf[7], buf[5]); input_report_abs(input_dev, ABS_HAT0X, i); i = decode_erp(buf[12], buf[14]); @@ -263,6 +273,8 @@ static void snd_caiaq_input_read_erp(struct snd_usb_caiaqdev *cdev, break; case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_MASCHINECONTROLLER): + if (len < 22) + return; /* 4 under the left screen */ input_report_abs(input_dev, ABS_HAT0X, decode_erp(buf[21], buf[20])); input_report_abs(input_dev, ABS_HAT0Y, decode_erp(buf[15], buf[14])); @@ -308,9 +320,13 @@ static void snd_caiaq_input_read_io(struct snd_usb_caiaqdev *cdev, switch (cdev->chip.usb_id) { case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER2): + if (len < 5) + return; input_report_abs(cdev->input_dev, ABS_MISC, 255 - buf[4]); break; case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_TRAKTORKONTROLX1): + if (len < 7) + return; /* rotary encoders */ input_report_abs(cdev->input_dev, ABS_X, buf[5] & 0xf); input_report_abs(cdev->input_dev, ABS_Y, buf[5] >> 4); @@ -330,7 +346,7 @@ static void snd_usb_caiaq_tks4_dispatch(struct snd_usb_caiaqdev *cdev, { struct device *dev = caiaqdev_to_dev(cdev); - while (len) { + while (len >= TKS4_MSGBLOCK_SIZE) { unsigned int i, block_id = (buf[0] << 8) | buf[1]; switch (block_id) { diff --git a/sound/usb/card.c b/sound/usb/card.c index 6a3b576fb067..b36f513dccb9 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -905,6 +905,40 @@ static int try_to_register_card(struct snd_usb_audio *chip, int ifnum) return 0; } +static void usb_audio_disconnect_components(struct snd_usb_audio *chip) +{ + struct snd_usb_stream *as; + struct snd_usb_endpoint *ep; + struct usb_mixer_interface *mixer; + struct list_head *p; + + /* release the pcm resources */ + list_for_each_entry(as, &chip->pcm_list, list) { + snd_usb_stream_disconnect(as); + } + /* release the endpoint resources */ + list_for_each_entry(ep, &chip->ep_list, list) { + snd_usb_endpoint_release(ep); + } + /* release the midi resources */ + list_for_each(p, &chip->midi_list) { + snd_usbmidi_disconnect(p); + } + snd_usb_midi_v2_disconnect_all(chip); + /* + * Nice to check quirk && quirk->shares_media_device and + * then call the snd_media_device_delete(). Don't have + * access to the quirk here. snd_media_device_delete() + * accesses mixer_list + */ + snd_media_device_delete(chip); + + /* release mixer resources */ + list_for_each_entry(mixer, &chip->mixer_list, list) { + snd_usb_mixer_disconnect(mixer); + } +} + /* * probe the active usb device * @@ -1077,8 +1111,10 @@ static int usb_audio_probe(struct usb_interface *intf, * decrement before memory is possibly returned. */ atomic_dec(&chip->active); - if (!chip->num_interfaces) + if (!chip->num_interfaces) { + usb_audio_disconnect_components(chip); snd_card_free(chip->card); + } } return err; } @@ -1091,48 +1127,18 @@ static bool __usb_audio_disconnect(struct usb_interface *intf, struct snd_usb_audio *chip, struct snd_card *card) { - struct list_head *p; - guard(mutex)(®ister_mutex); if (platform_ops && platform_ops->disconnect_cb) platform_ops->disconnect_cb(chip); if (atomic_inc_return(&chip->shutdown) == 1) { - struct snd_usb_stream *as; - struct snd_usb_endpoint *ep; - struct usb_mixer_interface *mixer; - /* wait until all pending tasks done; * they are protected by snd_usb_lock_shutdown() */ snd_refcount_sync(&chip->usage_count); snd_card_disconnect(card); - /* release the pcm resources */ - list_for_each_entry(as, &chip->pcm_list, list) { - snd_usb_stream_disconnect(as); - } - /* release the endpoint resources */ - list_for_each_entry(ep, &chip->ep_list, list) { - snd_usb_endpoint_release(ep); - } - /* release the midi resources */ - list_for_each(p, &chip->midi_list) { - snd_usbmidi_disconnect(p); - } - snd_usb_midi_v2_disconnect_all(chip); - /* - * Nice to check quirk && quirk->shares_media_device and - * then call the snd_media_device_delete(). Don't have - * access to the quirk here. snd_media_device_delete() - * accesses mixer_list - */ - snd_media_device_delete(chip); - - /* release mixer resources */ - list_for_each_entry(mixer, &chip->mixer_list, list) { - snd_usb_mixer_disconnect(mixer); - } + usb_audio_disconnect_components(chip); } if (chip->quirk_flags & QUIRK_FLAG_DISABLE_AUTOSUSPEND) diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index ea746bdb36ff..6f5dcd35e1d4 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -1083,6 +1083,8 @@ static int fcp_find_fc_interface(struct usb_mixer_interface *mixer) if (desc->bInterfaceClass != 255) continue; + if (desc->bNumEndpoints < 1) + continue; epd = get_endpoint(intf->altsetting, 0); private->bInterfaceNumber = desc->bInterfaceNumber; diff --git a/sound/usb/midi2.c b/sound/usb/midi2.c index 04aeb9052f13..3ec633291772 100644 --- a/sound/usb/midi2.c +++ b/sound/usb/midi2.c @@ -470,6 +470,11 @@ static int create_midi2_endpoint(struct snd_usb_midi2_interface *umidi, static void free_midi2_endpoint(struct snd_usb_midi2_endpoint *ep) { list_del(&ep->list); + if (!ep->disconnected) { + ep->disconnected = 1; + kill_midi_urbs(ep, false); + drain_urb_queue(ep); + } free_midi_urbs(ep); kfree(ep); } diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index b4c855c25eef..703c118f9d4e 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1473,8 +1473,8 @@ static int get_min_max_with_quirks(struct usb_mixer_elem_info *cval, if (!cval->cmask) { snd_usb_set_cur_mix_value(cval, 0, 0, cval->max); } else { + idx = 0; for (i = 0; i < MAX_CHANNELS; i++) { - idx = 0; if (cval->cmask & BIT(i)) { snd_usb_set_cur_mix_value(cval, i + 1, idx, cval->max); idx++; diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index 628f841b04aa..fb3bd360e5a9 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -577,46 +577,30 @@ static bool snd_dualsense_ih_match(struct input_handler *handler, { struct dualsense_mixer_elem_info *mei; struct usb_device *snd_dev; - char *input_dev_path, *usb_dev_path; - size_t usb_dev_path_len; - bool match = false; + struct device *parent; mei = container_of(handler, struct dualsense_mixer_elem_info, ih); snd_dev = mei->info.head.mixer->chip->dev; - input_dev_path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL); - if (!input_dev_path) { - dev_warn(&snd_dev->dev, "Failed to get input dev path\n"); - return false; - } - - usb_dev_path = kobject_get_path(&snd_dev->dev.kobj, GFP_KERNEL); - if (!usb_dev_path) { - dev_warn(&snd_dev->dev, "Failed to get USB dev path\n"); - goto free_paths; - } - /* * Ensure the VID:PID matched input device supposedly owned by the * hid-playstation driver belongs to the actual hardware handled by - * the current USB audio device, which implies input_dev_path being - * a subpath of usb_dev_path. + * the current USB audio device. * * This verification is necessary when there is more than one identical * controller attached to the host system. + * + * The input device is registered below the HID device, USB interface and + * USB device, so compare the parent chain directly instead of building + * kobject path strings. This avoids dereferencing kobject names while the + * USB device hierarchy is being torn down during disconnect. */ - usb_dev_path_len = strlen(usb_dev_path); - if (usb_dev_path_len >= strlen(input_dev_path)) - goto free_paths; - - usb_dev_path[usb_dev_path_len] = '/'; - match = !memcmp(input_dev_path, usb_dev_path, usb_dev_path_len + 1); - -free_paths: - kfree(input_dev_path); - kfree(usb_dev_path); + for (parent = dev->dev.parent; parent; parent = parent->parent) { + if (parent == &snd_dev->dev) + return true; + } - return match; + return false; } static int snd_dualsense_ih_connect(struct input_handler *handler, @@ -3936,6 +3920,7 @@ static int snd_rme_digiface_controls_create(struct usb_mixer_interface *mixer) #define SND_DJM_450_IDX 0x5 #define SND_DJM_A9_IDX 0x6 #define SND_DJM_V10_IDX 0x7 +#define SND_DJM_S11_IDX 0x8 #define SND_DJM_CTL(_name, suffix, _default_value, _windex) { \ .name = _name, \ @@ -4276,6 +4261,21 @@ static const struct snd_djm_ctl snd_djm_ctls_v10[] = { // playback channels are fixed and controlled by hardware knobs on the mixer }; +// DJM-S11 +static const u16 snd_djm_opts_s11_cap1[] = { + 0x0100, 0x0103, 0x0106, 0x0107, 0x0108, 0x0109, 0x010d }; +static const u16 snd_djm_opts_s11_cap2[] = { + 0x0200, 0x0203, 0x0206, 0x0207, 0x0208, 0x0209, 0x020d }; +static const u16 snd_djm_opts_s11_cap3[] = { + 0x0307, 0x0308, 0x0309, 0x030a, 0x030d, 0x0311, 0x0312 }; + +static const struct snd_djm_ctl snd_djm_ctls_s11[] = { + SND_DJM_CTL("Master Input Level Capture Switch", cap_level, 0, SND_DJM_WINDEX_CAPLVL), + SND_DJM_CTL("Input 1 Capture Switch", s11_cap1, 1, SND_DJM_WINDEX_CAP), + SND_DJM_CTL("Input 2 Capture Switch", s11_cap2, 1, SND_DJM_WINDEX_CAP), + SND_DJM_CTL("Input 3 Capture Switch", s11_cap3, 3, SND_DJM_WINDEX_CAP) +}; + static const struct snd_djm_device snd_djm_devices[] = { [SND_DJM_250MK2_IDX] = SND_DJM_DEVICE(250mk2), [SND_DJM_750_IDX] = SND_DJM_DEVICE(750), @@ -4285,6 +4285,7 @@ static const struct snd_djm_device snd_djm_devices[] = { [SND_DJM_450_IDX] = SND_DJM_DEVICE(450), [SND_DJM_A9_IDX] = SND_DJM_DEVICE(a9), [SND_DJM_V10_IDX] = SND_DJM_DEVICE(v10), + [SND_DJM_S11_IDX] = SND_DJM_DEVICE(s11), }; static int snd_djm_controls_info(struct snd_kcontrol *kctl, @@ -4526,6 +4527,7 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) case USB_ID(0x1235, 0x821b): /* Focusrite Scarlett 16i16 4th Gen */ case USB_ID(0x1235, 0x821c): /* Focusrite Scarlett 18i16 4th Gen */ case USB_ID(0x1235, 0x821d): /* Focusrite Scarlett 18i20 4th Gen */ + case USB_ID(0x1235, 0x821e): /* Focusrite ISA C8X */ err = snd_fcp_init(mixer); break; @@ -4588,6 +4590,9 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) case USB_ID(0x2b73, 0x0034): /* Pioneer DJ DJM-V10 */ err = snd_djm_controls_create(mixer, SND_DJM_V10_IDX); break; + case USB_ID(0x2b73, 0x0037): /* Pioneer DJ DJM-S11 */ + err = snd_djm_controls_create(mixer, SND_DJM_S11_IDX); + break; case USB_ID(0x03f0, 0x0269): /* HP TB Dock G2 */ err = hp_dock_mixer_create(mixer); break; diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index a3f90cc7c6ca..e4bfd43a2488 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -1160,6 +1160,7 @@ uaudio_endpoint_setup(struct snd_usb_substream *subs, tr_pa = page_to_phys(pg); mem_info->dma = sg_dma_address(sgt->sgl); sg_free_table(sgt); + kfree(sgt); /* data transfer ring */ iova = uaudio_iommu_map_pa(MEM_XFER_RING, dma_coherent, tr_pa, @@ -1229,6 +1230,7 @@ static int uaudio_event_ring_setup(struct snd_usb_substream *subs, er_pa = page_to_phys(pg); mem_info->dma = sg_dma_address(sgt->sgl); sg_free_table(sgt); + kfree(sgt); iova = uaudio_iommu_map_pa(MEM_EVENT_RING, dma_coherent, er_pa, PAGE_SIZE); @@ -1618,8 +1620,13 @@ static void handle_uaudio_stream_req(struct qmi_handle *handle, if (req_msg->service_interval_valid) { ret = get_data_interval_from_si(subs, req_msg->service_interval); - if (ret == -EINVAL) + if (ret == -EINVAL) { + if (req_msg->enable) { + guard(mutex)(&chip->mutex); + subs->opened = 0; + } goto response; + } datainterval = ret; } @@ -1640,6 +1647,11 @@ static void handle_uaudio_stream_req(struct qmi_handle *handle, subs->opened = 0; } } else { + if (info_idx < 0) { + ret = -EINVAL; + goto response; + } + info = &uadev[pcm_card_num].info[info_idx]; if (info->data_ep_pipe) { ep = usb_pipe_endpoint(uadev[pcm_card_num].udev, @@ -1976,6 +1988,7 @@ static int qc_usb_audio_probe(struct auxiliary_device *auxdev, release_qmi: qc_usb_audio_cleanup_qmi_dev(); qmi_handle_release(svc->uaudio_svc_hdl); + kfree(svc->uaudio_svc_hdl); free_svc: kfree(svc); @@ -2000,6 +2013,7 @@ static void qc_usb_audio_remove(struct auxiliary_device *auxdev) qc_usb_audio_cleanup_qmi_dev(); qmi_handle_release(svc->uaudio_svc_hdl); + kfree(svc->uaudio_svc_hdl); kfree(svc); uaudio_svc = NULL; } diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 97f28c53d201..938908671d93 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -390,6 +390,20 @@ YAMAHA_DEVICE(0x105d, NULL), } } }, +{ + USB_DEVICE(0x0499, 0x150d), + QUIRK_DRIVER_INFO { + /* .vendor_name = "Yamaha", */ + /* .product_name = "CDS3000", */ + QUIRK_DATA_COMPOSITE { + { QUIRK_DATA_STANDARD_AUDIO(1) }, + { QUIRK_DATA_STANDARD_AUDIO(2) }, + { QUIRK_DATA_MIDI_YAMAHA(3) }, + { QUIRK_DATA_IGNORE(4) }, + QUIRK_COMPOSITE_END + } + } +}, { USB_DEVICE(0x0499, 0x1718), QUIRK_DRIVER_INFO { @@ -3296,6 +3310,73 @@ YAMAHA_DEVICE(0x7010, "UB99"), } } }, +{ + /* + * Pioneer DJ / AlphaTheta DJM-S11 + */ + USB_DEVICE(0x2b73, 0x0037), + QUIRK_DRIVER_INFO { + QUIRK_DATA_COMPOSITE { + { + QUIRK_DATA_STANDARD_MIXER(0) + }, + { + QUIRK_DATA_AUDIOFORMAT(1) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 14, + .iface = 1, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x01, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 48000 }, + .clock = 1, + .fmt_type = UAC_FORMAT_TYPE_I + } + }, + { + QUIRK_DATA_AUDIOFORMAT(2) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 10, + .iface = 2, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x82, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC | + USB_ENDPOINT_USAGE_IMPLICIT_FB, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 48000 }, + .clock = 1, + .fmt_type = UAC_FORMAT_TYPE_I + } + }, + { + /* Audio Control (unknown purpose) */ + .ifnum = 3, + .type = QUIRK_IGNORE_INTERFACE + }, + { + .ifnum = 4, + .type = QUIRK_MIDI_STANDARD_INTERFACE + }, + { + /* HID */ + .ifnum = 5, + .type = QUIRK_IGNORE_INTERFACE + }, + QUIRK_COMPOSITE_END + } + } +}, /* * MacroSilicon MS2100/MS2106 based AV capture cards diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 30f6d572e4d1..1cb588691e16 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2413,6 +2413,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x1bcf, 0x2283, /* NexiGo N930AF FHD Webcam */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), + DEVICE_FLG(0x1ff7, 0x0f81, /* SC13A Webcam */ + QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x2040, 0x7200, /* Hauppauge HVR-950Q */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x2040, 0x7201, /* Hauppauge HVR-950Q-MXL */ diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index 94553b61013c..58ef23146f20 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -585,19 +585,24 @@ static void tascam_disconnect(struct usb_interface *intf) return; if (intf->cur_altsetting->desc.bInterfaceNumber == 0) { - /* Ensure all deferred work is complete before freeing resources */ snd_card_disconnect(tascam->card); - cancel_work_sync(&tascam->stop_work); - cancel_work_sync(&tascam->capture_work); - cancel_work_sync(&tascam->midi_in_work); - cancel_work_sync(&tascam->midi_out_work); - cancel_work_sync(&tascam->stop_pcm_work); + /* + * Kill the URBs before cancelling the work, so a late URB + * completion cannot re-arm a work that then runs after + * snd_card_free(). + */ usb_kill_anchored_urbs(&tascam->playback_anchor); usb_kill_anchored_urbs(&tascam->capture_anchor); usb_kill_anchored_urbs(&tascam->feedback_anchor); usb_kill_anchored_urbs(&tascam->midi_in_anchor); usb_kill_anchored_urbs(&tascam->midi_out_anchor); + + cancel_work_sync(&tascam->stop_work); + cancel_work_sync(&tascam->capture_work); + cancel_work_sync(&tascam->midi_in_work); + cancel_work_sync(&tascam->midi_out_work); + cancel_work_sync(&tascam->stop_pcm_work); timer_delete_sync(&tascam->error_timer); tascam_free_urbs(tascam); snd_card_free(tascam->card); diff --git a/sound/usb/usx2y/us144mkii_capture.c b/sound/usb/usx2y/us144mkii_capture.c index af120bf62173..fa01da98151a 100644 --- a/sound/usb/usx2y/us144mkii_capture.c +++ b/sound/usb/usx2y/us144mkii_capture.c @@ -302,7 +302,6 @@ void capture_urb_complete(struct urb *urb) } usb_get_urb(urb); - usb_anchor_urb(urb, &tascam->capture_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err_ratelimited(tascam->card->dev, @@ -312,6 +311,7 @@ void capture_urb_complete(struct urb *urb) usb_put_urb(urb); atomic_dec( &tascam->active_urbs); /* Decrement on failed resubmission */ + return; } out: usb_put_urb(urb); diff --git a/tools/accounting/delaytop.c b/tools/accounting/delaytop.c index 72cc500b44b1..af8a1e42a055 100644 --- a/tools/accounting/delaytop.c +++ b/tools/accounting/delaytop.c @@ -75,13 +75,23 @@ {#name, #cmd, \ offsetof(struct task_info, name##_delay_total), \ offsetof(struct task_info, name##_count), \ + offsetof(struct task_info, name##_delay_max), \ + offsetof(struct task_info, name##_delay_max_ts), \ modes} -#define END_FIELD {NULL, 0, 0} +#define SORT_FIELD_NO_MAX(name, cmd, modes) \ + {#name, #cmd, \ + offsetof(struct task_info, name##_delay_total), \ + offsetof(struct task_info, name##_count), \ + 0, \ + 0, \ + modes} +#define END_FIELD {NULL, 0, 0, 0, 0, 0, 0} /* Display mode types */ #define MODE_TYPE_ALL (0xFFFFFFFF) #define MODE_DEFAULT (1 << 0) #define MODE_MEMVERBOSE (1 << 1) +#define MODE_TYPE (1 << 2) /* Display specific type with avg/max */ /* PSI statistics structure */ struct psi_stats { @@ -108,20 +118,36 @@ struct task_info { char command[TASK_COMM_LEN]; unsigned long long cpu_count; unsigned long long cpu_delay_total; + unsigned long long cpu_delay_max; + struct __kernel_timespec cpu_delay_max_ts; unsigned long long blkio_count; unsigned long long blkio_delay_total; + unsigned long long blkio_delay_max; + struct __kernel_timespec blkio_delay_max_ts; unsigned long long swapin_count; unsigned long long swapin_delay_total; + unsigned long long swapin_delay_max; + struct __kernel_timespec swapin_delay_max_ts; unsigned long long freepages_count; unsigned long long freepages_delay_total; + unsigned long long freepages_delay_max; + struct __kernel_timespec freepages_delay_max_ts; unsigned long long thrashing_count; unsigned long long thrashing_delay_total; + unsigned long long thrashing_delay_max; + struct __kernel_timespec thrashing_delay_max_ts; unsigned long long compact_count; unsigned long long compact_delay_total; + unsigned long long compact_delay_max; + struct __kernel_timespec compact_delay_max_ts; unsigned long long wpcopy_count; unsigned long long wpcopy_delay_total; + unsigned long long wpcopy_delay_max; + struct __kernel_timespec wpcopy_delay_max_ts; unsigned long long irq_count; unsigned long long irq_delay_total; + unsigned long long irq_delay_max; + struct __kernel_timespec irq_delay_max_ts; unsigned long long mem_count; unsigned long long mem_delay_total; }; @@ -141,6 +167,8 @@ struct field_desc { const char *cmd_char; /* Interactive command */ unsigned long total_offset; /* Offset of total delay in task_info */ unsigned long count_offset; /* Offset of count in task_info */ + unsigned long max_offset; /* Offset of max delay in task_info */ + unsigned long max_ts_offset; /* Offset of max delay timestamp in task_info */ size_t supported_modes; /* Supported display modes */ }; @@ -153,6 +181,7 @@ struct config { int monitor_pid; /* Monitor specific PID */ char *container_path; /* Path to container cgroup */ const struct field_desc *sort_field; /* Current sort field */ + const struct field_desc *type_field; /* Type field for -t option */ size_t display_mode; /* Current display mode */ }; @@ -164,15 +193,15 @@ static int task_count; static int running = 1; static struct container_stats container_stats; static const struct field_desc sort_fields[] = { - SORT_FIELD(cpu, c, MODE_DEFAULT), - SORT_FIELD(blkio, i, MODE_DEFAULT), - SORT_FIELD(irq, q, MODE_DEFAULT), - SORT_FIELD(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE), - SORT_FIELD(swapin, s, MODE_MEMVERBOSE), - SORT_FIELD(freepages, r, MODE_MEMVERBOSE), - SORT_FIELD(thrashing, t, MODE_MEMVERBOSE), - SORT_FIELD(compact, p, MODE_MEMVERBOSE), - SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE), + SORT_FIELD(cpu, c, MODE_DEFAULT | MODE_TYPE), + SORT_FIELD(blkio, i, MODE_DEFAULT | MODE_TYPE), + SORT_FIELD(irq, q, MODE_DEFAULT | MODE_TYPE), + SORT_FIELD_NO_MAX(mem, m, MODE_DEFAULT | MODE_MEMVERBOSE), + SORT_FIELD(swapin, s, MODE_MEMVERBOSE | MODE_TYPE), + SORT_FIELD(freepages, r, MODE_MEMVERBOSE | MODE_TYPE), + SORT_FIELD(thrashing, t, MODE_MEMVERBOSE | MODE_TYPE), + SORT_FIELD(compact, p, MODE_MEMVERBOSE | MODE_TYPE), + SORT_FIELD(wpcopy, w, MODE_MEMVERBOSE | MODE_TYPE), END_FIELD }; static int sort_selected; @@ -265,6 +294,8 @@ static void usage(void) " -p, --pid=PID Monitor only the specified PID\n" " -C, --container=PATH Monitor the container at specified cgroup path\n" " -s, --sort=FIELD Sort by delay field (default: cpu)\n" + " -t, --type=FIELD Display only specified delay type with avg/max/timestamp\n" + " (rows sorted by MAX for that type, largest first)\n" " -M, --memverbose Display memory detailed information\n"); exit(0); } @@ -283,6 +314,7 @@ static void parse_args(int argc, char **argv) {"processes", required_argument, 0, 'P'}, {"sort", required_argument, 0, 's'}, {"container", required_argument, 0, 'C'}, + {"type", required_argument, 0, 't'}, {"memverbose", no_argument, 0, 'M'}, {0, 0, 0, 0} }; @@ -292,6 +324,7 @@ static void parse_args(int argc, char **argv) cfg.iterations = 0; cfg.max_processes = 20; cfg.sort_field = &sort_fields[0]; /* Default sorted by CPU delay */ + cfg.type_field = NULL; /* No type field by default */ cfg.output_one_time = 0; cfg.monitor_pid = 0; /* 0 means monitor all PIDs */ cfg.container_path = NULL; @@ -300,7 +333,7 @@ static void parse_args(int argc, char **argv) while (1) { int option_index = 0; - c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index); + c = getopt_long(argc, argv, "hd:n:p:oP:C:s:t:M", long_options, &option_index); if (c == -1) break; @@ -363,9 +396,32 @@ static void parse_args(int argc, char **argv) cfg.sort_field = field; break; + case 't': + if (strlen(optarg) == 0) { + fprintf(stderr, "Error: empty type field\n"); + exit(1); + } + + field = get_field_by_name(optarg); + /* Show available fields if invalid option provided */ + if (!field || !(field->supported_modes & MODE_TYPE)) { + fprintf(stderr, "Error: invalid type field '%s'\n", optarg); + display_available_fields(MODE_TYPE); + exit(1); + } + + cfg.type_field = field; + cfg.display_mode = MODE_TYPE; + break; case 'M': cfg.display_mode = MODE_MEMVERBOSE; - cfg.sort_field = get_field_by_name("mem"); + /* Find first field supporting MODE_MEMVERBOSE for sorting */ + for (field = sort_fields; field->name != NULL; field++) { + if (field->supported_modes & MODE_MEMVERBOSE) { + cfg.sort_field = field; + break; + } + } break; default: fprintf(stderr, "Try 'delaytop --help' for more information.\n"); @@ -690,7 +746,12 @@ static void fetch_and_fill_task_info(int pid, const char *comm) nested_len = NLA_PAYLOAD(na->nla_len); while (nested_len > 0) { if (nested->nla_type == TASKSTATS_TYPE_STATS) { - memcpy(&stats, NLA_DATA(nested), sizeof(stats)); + size_t payload_len = NLA_PAYLOAD(nested->nla_len); + + memset(&stats, 0, sizeof(stats)); + if (payload_len > sizeof(stats)) + payload_len = sizeof(stats); + memcpy(&stats, NLA_DATA(nested), payload_len); if (task_count < MAX_TASKS) { tasks[task_count].pid = pid; tasks[task_count].tgid = pid; @@ -699,20 +760,36 @@ static void fetch_and_fill_task_info(int pid, const char *comm) tasks[task_count].command[TASK_COMM_LEN - 1] = '\0'; SET_TASK_STAT(task_count, cpu_count); SET_TASK_STAT(task_count, cpu_delay_total); + SET_TASK_STAT(task_count, cpu_delay_max); + SET_TASK_STAT(task_count, cpu_delay_max_ts); SET_TASK_STAT(task_count, blkio_count); SET_TASK_STAT(task_count, blkio_delay_total); + SET_TASK_STAT(task_count, blkio_delay_max); + SET_TASK_STAT(task_count, blkio_delay_max_ts); SET_TASK_STAT(task_count, swapin_count); SET_TASK_STAT(task_count, swapin_delay_total); + SET_TASK_STAT(task_count, swapin_delay_max); + SET_TASK_STAT(task_count, swapin_delay_max_ts); SET_TASK_STAT(task_count, freepages_count); SET_TASK_STAT(task_count, freepages_delay_total); + SET_TASK_STAT(task_count, freepages_delay_max); + SET_TASK_STAT(task_count, freepages_delay_max_ts); SET_TASK_STAT(task_count, thrashing_count); SET_TASK_STAT(task_count, thrashing_delay_total); + SET_TASK_STAT(task_count, thrashing_delay_max); + SET_TASK_STAT(task_count, thrashing_delay_max_ts); SET_TASK_STAT(task_count, compact_count); SET_TASK_STAT(task_count, compact_delay_total); + SET_TASK_STAT(task_count, compact_delay_max); + SET_TASK_STAT(task_count, compact_delay_max_ts); SET_TASK_STAT(task_count, wpcopy_count); SET_TASK_STAT(task_count, wpcopy_delay_total); + SET_TASK_STAT(task_count, wpcopy_delay_max); + SET_TASK_STAT(task_count, wpcopy_delay_max_ts); SET_TASK_STAT(task_count, irq_count); SET_TASK_STAT(task_count, irq_delay_total); + SET_TASK_STAT(task_count, irq_delay_max); + SET_TASK_STAT(task_count, irq_delay_max_ts); set_mem_count(&tasks[task_count]); set_mem_delay_total(&tasks[task_count]); task_count++; @@ -762,6 +839,15 @@ static void get_task_delays(void) closedir(dir); } +static void field_delay_max_and_ts(const struct task_info *task, + const struct field_desc *field, + unsigned long long *max_ns, + struct __kernel_timespec *max_ts); +static void get_field_delay_values(const struct task_info *task, + const struct field_desc *field, + double *avg_ms, double *max_ms, + struct __kernel_timespec *max_ts); + /* Calculate average delay in milliseconds */ static double average_ms(unsigned long long total, unsigned long long count) { @@ -770,6 +856,41 @@ static double average_ms(unsigned long long total, unsigned long long count) return (double)total / 1000000.0 / count; } +/* + * Format __kernel_timespec to human readable string (YYYY-MM-DDTHH:MM:SS) + * Returns formatted string or "N/A" if timestamp is zero + */ +static const char *format_kernel_timespec(struct __kernel_timespec *ts) +{ + static char buffer[32]; + time_t time_sec; + struct tm tm_info; + + /* Check if timestamp is zero (not set) */ + if (ts->tv_sec == 0 && ts->tv_nsec == 0) + return "N/A"; + + /* Avoid Y2038 truncation: check if timestamp fits in time_t on 32-bit platforms */ + if (sizeof(time_t) < sizeof(ts->tv_sec) && + ts->tv_sec > (__u64)((1ULL << (sizeof(time_t) * 8 - 1)) - 1)) + return "N/A"; + + time_sec = (time_t)ts->tv_sec; + + if (localtime_r(&time_sec, &tm_info) == NULL) + return "N/A"; + + snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02d", + tm_info.tm_year + 1900, + tm_info.tm_mon + 1, + tm_info.tm_mday, + tm_info.tm_hour, + tm_info.tm_min, + tm_info.tm_sec); + + return buffer; +} + /* Comparison function for sorting tasks */ static int compare_tasks(const void *a, const void *b) { @@ -777,14 +898,24 @@ static int compare_tasks(const void *a, const void *b) const struct task_info *t2 = (const struct task_info *)b; unsigned long long total1; unsigned long long total2; - unsigned long count1; - unsigned long count2; + unsigned long long count1; + unsigned long long count2; double avg1, avg2; + unsigned long long max1, max2; + + /* -t/--type: default sort by MAX column for the selected type (descending) */ + if (cfg.display_mode == MODE_TYPE && cfg.type_field) { + field_delay_max_and_ts(t1, cfg.type_field, &max1, NULL); + field_delay_max_and_ts(t2, cfg.type_field, &max2, NULL); + if (max1 != max2) + return max2 > max1 ? 1 : -1; + return 0; + } total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset); total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset); - count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset); - count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset); + count1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->count_offset); + count2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->count_offset); avg1 = average_ms(total1, count1); avg2 = average_ms(total2, count2); @@ -794,6 +925,50 @@ static int compare_tasks(const void *a, const void *b) return 0; } +/* Max delay (ns) and timestamp for field (shared by display and sort) */ +static void field_delay_max_and_ts(const struct task_info *task, const struct field_desc *field, + unsigned long long *max_ns, struct __kernel_timespec *max_ts) +{ + if (!field || !field->max_offset) { + *max_ns = 0; + if (max_ts) + memset(max_ts, 0, sizeof(*max_ts)); + return; + } + + *max_ns = *(unsigned long long *)((char *)task + field->max_offset); + + if (max_ts) { + if (field->max_ts_offset) + *max_ts = *(struct __kernel_timespec *)((char *)task + + field->max_ts_offset); + else + memset(max_ts, 0, sizeof(*max_ts)); + } +} + +/* Get delay values for a specific field */ +static void get_field_delay_values(const struct task_info *task, const struct field_desc *field, + double *avg_ms, double *max_ms, + struct __kernel_timespec *max_ts) +{ + unsigned long long total, count, max; + + if (!field || !field->max_offset) { + *avg_ms = 0; + *max_ms = 0; + memset(max_ts, 0, sizeof(*max_ts)); + return; + } + + total = *(unsigned long long *)((char *)task + field->total_offset); + count = *(unsigned long long *)((char *)task + field->count_offset); + *avg_ms = average_ms(total, count); + + field_delay_max_and_ts(task, field, &max, max_ts); + *max_ms = (double)max / 1000000.0; /* Convert nanoseconds to milliseconds */ +} + /* Sort tasks by selected field */ static void sort_tasks(void) { @@ -847,7 +1022,12 @@ static void get_container_stats(void) while (nl_len > 0) { if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) { /* Get the cgroupstats structure */ - memcpy(&stats, NLA_DATA(na), sizeof(stats)); + size_t payload_len = NLA_PAYLOAD(na->nla_len); + + memset(&stats, 0, sizeof(stats)); + if (payload_len > sizeof(stats)) + payload_len = sizeof(stats); + memcpy(&stats, NLA_DATA(na), payload_len); /* Fill container stats */ container_stats.nr_sleeping = stats.nr_sleeping; @@ -878,7 +1058,7 @@ static void display_results(int psi_ret) suc &= BOOL_FPRINT(out, "\033[H\033[J"); /* PSI output (one-line, no cat style) */ - suc &= BOOL_FPRINT(out, "System Pressure Information: (avg10/avg60vg300/total)\n"); + suc &= BOOL_FPRINT(out, "System Pressure Information: (avg10/avg60/avg300/total)\n"); if (psi_ret) { suc &= BOOL_FPRINT(out, " PSI not found: check if psi=1 enabled in cmdline\n"); } else { @@ -936,7 +1116,10 @@ static void display_results(int psi_ret) } /* Interacive command */ - suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n"); + if (cfg.display_mode == MODE_TYPE && cfg.type_field) + suc &= BOOL_FPRINT(out, "[q]quit\n"); + else + suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n"); if (sort_selected) { if (cfg.display_mode == MODE_MEMVERBOSE) suc &= BOOL_FPRINT(out, @@ -947,32 +1130,55 @@ static void display_results(int psi_ret) } /* Task delay output */ - suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n", - cfg.max_processes, get_name_by_field(cfg.sort_field)); - - suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND"); - if (cfg.display_mode == MODE_MEMVERBOSE) { - suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n", - "MEM(ms)", "SWAP(ms)", "RCL(ms)", - "THR(ms)", "CMP(ms)", "WP(ms)"); - suc &= BOOL_FPRINT(out, "-----------------------"); - suc &= BOOL_FPRINT(out, "-----------------------"); - suc &= BOOL_FPRINT(out, "-----------------------"); - suc &= BOOL_FPRINT(out, "---------------------\n"); + if (cfg.display_mode == MODE_TYPE && cfg.type_field) + suc &= BOOL_FPRINT(out, + "Top %d processes (sorted by %s MAX delay, largest first):\n", + cfg.max_processes, get_name_by_field(cfg.type_field)); + else + suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n", + cfg.max_processes, get_name_by_field(cfg.sort_field)); + + if (cfg.display_mode == MODE_TYPE && cfg.type_field) { + /* Display mode for -t option: show only specified type with avg/max/timestamp */ + suc &= BOOL_FPRINT(out, "%8s %8s %-17s %12s %12s %20s\n", + "PID", "TGID", "COMMAND", + "AVG(ms)", "MAX(ms)", "MAX_TIMESTAMP"); + suc &= BOOL_FPRINT(out, "--------------------------------------------------------"); + suc &= BOOL_FPRINT(out, "----------------------------------------\n"); } else { - suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n", - "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)"); - suc &= BOOL_FPRINT(out, "-----------------------"); - suc &= BOOL_FPRINT(out, "-----------------------"); - suc &= BOOL_FPRINT(out, "--------------------------\n"); + suc &= BOOL_FPRINT(out, "%8s %8s %-17s", "PID", "TGID", "COMMAND"); + if (cfg.display_mode == MODE_MEMVERBOSE) { + suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n", + "MEM(ms)", "SWAP(ms)", "RCL(ms)", + "THR(ms)", "CMP(ms)", "WP(ms)"); + suc &= BOOL_FPRINT(out, "-----------------------"); + suc &= BOOL_FPRINT(out, "-----------------------"); + suc &= BOOL_FPRINT(out, "-----------------------"); + suc &= BOOL_FPRINT(out, "---------------------\n"); + } else { + suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n", + "CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)"); + suc &= BOOL_FPRINT(out, "-----------------------"); + suc &= BOOL_FPRINT(out, "-----------------------"); + suc &= BOOL_FPRINT(out, "--------------------------\n"); + } } count = task_count < cfg.max_processes ? task_count : cfg.max_processes; for (i = 0; i < count; i++) { - suc &= BOOL_FPRINT(out, "%8d %8d %-15s", + suc &= BOOL_FPRINT(out, "%8d %8d %-17s", tasks[i].pid, tasks[i].tgid, tasks[i].command); - if (cfg.display_mode == MODE_MEMVERBOSE) { + if (cfg.display_mode == MODE_TYPE && cfg.type_field) { + double avg_ms, max_ms; + struct __kernel_timespec max_ts; + + get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms, + &max_ms, &max_ts); + + suc &= BOOL_FPRINT(out, "%12.2f %12.2f %20s\n", + avg_ms, max_ms, format_kernel_timespec(&max_ts)); + } else if (cfg.display_mode == MODE_MEMVERBOSE) { suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE, TASK_AVG(tasks[i], mem), TASK_AVG(tasks[i], swapin), @@ -1040,9 +1246,13 @@ static void handle_keypress(char ch, int *running) } else { switch (ch) { case 'o': + if (cfg.display_mode == MODE_TYPE) + break; sort_selected = 1; break; case 'M': + if (cfg.display_mode == MODE_TYPE) + break; toggle_display_mode(); for (field = sort_fields; field->name != NULL; field++) { if (field->supported_modes & cfg.display_mode) { diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 368a622ca027..6ac30d4f96f7 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -235,19 +235,18 @@ static const char *format_timespec(struct __kernel_timespec *ts) if (ts->tv_sec == 0 && ts->tv_nsec == 0) return "N/A"; + /* Avoid Y2038 truncation on 32-bit platforms */ + if (sizeof(time_sec) < sizeof(ts->tv_sec) && + ts->tv_sec > (__u64)((1ULL << (sizeof(time_sec) * 8 - 1)) - 1)) + return "N/A"; + time_sec = ts->tv_sec; /* Use thread-safe localtime_r */ if (localtime_r(&time_sec, &tm_info) == NULL) return "N/A"; - snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02d", - tm_info.tm_year + 1900, - tm_info.tm_mon + 1, - tm_info.tm_mday, - tm_info.tm_hour, - tm_info.tm_min, - tm_info.tm_sec); + strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &tm_info); return buffer; } diff --git a/tools/arch/x86/include/asm/amd/ibs.h b/tools/arch/x86/include/asm/amd/ibs.h index d0777b597322..6f25074e669f 100644 --- a/tools/arch/x86/include/asm/amd/ibs.h +++ b/tools/arch/x86/include/asm/amd/ibs.h @@ -99,7 +99,9 @@ union ibs_op_data2 { rmt_node:1, /* 4: destination node */ cache_hit_st:1, /* 5: cache hit state */ data_src_hi:2, /* 6-7: data source high */ - reserved1:56; /* 8-63: reserved */ + strm_st:1, /* 8: streaming store */ + rmt_socket:1, /* 9: remote socket */ + reserved1:54; /* 10-63: reserved */ }; }; diff --git a/tools/bootconfig/Makefile b/tools/bootconfig/Makefile index 90eb47c9d8de..3cb8066d5141 100644 --- a/tools/bootconfig/Makefile +++ b/tools/bootconfig/Makefile @@ -15,7 +15,7 @@ override CFLAGS += -Wall -g -I$(CURDIR)/include ALL_TARGETS := bootconfig ALL_PROGRAMS := $(patsubst %,$(OUTPUT)%,$(ALL_TARGETS)) -all: $(ALL_PROGRAMS) test +all: $(ALL_PROGRAMS) $(OUTPUT)bootconfig: main.c include/linux/bootconfig.h $(LIBSRC) $(CC) $(filter %.c,$^) $(CFLAGS) $(LDFLAGS) -o $@ @@ -27,4 +27,4 @@ install: $(ALL_PROGRAMS) install $(OUTPUT)bootconfig $(DESTDIR)$(bindir) clean: - $(RM) -f $(OUTPUT)*.o $(ALL_PROGRAMS) + rm -f $(OUTPUT)*.o $(ALL_PROGRAMS) diff --git a/tools/bootconfig/scripts/ftrace2bconf.sh b/tools/bootconfig/scripts/ftrace2bconf.sh index 1603801cf126..8eed445c295e 100755 --- a/tools/bootconfig/scripts/ftrace2bconf.sh +++ b/tools/bootconfig/scripts/ftrace2bconf.sh @@ -57,6 +57,8 @@ EOF kprobe_event_options() { cat $TRACEFS/kprobe_events | while read p args; do case $p in + \#*) + continue;; r*) cat 1>&2 << EOF # WARN: A return probe found but it is not supported by bootconfig. Skip it. diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 271a7dc77273..b0f7168e7943 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -99,7 +99,7 @@ endif HOST_LDFLAGS := $(LDFLAGS) # Remove warnings for libbpf bootstrap build -LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS)) +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 6ef908adf3a4..c9589026da8d 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -179,7 +179,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *m = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); @@ -193,7 +193,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, const char *name = btf_str(btf, m->name_off); __u32 bit_off, bit_sz; - if (BTF_INFO_KFLAG(t->info)) { + if (btf_kflag(t)) { bit_off = BTF_MEMBER_BIT_OFFSET(m->offset); bit_sz = BTF_MEMBER_BITFIELD_SIZE(m->offset); } else { @@ -224,7 +224,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_ENUM: { const struct btf_enum *v = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); const char *encoding; encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED"; @@ -300,8 +300,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, break; } case BTF_KIND_FWD: { - const char *fwd_kind = BTF_INFO_KFLAG(t->info) ? "union" - : "struct"; + const char *fwd_kind = btf_kflag(t) ? "union" : "struct"; if (json_output) jsonw_string_field(w, "fwd_kind", fwd_kind); @@ -322,7 +321,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_FUNC_PROTO: { const struct btf_param *p = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "ret_type_id", t->type); @@ -365,7 +364,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_DATASEC: { const struct btf_var_secinfo *v = (const void *)(t + 1); const struct btf_type *vt; - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 9dc8425b1789..e4075824343f 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -476,8 +476,8 @@ static int btf_dumper_struct(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - kind_flag = BTF_INFO_KFLAG(t->info); - vlen = BTF_INFO_VLEN(t->info); + kind_flag = btf_kflag(t); + vlen = btf_vlen(t); jsonw_start_object(d->jw); m = (struct btf_member *)(t + 1); @@ -535,7 +535,7 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - vlen = BTF_INFO_VLEN(t->info); + vlen = btf_vlen(t); vsi = (struct btf_var_secinfo *)(t + 1); jsonw_start_object(d->jw); @@ -557,7 +557,7 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id, { const struct btf_type *t = btf__type_by_id(d->btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: return btf_dumper_int(t, bit_offset, data, d->jw, d->is_plain_text); @@ -631,7 +631,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, t = btf__type_by_id(btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: case BTF_KIND_TYPEDEF: case BTF_KIND_FLOAT: @@ -661,7 +661,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, break; case BTF_KIND_FWD: BTF_PRINT_ARG("%s %s ", - BTF_INFO_KFLAG(t->info) ? "union" : "struct", + btf_kflag(t) ? "union" : "struct", btf__name_by_offset(btf, t->name_off)); break; case BTF_KIND_VOLATILE: @@ -718,7 +718,7 @@ static int btf_dump_func(const struct btf *btf, char *func_sig, BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off)); else BTF_PRINT_ARG("("); - vlen = BTF_INFO_VLEN(func_proto->info); + vlen = btf_vlen(func_proto); for (i = 0; i < vlen; i++) { struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i]; diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index ec356deb27c9..ce69d1e5468e 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c @@ -78,6 +78,13 @@ static unsigned int query_flags; static struct btf *btf_vmlinux; static __u32 btf_vmlinux_id; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; + btf_vmlinux_id = 0; +} + static enum bpf_attach_type parse_attach_type(const char *str) { const char *attach_type_str; @@ -388,6 +395,8 @@ static int do_show(int argc, char **argv) if (json_output) jsonw_end_array(json_wtr); + free_btf_vmlinux(); + exit_cgroup: close(cgroup_fd); exit: @@ -437,7 +446,9 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb, printf("%s\n", fpath); } - btf_vmlinux = libbpf_find_kernel_btf(); + if (!btf_vmlinux) + btf_vmlinux = libbpf_find_kernel_btf(); + for (i = 0; i < ARRAY_SIZE(cgroup_attach_types); i++) show_bpf_progs(cgroup_fd, cgroup_attach_types[i], ftw->level); @@ -540,6 +551,7 @@ static int do_show_tree(int argc, char **argv) if (json_output) jsonw_end_array(json_wtr); + free_btf_vmlinux(); free(cgroup_alloced); return ret; diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index bdcd717b0348..088d1d206065 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val) return (__u64 *) u64_to_ptr(val); } +static __u32 *u64_to_u32_arr(__u64 val) +{ + return (__u32 *)u64_to_ptr(val); +} + +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled) +{ + struct kernel_sym *sym; + + if (!addr) + return NULL; + + sym = kernel_syms_search(&dd, addr); + if (!sym && is_ibt_enabled && addr >= 4) + sym = kernel_syms_search(&dd, addr - 4); + + return sym; +} + static void show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -403,6 +422,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) jsonw_end_array(json_wtr); } +static void +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + show_link_attach_type_json(info->tracing_multi.attach_type, wtr); + jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count); + jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id); + jsonw_name(wtr, "funcs"); + + jsonw_start_array(wtr); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + struct kernel_sym *sym; + __u64 addr = addrs[i]; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + jsonw_start_object(wtr); + jsonw_uint_field(wtr, "id", ids[i]); + jsonw_uint_field(wtr, "addr", addr); + if (sym) { + jsonw_string_field(wtr, "func", sym->name); + if (sym->module[0] == '\0') { + jsonw_name(wtr, "module"); + jsonw_null(wtr); + } else { + jsonw_string_field(wtr, "module", sym->module); + } + } + jsonw_uint_field(wtr, "cookie", cookies[i]); + jsonw_end_object(wtr); + } + jsonw_end_array(wtr); +} + static void show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_json(info, json_wtr); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_json(info, json_wtr); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) } } +static void show_tracing_multi_plain(struct bpf_link_info *info) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!info->tracing_multi.count) + return; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + printf("\n\t"); + show_link_attach_type_plain(info->tracing_multi.attach_type); + printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id); + printf("count %u ", info->tracing_multi.count); + + printf("\n\t%-16s %-16s %-16s %s", + "btf_id", "addr", "cookie", "func [module]"); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + __u64 addr = addrs[i]; + struct kernel_sym *sym; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); + if (sym) { + printf(" %s", sym->name); + if (sym->module[0] != '\0') + printf(" [%s]", sym->module); + } + } +} + static void show_perf_event_kprobe_plain(struct bpf_link_info *info) { const char *buf; @@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_plain(info); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_plain(info); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) static int do_show_link(int fd) { __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL; + __u32 *ids = NULL; struct bpf_link_info info; __u32 len = sizeof(info); char path_buf[PATH_MAX]; @@ -1114,6 +1226,26 @@ static int do_show_link(int fd) goto again; } } + if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) { + count = info.tracing_multi.count; + if (count) { + ids = calloc(count, sizeof(__u32)); + addrs = calloc(count, sizeof(__u64)); + cookies = calloc(count, sizeof(__u64)); + if (!ids || !addrs || !cookies) { + p_err("mem alloc failed"); + close(fd); + free(cookies); + free(addrs); + free(ids); + return -ENOMEM; + } + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = ptr_to_u64(cookies); + goto again; + } + } if (info.type == BPF_LINK_TYPE_PERF_EVENT) { switch (info.perf_event.type) { case BPF_PERF_EVENT_TRACEPOINT: @@ -1153,6 +1285,7 @@ static int do_show_link(int fd) free(cookies); free(offsets); free(addrs); + free(ids); close(fd); return 0; } diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 71a45d96617e..6b9649294ca1 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -790,6 +790,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -958,7 +964,7 @@ static int do_dump(int argc, char **argv) close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1055,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1071,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249c..835e5e561f7f 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 2f192d3bf61b..ed1374af31c1 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -104,6 +104,9 @@ FEATURE_TESTS_BASIC := \ # FEATURE_TESTS_BASIC + FEATURE_TESTS_EXTRA is the complete list # of all feature tests + +LIBUNWIND_ARCHS:=aarch64 arm loongarch64 mips ppc32 ppc64 riscv s390x x86 x86_64 + FEATURE_TESTS_EXTRA := \ bionic \ compile-32 \ @@ -113,7 +116,7 @@ FEATURE_TESTS_EXTRA := \ gtk2 \ gtk2-infobar \ hello \ - libbabeltrace \ + babeltrace2-ctf-writer \ libcapstone \ libcheck \ libbfd-liberty \ @@ -127,7 +130,10 @@ FEATURE_TESTS_EXTRA := \ libpfm4 \ libdebuginfod \ clang-bpf-co-re \ - bpftool-skeletons + bpftool-skeletons \ + libunwind \ + libunwind-debug-frame \ + $(foreach arch,$(LIBUNWIND_ARCHS),libunwind-$(arch) libunwind-debug-frame-$(arch)) FEATURE_TESTS ?= $(FEATURE_TESTS_BASIC) @@ -163,6 +169,7 @@ FEATURE_GROUP_MEMBERS-libbfd = libbfd-liberty libbfd-liberty-z # Declare list of feature dependency packages that provide pkg-config files. # FEATURE_PKG_CONFIG ?= \ + babeltrace2-ctf-writer \ libtraceevent \ libtracefs @@ -211,7 +218,11 @@ ifeq ($(feature-all), 1) $(call feature_check,compile-32) $(call feature_check,compile-x32) $(call feature_check,bionic) - $(call feature_check,libbabeltrace) + $(call feature_check,babeltrace2-ctf-writer) + $(call feature_check,libunwind) + $(call feature_check,libunwind-debug-frame) + $(foreach arch,$(LIBUNWIND_ARCHS),$(call feature_check,libunwind-$(arch))) + $(foreach arch,$(LIBUNWIND_ARCHS),$(call feature_check,libunwind-debug-frame-$(arch))) else $(foreach feat,$(FEATURE_TESTS),$(call feature_check,$(feat))) endif diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 704c687ed3ad..62909a9c799d 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 include ../../scripts/Makefile.include +LIBUNWIND_ARCHS:=aarch64 arm loongarch64 mips ppc32 ppc64 riscv s390x x86 x86_64 + FILES= \ test-all.bin \ test-backtrace.bin \ @@ -38,17 +40,12 @@ FILES= \ test-libtracefs.bin \ test-libunwind.bin \ test-libunwind-debug-frame.bin \ - test-libunwind-x86.bin \ - test-libunwind-x86_64.bin \ - test-libunwind-arm.bin \ - test-libunwind-aarch64.bin \ - test-libunwind-debug-frame-arm.bin \ - test-libunwind-debug-frame-aarch64.bin \ + $(foreach arch,$(LIBUNWIND_ARCHS),test-libunwind-$(arch).bin test-libunwind-debug-frame-$(arch).bin) \ test-pthread-attr-setaffinity-np.bin \ test-pthread-barrier.bin \ test-stackprotector-all.bin \ test-timerfd.bin \ - test-libbabeltrace.bin \ + test-babeltrace2-ctf-writer.bin \ test-libcapstone.bin \ test-libcheck.bin \ test-compile-32.bin \ @@ -75,7 +72,8 @@ FILES= \ test-file-handle.bin \ test-libpfm4.bin \ test-rust.bin \ - test-libopenssl.bin + test-libopenssl.bin \ + test-bpftool-skeletons.bin FILES := $(addprefix $(OUTPUT),$(FILES)) @@ -211,27 +209,26 @@ $(OUTPUT)test-numa_num_possible_cpus.bin: $(BUILD) -lnuma $(OUTPUT)test-libunwind.bin: - $(BUILD) -lelf -llzma + $(BUILD) -include libunwind.h -lelf -llzma -lunwind $(OUTPUT)test-libunwind-debug-frame.bin: - $(BUILD) -lelf -llzma -$(OUTPUT)test-libunwind-x86.bin: - $(BUILD) -lelf -llzma -lunwind-x86 - -$(OUTPUT)test-libunwind-x86_64.bin: - $(BUILD) -lelf -llzma -lunwind-x86_64 + $(BUILD) -include libunwind.h -lelf -llzma -lunwind -$(OUTPUT)test-libunwind-arm.bin: - $(BUILD) -lelf -llzma -lunwind-arm +define LIBUNWIND_RULE +$$(OUTPUT)test-libunwind-$(1).bin: + $$(CC) $$(CFLAGS) -MD -Wall -Werror -include libunwind-$(1).h -o $$@ \ + test-libunwind.c $$(LDFLAGS) -lelf -llzma -lunwind-$(1) \ + > $$(@:.bin=.make.output) 2>&1 -$(OUTPUT)test-libunwind-aarch64.bin: - $(BUILD) -lelf -llzma -lunwind-aarch64 +$$(OUTPUT)test-libunwind-debug-frame-$(1).bin: + $$(CC) $$(CFLAGS) -MD -Wall -Werror -include libunwind-$(1).h -o $$@ \ + test-libunwind-debug-frame.c $$(LDFLAGS) -lelf -llzma -lunwind-$(1) \ + > $$(@:.bin=.make.output) 2>&1 -$(OUTPUT)test-libunwind-debug-frame-arm.bin: - $(BUILD) -lelf -llzma -lunwind-arm - -$(OUTPUT)test-libunwind-debug-frame-aarch64.bin: - $(BUILD) -lelf -llzma -lunwind-aarch64 +endef +$(foreach arch,$(LIBUNWIND_ARCHS), \ + $(eval $(call LIBUNWIND_RULE,$(arch))) \ +) $(OUTPUT)test-libslang.bin: $(BUILD) -lslang @@ -308,8 +305,8 @@ $(OUTPUT)test-backtrace.bin: $(OUTPUT)test-timerfd.bin: $(BUILD) -$(OUTPUT)test-libbabeltrace.bin: - $(BUILD) # -lbabeltrace provided by $(FEATURE_CHECK_LDFLAGS-libbabeltrace) +$(OUTPUT)test-babeltrace2-ctf-writer.bin: + $(BUILD) # -lbabeltrace2-ctf-writer provided by $(FEATURE_CHECK_LDFLAGS-babeltrace2-ctf-writer) $(OUTPUT)test-libcapstone.bin: $(BUILD) # -lcapstone provided by $(FEATURE_CHECK_LDFLAGS-libcapstone) @@ -383,9 +380,9 @@ $(OUTPUT)test-libaio.bin: $(OUTPUT)test-libzstd.bin: $(BUILD) -lzstd -$(OUTPUT)test-clang-bpf-co-re.bin: - $(CLANG) -S -g --target=bpf -o - $(patsubst %.bin,%.c,$(@F)) | \ - grep BTF_KIND_VAR +$(OUTPUT)test-clang-bpf-co-re.bin: test-clang-bpf-co-re.c + { $(CLANG) -S -g --target=bpf -o - $< | \ + grep BTF_KIND_VAR; } > $(@:.bin=.make.output) 2>&1 && touch $@ $(OUTPUT)test-file-handle.bin: $(BUILD) @@ -397,8 +394,8 @@ $(OUTPUT)test-libopenssl.bin: $(BUILD) $(shell $(PKG_CONFIG) --libs --cflags openssl 2>/dev/null) $(OUTPUT)test-bpftool-skeletons.bin: - $(SYSTEM_BPFTOOL) version | grep '^features:.*skeletons' \ - > $(@:.bin=.make.output) 2>&1 + { $(SYSTEM_BPFTOOL) version | grep '^features:.*skeletons'; } \ + > $(@:.bin=.make.output) 2>&1 && touch $@ # Testing Rust is special: we don't compile anything, it's enough to check the # compiler presence. Compiling a test code for this purposes is problematic, diff --git a/tools/build/feature/test-all.c b/tools/build/feature/test-all.c index 1488bf6e6078..544563d62950 100644 --- a/tools/build/feature/test-all.c +++ b/tools/build/feature/test-all.c @@ -100,13 +100,13 @@ # if 0 /* - * Disable libbabeltrace check for test-all, because the requested + * Disable babeltrace2-ctf-writer check for test-all, because the requested * library version is not released yet in most distributions. Will * reenable later. */ -#define main main_test_libbabeltrace -# include "test-libbabeltrace.c" +#define main main_test_babeltrace2_ctf_writer +# include "test-babeltrace2-ctf-writer.c" #undef main #endif diff --git a/tools/build/feature/test-libbabeltrace.c b/tools/build/feature/test-babeltrace2-ctf-writer.c similarity index 59% rename from tools/build/feature/test-libbabeltrace.c rename to tools/build/feature/test-babeltrace2-ctf-writer.c index 10bb69d55694..9c89082e9f88 100644 --- a/tools/build/feature/test-libbabeltrace.c +++ b/tools/build/feature/test-babeltrace2-ctf-writer.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include +#include int main(void) { diff --git a/tools/build/feature/test-libunwind-aarch64.c b/tools/build/feature/test-libunwind-aarch64.c deleted file mode 100644 index 323803f49212..000000000000 --- a/tools/build/feature/test-libunwind-aarch64.c +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -static unw_accessors_t accessors; - -int main(void) -{ - unw_addr_space_t addr_space; - - addr_space = unw_create_addr_space(&accessors, 0); - if (addr_space) - return 0; - - unw_init_remote(NULL, addr_space, NULL); - dwarf_search_unwind_table(addr_space, 0, NULL, NULL, 0, NULL); - - return 0; -} diff --git a/tools/build/feature/test-libunwind-arm.c b/tools/build/feature/test-libunwind-arm.c deleted file mode 100644 index cb378b7d6866..000000000000 --- a/tools/build/feature/test-libunwind-arm.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -static unw_accessors_t accessors; - -int main(void) -{ - unw_addr_space_t addr_space; - - addr_space = unw_create_addr_space(&accessors, 0); - if (addr_space) - return 0; - - unw_init_remote(NULL, addr_space, NULL); - dwarf_search_unwind_table(addr_space, 0, NULL, NULL, 0, NULL); - - return 0; -} diff --git a/tools/build/feature/test-libunwind-debug-frame-aarch64.c b/tools/build/feature/test-libunwind-debug-frame-aarch64.c deleted file mode 100644 index 36d6646c185e..000000000000 --- a/tools/build/feature/test-libunwind-debug-frame-aarch64.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int -UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, - unw_word_t ip, unw_word_t segbase, - const char *obj_name, unw_word_t start, - unw_word_t end); - -#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) - -int main(void) -{ - dwarf_find_debug_frame(0, NULL, 0, 0, NULL, 0, 0); - return 0; -} diff --git a/tools/build/feature/test-libunwind-debug-frame-arm.c b/tools/build/feature/test-libunwind-debug-frame-arm.c deleted file mode 100644 index 8696e48e1268..000000000000 --- a/tools/build/feature/test-libunwind-debug-frame-arm.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int -UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, - unw_word_t ip, unw_word_t segbase, - const char *obj_name, unw_word_t start, - unw_word_t end); - -#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) - -int main(void) -{ - dwarf_find_debug_frame(0, NULL, 0, 0, NULL, 0, 0); - return 0; -} diff --git a/tools/build/feature/test-libunwind-debug-frame.c b/tools/build/feature/test-libunwind-debug-frame.c index efb55cdd8d01..4c57e37004b3 100644 --- a/tools/build/feature/test-libunwind-debug-frame.c +++ b/tools/build/feature/test-libunwind-debug-frame.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include extern int diff --git a/tools/build/feature/test-libunwind-x86.c b/tools/build/feature/test-libunwind-x86.c deleted file mode 100644 index e5e0f6c89637..000000000000 --- a/tools/build/feature/test-libunwind-x86.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -static unw_accessors_t accessors; - -int main(void) -{ - unw_addr_space_t addr_space; - - addr_space = unw_create_addr_space(&accessors, 0); - if (addr_space) - return 0; - - unw_init_remote(NULL, addr_space, NULL); - dwarf_search_unwind_table(addr_space, 0, NULL, NULL, 0, NULL); - - return 0; -} diff --git a/tools/build/feature/test-libunwind-x86_64.c b/tools/build/feature/test-libunwind-x86_64.c deleted file mode 100644 index 62ae4db597dc..000000000000 --- a/tools/build/feature/test-libunwind-x86_64.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -static unw_accessors_t accessors; - -int main(void) -{ - unw_addr_space_t addr_space; - - addr_space = unw_create_addr_space(&accessors, 0); - if (addr_space) - return 0; - - unw_init_remote(NULL, addr_space, NULL); - dwarf_search_unwind_table(addr_space, 0, NULL, NULL, 0, NULL); - - return 0; -} diff --git a/tools/build/feature/test-libunwind.c b/tools/build/feature/test-libunwind.c index 53fd26614ff0..5af5dc3a73d4 100644 --- a/tools/build/feature/test-libunwind.c +++ b/tools/build/feature/test-libunwind.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, diff --git a/tools/cgroup/iocost_monitor.py b/tools/cgroup/iocost_monitor.py index 933c750b319b..bdd78ba271b0 100644 --- a/tools/cgroup/iocost_monitor.py +++ b/tools/cgroup/iocost_monitor.py @@ -15,11 +15,6 @@ import json import math -import drgn -from drgn import container_of -from drgn.helpers.linux.list import list_for_each_entry,list_empty -from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -34,6 +29,11 @@ help='Output in json') args = parser.parse_args() +import drgn +from drgn import container_of +from drgn.helpers.linux.list import list_for_each_entry,list_empty +from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup + def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) diff --git a/tools/iio/iio_event_monitor.c b/tools/iio/iio_event_monitor.c index 03ca33869ce8..bc3ef4c77c2b 100644 --- a/tools/iio/iio_event_monitor.c +++ b/tools/iio/iio_event_monitor.c @@ -65,6 +65,7 @@ static const char * const iio_chan_type_name_spec[] = { [IIO_CHROMATICITY] = "chromaticity", [IIO_ATTENTION] = "attention", [IIO_ALTCURRENT] = "altcurrent", + [IIO_COVERAGE] = "coverage", }; static const char * const iio_ev_type_text[] = { @@ -145,6 +146,7 @@ static const char * const iio_modifier_names[] = { [IIO_MOD_ACTIVE] = "active", [IIO_MOD_REACTIVE] = "reactive", [IIO_MOD_APPARENT] = "apparent", + [IIO_MOD_QUATERNION_AXIS] = "quaternionaxis", }; static bool event_is_known(struct iio_event_data *event) @@ -193,6 +195,7 @@ static bool event_is_known(struct iio_event_data *event) case IIO_CHROMATICITY: case IIO_ATTENTION: case IIO_ALTCURRENT: + case IIO_COVERAGE: break; default: return false; diff --git a/tools/iio/iio_generic_buffer.c b/tools/iio/iio_generic_buffer.c index bc82bb6a7a2a..000193612aad 100644 --- a/tools/iio/iio_generic_buffer.c +++ b/tools/iio/iio_generic_buffer.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -89,12 +90,19 @@ static void print1byte(uint8_t input, struct iio_channel_info *info) */ input >>= info->shift; input &= info->mask; - if (info->is_signed) { + switch (info->format) { + case 's': { int8_t val = (int8_t)(input << (8 - info->bits_used)) >> (8 - info->bits_used); printf("%05f ", ((float)val + info->offset) * info->scale); - } else { + break; + } + case 'u': printf("%05f ", ((float)input + info->offset) * info->scale); + break; + case 'f': + printf(" "); + break; } } @@ -112,12 +120,30 @@ static void print2byte(uint16_t input, struct iio_channel_info *info) */ input >>= info->shift; input &= info->mask; - if (info->is_signed) { + switch (info->format) { + case 's': { int16_t val = (int16_t)(input << (16 - info->bits_used)) >> (16 - info->bits_used); printf("%05f ", ((float)val + info->offset) * info->scale); - } else { + break; + } + case 'u': printf("%05f ", ((float)input + info->offset) * info->scale); + break; + case 'f': { +#if defined(__FLT16_MAX__) + union { + uint16_t u; + _Float16 f; + } converter; + + converter.u = input; + printf("%05f ", ((float)converter.f + info->offset) * info->scale); +#else + printf(" "); +#endif + break; + } } } @@ -135,12 +161,26 @@ static void print4byte(uint32_t input, struct iio_channel_info *info) */ input >>= info->shift; input &= info->mask; - if (info->is_signed) { + switch (info->format) { + case 's': { int32_t val = (int32_t)(input << (32 - info->bits_used)) >> (32 - info->bits_used); printf("%05f ", ((float)val + info->offset) * info->scale); - } else { + break; + } + case 'u': printf("%05f ", ((float)input + info->offset) * info->scale); + break; + case 'f': { + union { + uint32_t u; + float f; + } converter; + + converter.u = input; + printf("%05f ", (converter.f + info->offset) * info->scale); + break; + } } } @@ -158,7 +198,8 @@ static void print8byte(uint64_t input, struct iio_channel_info *info) */ input >>= info->shift; input &= info->mask; - if (info->is_signed) { + switch (info->format) { + case 's': { int64_t val = (int64_t)(input << (64 - info->bits_used)) >> (64 - info->bits_used); /* special case for timestamp */ @@ -167,8 +208,21 @@ static void print8byte(uint64_t input, struct iio_channel_info *info) else printf("%05f ", ((float)val + info->offset) * info->scale); - } else { + break; + } + case 'u': printf("%05f ", ((float)input + info->offset) * info->scale); + break; + case 'f': { + union { + uint64_t u; + double f; + } converter; + + converter.u = input; + printf("%05f ", (converter.f + info->offset) * info->scale); + break; + } } } diff --git a/tools/iio/iio_utils.c b/tools/iio/iio_utils.c index c5c5082cb24e..e274a0d8376b 100644 --- a/tools/iio/iio_utils.c +++ b/tools/iio/iio_utils.c @@ -70,7 +70,7 @@ int iioutils_break_up_name(const char *full_name, char **generic_name) /** * iioutils_get_type() - find and process _type attribute data - * @is_signed: output whether channel is signed + * @format: output channel format * @bytes: output how many bytes the channel storage occupies * @bits_used: output number of valid bits of data * @shift: output amount of bits to shift right data before applying bit mask @@ -83,7 +83,7 @@ int iioutils_break_up_name(const char *full_name, char **generic_name) * * Returns a value >= 0 on success, otherwise a negative error code. **/ -static int iioutils_get_type(unsigned int *is_signed, unsigned int *bytes, +static int iioutils_get_type(char *format, unsigned int *bytes, unsigned int *bits_used, unsigned int *shift, uint64_t *mask, unsigned int *be, const char *device_dir, int buffer_idx, @@ -93,7 +93,7 @@ static int iioutils_get_type(unsigned int *is_signed, unsigned int *bytes, int ret; DIR *dp; char *scan_el_dir, *builtname, *builtname_generic, *filename = 0; - char signchar, endianchar; + char formatchar, endianchar; unsigned padint; const struct dirent *ent; @@ -140,7 +140,7 @@ static int iioutils_get_type(unsigned int *is_signed, unsigned int *bytes, ret = fscanf(sysfsfp, "%ce:%c%u/%u>>%u", &endianchar, - &signchar, + &formatchar, bits_used, &padint, shift); if (ret < 0) { @@ -162,7 +162,7 @@ static int iioutils_get_type(unsigned int *is_signed, unsigned int *bytes, else *mask = (1ULL << *bits_used) - 1ULL; - *is_signed = (signchar == 's'); + *format = formatchar; if (fclose(sysfsfp)) { ret = -errno; fprintf(stderr, "Failed to close %s\n", @@ -487,7 +487,7 @@ int build_channel_array(const char *device_dir, int buffer_idx, if ((ret < 0) && (ret != -ENOENT)) goto error_cleanup_array; - ret = iioutils_get_type(¤t->is_signed, + ret = iioutils_get_type(¤t->format, ¤t->bytes, ¤t->bits_used, ¤t->shift, diff --git a/tools/iio/iio_utils.h b/tools/iio/iio_utils.h index 663c94a6c705..8c72f002d050 100644 --- a/tools/iio/iio_utils.h +++ b/tools/iio/iio_utils.h @@ -32,7 +32,7 @@ extern const char *iio_dir; * @shift: amount of bits to shift right data before applying bit mask * @mask: a bit mask for the raw output * @be: flag if data is big endian - * @is_signed: is the raw value stored signed + * @format: format of the raw value * @location: data offset for this channel inside the buffer (in bytes) **/ struct iio_channel_info { @@ -46,7 +46,7 @@ struct iio_channel_info { unsigned shift; uint64_t mask; unsigned be; - unsigned is_signed; + char format; unsigned location; }; diff --git a/tools/include/linux/bits.h b/tools/include/linux/bits.h index a40cc861b3a7..b7509f15718e 100644 --- a/tools/include/linux/bits.h +++ b/tools/include/linux/bits.h @@ -17,7 +17,7 @@ * position @h. For example * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) /* * Missing asm support @@ -75,7 +75,7 @@ #define BIT_U32(nr) BIT_TYPE(u32, nr) #define BIT_U64(nr) BIT_TYPE(u64, nr) -#else /* defined(__ASSEMBLY__) */ +#else /* defined(__ASSEMBLER__) */ /* * BUILD_BUG_ON_ZERO is not available in h files included from asm files, @@ -84,6 +84,6 @@ #define GENMASK(h, l) __GENMASK(h, l) #define GENMASK_ULL(h, l) __GENMASK_ULL(h, l) -#endif /* !defined(__ASSEMBLY__) */ +#endif /* !defined(__ASSEMBLER__) */ #endif /* __LINUX_BITS_H */ diff --git a/tools/include/linux/btf_ids.h b/tools/include/linux/btf_ids.h index 72ea363d434d..4fe5c5f1558c 100644 --- a/tools/include/linux/btf_ids.h +++ b/tools/include/linux/btf_ids.h @@ -10,6 +10,9 @@ struct btf_id_set { u32 ids[]; }; +/* This flag implies BTF_SET8 holds kfunc(s) */ +#define BTF_SET8_KFUNCS (1 << 0) + struct btf_id_set8 { u32 cnt; u32 flags; @@ -22,6 +25,7 @@ struct btf_id_set8 { #ifdef CONFIG_DEBUG_INFO_BTF #include /* for __PASTE */ +#include /* * Following macros help to define lists of BTF IDs placed @@ -35,7 +39,7 @@ struct btf_id_set8 { #define BTF_IDS_SECTION ".BTF_ids" -#define ____BTF_ID(symbol) \ +#define ____BTF_ID(symbol, word) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ ".local " #symbol " ; \n" \ @@ -43,10 +47,11 @@ asm( \ ".size " #symbol ", 4; \n" \ #symbol ": \n" \ ".zero 4 \n" \ +word \ ".popsection; \n"); -#define __BTF_ID(symbol) \ - ____BTF_ID(symbol) +#define __BTF_ID(symbol, word) \ + ____BTF_ID(symbol, word) #define __ID(prefix) \ __PASTE(__PASTE(prefix, __COUNTER__), __LINE__) @@ -56,7 +61,14 @@ asm( \ * to 4 zero bytes. */ #define BTF_ID(prefix, name) \ - __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__)) + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "") + +#define ____BTF_ID_FLAGS(prefix, name, flags) \ + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n") +#define __BTF_ID_FLAGS(prefix, name, flags, ...) \ + ____BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) \ + __BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0) /* * The BTF_ID_LIST macro defines pure (unsorted) list @@ -155,10 +167,58 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set name; +/* + * The BTF_SET8_START/END macros pair defines sorted list of + * BTF IDs and their flags plus its members count, with the + * following layout: + * + * BTF_SET8_START(list) + * BTF_ID_FLAGS(type1, name1, flags) + * BTF_ID_FLAGS(type2, name2, flags) + * BTF_SET8_END(list) + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__type1__name1__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__type2__name2__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) + * + */ +#define __BTF_SET8_START(name, scope, flags) \ +__BTF_ID_LIST(name, local) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +"." #scope " __BTF_ID__set8__" #name "; \n" \ +"__BTF_ID__set8__" #name ":; \n" \ +".zero 4 \n" \ +".long " __stringify(flags) "\n" \ +".popsection; \n"); + +#define BTF_SET8_START(name) \ +__BTF_SET8_START(name, local, 0) + +#define BTF_SET8_END(name) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +".size __BTF_ID__set8__" #name ", .-" #name " \n" \ +".popsection; \n"); \ +extern struct btf_id_set8 name; + +#define BTF_KFUNCS_START(name) \ +__BTF_SET8_START(name, local, BTF_SET8_KFUNCS) + +#define BTF_KFUNCS_END(name) \ +BTF_SET8_END(name) + #else -#define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; +#define BTF_ID_LIST(name) static u32 __maybe_unused name[128]; #define BTF_ID(prefix, name) +#define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -166,6 +226,10 @@ extern struct btf_id_set name; #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) +#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) +#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS }; +#define BTF_KFUNCS_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ @@ -215,5 +279,9 @@ MAX_BTF_TRACING_TYPE, }; extern u32 btf_tracing_ids[]; +extern u32 bpf_cgroup_btf_id[]; +extern u32 bpf_local_storage_map_btf_id[]; +extern u32 btf_bpf_map_id[]; +extern u32 bpf_kmem_cache_btf_id[]; #endif diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index f40bd2b04c29..f2f54b038168 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -119,7 +119,7 @@ #define __read_mostly #ifndef __attribute_const__ -# define __attribute_const__ +# define __attribute_const__ __attribute__ ((__const__)) #endif #ifndef __maybe_unused diff --git a/tools/include/linux/overflow.h b/tools/include/linux/overflow.h index 3427d7880326..98963688143f 100644 --- a/tools/include/linux/overflow.h +++ b/tools/include/linux/overflow.h @@ -1,4 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 OR MIT */ +#include #ifndef __LINUX_OVERFLOW_H #define __LINUX_OVERFLOW_H diff --git a/tools/include/nolibc/arch-sparc.h b/tools/include/nolibc/arch-sparc.h index ddae9bc10dfe..23fab40accfa 100644 --- a/tools/include/nolibc/arch-sparc.h +++ b/tools/include/nolibc/arch-sparc.h @@ -45,7 +45,7 @@ \ __asm__ volatile ( \ _NOLIBC_SYSCALL \ - : "+r"(_arg1) \ + : "=r"(_arg1) \ : "r"(_num) \ : "memory", "cc" \ ); \ diff --git a/tools/include/nolibc/unistd.h b/tools/include/nolibc/unistd.h index 79599ceef45d..a264a20da13d 100644 --- a/tools/include/nolibc/unistd.h +++ b/tools/include/nolibc/unistd.h @@ -73,6 +73,48 @@ int ftruncate(int fd, off_t length) return __sysret(_sys_ftruncate(fd, length)); } +/* + * char *getcwd(char *buf, size_t size); + */ + +static __attribute__((unused)) +int _sys_getcwd(char *buf, size_t size) +{ + return __nolibc_syscall2(__NR_getcwd, buf, size); +} + +static __attribute__((unused)) +char *getcwd(char *buf, size_t size) +{ + int ret; + + /* Unlike other libc's we don't handle passing NULL for buf */ + if (!buf || !size) { + SET_ERRNO(EINVAL); + return NULL; + } + + ret = __sysret(_sys_getcwd(buf, size)); + + /* On error return NULL, __sysret() above will have set errno */ + if (ret < 0) + return NULL; + + /* Handle no path being written or the kernel putting + * "(unreachable)" into the buffer instead of a path. + * This matches what musl is doing. + */ + if (ret == 0 || buf[0] != '/') { + SET_ERRNO(ENOENT); + return NULL; + } + + /* ret must be the number of bytes written at this point, + * so return the pointer to buf. + */ + return buf; +} + static __attribute__((unused)) int msleep(unsigned int msecs) { @@ -86,6 +128,22 @@ int msleep(unsigned int msecs) return 0; } +/* + * ssize_t readlink(const char *path, char *buf, size_t bufsiz); + */ + +static __attribute__((unused)) +ssize_t _sys_readlink(const char *path, char *buf, size_t bufsiz) +{ + return __nolibc_syscall4(__NR_readlinkat, AT_FDCWD, path, buf, bufsiz); +} + +static __attribute__((unused)) +ssize_t readlink(const char *path, char *buf, size_t bufsiz) +{ + return __sysret(_sys_readlink(path, buf, bufsiz)); +} + static __attribute__((unused)) unsigned int sleep(unsigned int seconds) { diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 89b36de5fdbb..2f1d24fef857 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -5079,17 +5079,19 @@ union bpf_attr { * Description * Redirect the packet to another net device of index *ifindex*. * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. + * that the redirection happens to the *ifindex*' peer device. If + * *flags* is 0, the netns switch takes place from ingress to + * ingress without going through the CPU's backlog queue. If the + * **BPF_F_EGRESS** flag is provided then redirection happens in + * the egress direction of the peer device. * * *skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during * the netns switch. * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. + * If the *flags* argument is 0, the helper is currently only + * supported for tc BPF program types at the ingress hook and for + * veth and netkit target device types. The peer device must reside + * in a different network namespace. * Return * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. @@ -6336,9 +6338,10 @@ enum { /* Flags for bpf_redirect and bpf_redirect_map helpers */ enum { BPF_F_INGRESS = (1ULL << 0), /* used for skb path */ + BPF_F_EGRESS = (1ULL << 1), /* used for skb path */ BPF_F_BROADCAST = (1ULL << 3), /* used for XDP path */ BPF_F_EXCLUDE_INGRESS = (1ULL << 4), /* used for XDP path */ -#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) +#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) }; #define __bpf_md_ptr(type, name) \ @@ -6839,6 +6842,15 @@ struct bpf_link_info { __u32 flags; __u32 pid; } uprobe_multi; + struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; diff --git a/tools/include/uapi/linux/fscrypt.h b/tools/include/uapi/linux/fscrypt.h index 3aff99f2696a..84507280b3ea 100644 --- a/tools/include/uapi/linux/fscrypt.h +++ b/tools/include/uapi/linux/fscrypt.h @@ -30,7 +30,6 @@ #define FSCRYPT_MODE_SM4_CTS 8 #define FSCRYPT_MODE_ADIANTUM 9 #define FSCRYPT_MODE_AES_256_HCTR2 10 -/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */ /* * Legacy policy version; ad-hoc KDF and no key verification. diff --git a/tools/lib/api/fs/fs.c b/tools/lib/api/fs/fs.c index edec23406dbc..cbd8eab0d1df 100644 --- a/tools/lib/api/fs/fs.c +++ b/tools/lib/api/fs/fs.c @@ -261,8 +261,8 @@ static const char *mount_overload(struct fs *fs) /* "PERF_" + name + "_ENVIRONMENT" + '\0' */ char upper_name[5 + name_len + 12 + 1]; - snprintf(upper_name, name_len, "PERF_%s_ENVIRONMENT", fs->name); - mem_toupper(upper_name, name_len); + snprintf(upper_name, sizeof(upper_name), "PERF_%s_ENVIRONMENT", fs->name); + mem_toupper(upper_name, strlen(upper_name)); return getenv(upper_name) ?: *fs->mounts; } @@ -294,11 +294,14 @@ int filename__read_int(const char *filename, int *value) { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = atoi(line); err = 0; } @@ -312,11 +315,14 @@ static int filename__read_ull_base(const char *filename, { char line[64]; int fd = open(filename, O_RDONLY), err = -1; + ssize_t n; if (fd < 0) return -errno; - if (read(fd, line, sizeof(line)) > 0) { + n = read(fd, line, sizeof(line) - 1); + if (n > 0) { + line[n] = '\0'; *value = strtoull(line, NULL, base); if (*value != ULLONG_MAX) err = 0; @@ -370,12 +376,13 @@ int filename__write_int(const char *filename, int value) { int fd = open(filename, O_WRONLY), err = -1; char buf[64]; + int len; if (fd < 0) return -errno; - sprintf(buf, "%d", value); - if (write(fd, buf, sizeof(buf)) == sizeof(buf)) + len = sprintf(buf, "%d", value); + if (write(fd, buf, len) == len) err = 0; close(fd); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 823bce895178..bf6a68118405 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -589,7 +589,7 @@ static int btf_parse_type_sec(struct btf *btf) if (type_size < 0) return type_size; if (next_type + type_size > end_type) { - pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); + pr_warn("BTF type [%u] is malformed\n", btf->start_id + btf->nr_types); return -EINVAL; } @@ -1424,7 +1424,7 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs continue; if (sh.sh_type != SHT_PROGBITS) { - pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n", + pr_warn("unexpected section type (%u) of section(%d, %s) from %s\n", sh.sh_type, idx, name, path); goto err; } @@ -4854,7 +4854,7 @@ static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2, continue; if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) { if (t1->name_off) { - pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", + pr_debug("%s '%s' size=%u vlen=%u id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, t1->name_off), t1->size, btf_vlen(t1), id1, id2, i, @@ -5104,7 +5104,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); if (eq <= 0) { if (cand_type->name_off) { - pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", + pr_debug("%s '%s' size=%u vlen=%u cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, cand_type->name_off), cand_type->size, vlen, cand_id, canon_id, i, @@ -6069,7 +6069,7 @@ static int btf_add_distilled_types(struct btf_distill *dist) err = btf_add_type(&dist->pipe, t); break; default: - pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", + pr_warn("unexpected kind when adding base type '%s'[%d] of kind [%d] to distilled base BTF.\n", name, i, kind); return -EINVAL; diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index cc1ba65bb6c5..123c448f20c7 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1776,7 +1776,7 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { - pr_warn("unexpected bitfield size %d\n", t->size); + pr_warn("unexpected bitfield size %u\n", t->size); return -EINVAL; } @@ -2251,7 +2251,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = is_signed ? *(__s8 *)data : *(__u8 *)data; return 0; default: - pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); + pr_warn("unexpected size %u for enum, id:[%u]\n", t->size, id); return -EINVAL; } } diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index 53d1f3541bce..df5fa4bd87d6 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c @@ -280,7 +280,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) cmp_btf_name_size(&base_info, dist_info) == 0; dist_info++) { if (!dist_info->id || dist_info->id >= r->nr_dist_base_types) { - pr_warn("base BTF id [%d] maps to invalid distilled base BTF id [%d]\n", + pr_warn("base BTF id [%u] maps to invalid distilled base BTF id [%u]\n", id, dist_info->id); err = -EINVAL; goto done; @@ -368,7 +368,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) continue; dist_t = btf_type_by_id(r->dist_base_btf, id); name = btf__name_by_offset(r->dist_base_btf, dist_t->name_off); - pr_warn("distilled base BTF type '%s' [%d] is not mapped to base BTF id\n", + pr_warn("distilled base BTF type '%s' [%u] is not mapped to base BTF id\n", name, id); err = -EINVAL; break; @@ -397,11 +397,11 @@ static int btf_relocate_validate_distilled_base(struct btf_relocate *r) case BTF_KIND_FWD: if (t->name_off) break; - pr_warn("type [%d], kind [%d] is invalid for distilled base BTF; it is anonymous\n", + pr_warn("type [%u], kind [%d] is invalid for distilled base BTF; it is anonymous\n", i, kind); return -EINVAL; default: - pr_warn("type [%d] in distilled based BTF has unexpected kind [%d]\n", + pr_warn("type [%u] in distilled based BTF has unexpected kind [%d]\n", i, kind); return -EINVAL; } diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index 295dbda24580..fe136d025967 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -354,7 +354,7 @@ long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name) if (ret > 0) { pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, - ret); + (unsigned long)ret); } else { if (ret == 0) { pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0b..e5641fa60163 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd) static int probe_uprobe_syscall(int token_fd) { /* - * If kernel supports uprobe() syscall, it will return -ENXIO when called + * If kernel supports uprobe() syscall, it will return -EPROTO when called * from the outside of a kernel-generated uprobe trampoline. */ - return syscall(__NR_uprobe) < 0 && errno == ENXIO; + return syscall(__NR_uprobe) < 0 && errno == EPROTO; } #else static int probe_uprobe_syscall(int token_fd) diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index d79695f01c87..c7f2d2ac7bb3 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -384,7 +384,7 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) int i; if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", + pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n", nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); gen->error = -EFAULT; return gen->error; @@ -488,7 +488,7 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, attr.btf_size = tgt_endian(btf_raw_size); btf_load_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n", btf_data, btf_raw_size, btf_load_attr, attr_size); /* populate union bpf_attr with user provided log details */ @@ -534,7 +534,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); map_create_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", + pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n", map_name, map_idx, map_type, map_attr->btf_value_type_id, map_create_attr, attr_size); @@ -1082,7 +1082,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", + pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n", prog_idx, prog_type, insns_off, insn_cnt, license_off); /* convert blob insns to target endianness */ @@ -1105,21 +1105,21 @@ void bpf_gen__prog_load(struct bpf_gen *gen, attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); - pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n", func_info, load_attr->func_info_cnt, load_attr->func_info_rec_size); attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); - pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n", line_info, load_attr->line_info_cnt, load_attr->line_info_rec_size); attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); - pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", + pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n", core_relos, gen->core_relo_cnt, sizeof(struct bpf_core_relo)); @@ -1234,7 +1234,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, } map_update_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, blob_fd_array_off(gen, map_idx)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c..f88b7c8de304 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -1486,7 +1486,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, type->size); st_ops->type_id = type_id; - pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", + pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n", tname, type_id, var_name, vsi->offset); } @@ -2626,7 +2626,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': key type [%d] not found.\n", + pr_warn("map '%s': key type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2666,7 +2666,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': value type [%d] not found.\n", + pr_warn("map '%s': value type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2720,7 +2720,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, map_def->value_size = 4; t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': %s type [%d] not found.\n", + pr_warn("map '%s': %s type [%u] not found.\n", map_name, desc, m->type); return -EINVAL; } @@ -3476,7 +3476,7 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, var_name = btf__name_by_offset(btf, t_var->name_off); if (!var_name) { - pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", + pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n", sec_name, i); return -ENOENT; } @@ -3971,7 +3971,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!data) return -LIBBPF_ERRNO__FORMAT; - pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", + pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n", idx, name, (unsigned long)data->d_size, (int)sh->sh_link, (unsigned long)sh->sh_flags, (int)sh->sh_type); @@ -4494,7 +4494,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.data_off = roundup(off, ext->kcfg.align); off = ext->kcfg.data_off + ext->kcfg.sz; - pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", + pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n", i, ext->sym_idx, ext->kcfg.data_off, ext->name); } sec->size = off; @@ -4626,7 +4626,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, struct bpf_map *map; if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { - pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", + pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n", prog->name, sym_name, insn_idx, insn->code); return -LIBBPF_ERRNO__RELOC; } @@ -4749,7 +4749,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map->sec_idx != sym->st_shndx || map->sec_offset != sym->st_value) continue; - pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", + pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4776,7 +4776,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map = &obj->maps[map_idx]; if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) continue; - pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", + pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4985,7 +4985,7 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) info->value_size = val; else if (sscanf(buff, "max_entries:\t%u", &val) == 1) info->max_entries = val; - else if (sscanf(buff, "map_flags:\t%i", &val) == 1) + else if (sscanf(buff, "map_flags:\t%x", &val) == 1) info->map_flags = val; } @@ -5172,12 +5172,8 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = obj->token_fd, - .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, - ); - if (obj->gen_loader) + if (obj->gen_loader || obj->token_fd) return 0; ret = bump_rlimit_memlock(); @@ -5186,9 +5182,9 @@ bpf_object__probe_loading(struct bpf_object *obj) errstr(ret)); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n", @@ -5521,11 +5517,11 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) } if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n", map->name, i, targ_map->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n", map->name, i, targ_map->name, fd); } @@ -5554,11 +5550,11 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) err = bpf_map_update_elem(map->fd, &i, &fd, 0); if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n", map->name, i, targ_prog->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n", map->name, i, targ_prog->name, fd); } @@ -5788,7 +5784,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, if (strncmp(local_name, targ_name, local_essent_len) != 0) continue; - pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", + pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n", local_cand->id, btf_kind_str(local_t), local_name, i, btf_kind_str(t), targ_name, targ_btf_name); @@ -5848,7 +5844,7 @@ static int load_module_btfs(struct bpf_object *obj) if (errno == ENOENT) continue; /* expected race: BTF was unloaded */ err = -errno; - pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err)); return err; } @@ -5861,7 +5857,7 @@ static int load_module_btfs(struct bpf_object *obj) err = bpf_btf_get_info_by_fd(fd, &info, &len); if (err) { err = -errno; - pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err)); break; } @@ -5874,7 +5870,7 @@ static int load_module_btfs(struct bpf_object *obj) btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", + pr_warn("failed to load module [%s]'s BTF object #%u: %s\n", name, id, errstr(err)); break; } @@ -6067,7 +6063,7 @@ static int bpf_core_resolve_relo(struct bpf_program *prog, !hashmap__find(cand_cache, local_id, &cands)) { cands = bpf_core_find_cands(prog->obj, local_btf, local_id); if (IS_ERR(cands)) { - pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", + pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), local_name, PTR_ERR(cands)); return PTR_ERR(cands); @@ -6127,7 +6123,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) goto out; } - pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); + pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { if (rec->insn_off % BPF_INSN_SZ) @@ -6181,7 +6177,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); if (err) { - pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n", + pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n", prog->name, i, insn_idx, errstr(err)); goto out; } @@ -6346,7 +6342,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc goto err_close; } if (sym_off + jt_size > obj->jumptables_data_sz) { - pr_warn("map '.jumptables': jumptables_data size is %zd, trying to access %d\n", + pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n", obj->jumptables_data_sz, sym_off + jt_size); err = -EINVAL; goto err_close; @@ -6381,7 +6377,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc */ if (insn_off > UINT32_MAX) { pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n", - (long long)jt[i], sym_off + i * jt_entry_size); + (unsigned long long)jt[i], sym_off + i * jt_entry_size); err = -EINVAL; goto err_close; } @@ -6517,7 +6513,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) } break; default: - pr_warn("prog '%s': relo #%d: bad relo type %d\n", + pr_warn("prog '%s': relo #%d: bad relo type %u\n", prog->name, i, relo->type); return -EINVAL; } @@ -6797,7 +6793,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, */ continue; if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { - pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", + pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n", prog->name, insn_idx, relo->type); return -LIBBPF_ERRNO__RELOC; } @@ -7587,7 +7583,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } name = elf_sym_str(obj, sym->st_name) ?: ""; - pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", + pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n", i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, (size_t)rel->r_offset, sym->st_name, name); @@ -7678,7 +7674,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; - pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", + pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n", i, map->name, moff, type, name); } @@ -8738,7 +8734,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, local_name = btf__name_by_offset(obj->btf, local_type->name_off); targ_name = btf__name_by_offset(btf, targ_type->name_off); - pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", + pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n", ext->name, local_type_id, btf_kind_str(local_type), local_name, targ_type_id, btf_kind_str(targ_type), targ_name); @@ -8915,7 +8911,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, if (err) return err; pr_debug("extern (kcfg) '%s': set to 0x%llx\n", - ext->name, (long long)value); + ext->name, (unsigned long long)value); } else { pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; @@ -10494,7 +10490,7 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, moff = rel->r_offset - map->sec_offset; shdr_idx = sym->st_shndx; st_ops = map->st_ops; - pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", + pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n", map->name, (long long)(rel->r_info >> 32), (long long)sym->st_value, @@ -10643,7 +10639,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t memset(&info, 0, info_len); err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); if (err) { - pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n", + pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n", attach_prog_fd, errstr(err)); return err; } @@ -10656,7 +10652,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t btf = btf_load_from_kernel(info.btf_id, NULL, token_fd); err = libbpf_get_error(btf); if (err) { - pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err)); + pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err)); goto out; } err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); @@ -10738,7 +10734,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac } err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); if (err < 0) { - pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n", + pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n", prog->name, attach_prog_fd, attach_name, errstr(err)); return err; } @@ -11233,7 +11229,7 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz, } if (value_sz != num_cpu * elem_sz) { - pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", + pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n", map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); return -EINVAL; } @@ -11774,7 +11770,7 @@ static void gen_probe_legacy_event_name(char *buf, size_t buf_sz, static int index = 0; int i; - snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(), + snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(), __sync_fetch_and_add(&index, 1), name, offset); /* sanitize name in the probe name */ @@ -12924,8 +12920,8 @@ static long elf_find_func_offset_from_archive(const char *archive_path, const ch ret = elf_find_func_offset(elf, file_name, func_name); if (ret > 0) { pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", - func_name, file_name, archive_path, entry.data_offset, ret, - ret + entry.data_offset); + func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret, + (unsigned long)(ret + entry.data_offset)); ret += entry.data_offset; } elf_end(elf); @@ -14570,7 +14566,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx) break; } default: - pr_warn("unknown perf sample type %d\n", e->type); + pr_warn("unknown perf sample type %u\n", e->type); return LIBBPF_PERF_EVENT_ERROR; } return LIBBPF_PERF_EVENT_CONT; diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index 06663f9ea581..007fe17d17b4 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -123,7 +123,7 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, if (tb[type]) { pr_warn("Attribute of type %#x found multiple times in message, " - "previous attribute is being ignored.\n", type); + "previous attribute is being ignored.\n", (unsigned)type); } tb[type] = nla; diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6ae3f2a15ad0..8ad2715721cf 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -216,7 +216,7 @@ int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_type), local_id, targ_id); return 0; } @@ -384,7 +384,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%u] of unexpected kind %s\n", prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } @@ -725,7 +725,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EINVAL; *val = sz; } else { - pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", + pr_warn("prog '%s': relo %u at insn #%u can't be applied to array access\n", prog_name, relo->kind, relo->insn_off / 8); return -EINVAL; } @@ -747,7 +747,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { if (byte_sz >= 8) { /* bitfield can't be read with 64-bit read */ - pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", + pr_warn("prog '%s': relo %u at insn #%u can't be satisfied for bitfield\n", prog_name, relo->kind, relo->insn_off / 8); return -E2BIG; } @@ -971,7 +971,7 @@ static int bpf_core_calc_relo(const char *prog_name, err = 0; } else if (err == -EOPNOTSUPP) { /* EOPNOTSUPP means unknown/unsupported relocation */ - pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", + pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%u) at insn #%u\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind, relo->insn_off / 8); } @@ -1067,7 +1067,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->imm, (unsigned long long)orig_val, (unsigned long long)new_val); @@ -1083,7 +1083,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, case BPF_ST: case BPF_STX: if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, (unsigned long long)new_val); return -EINVAL; @@ -1159,7 +1159,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, default: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, - insn->src_reg, insn->dst_reg, insn->off, insn->imm); + (unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm); return -EINVAL; } @@ -1323,7 +1323,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, const char *spec_str; spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", + pr_warn("prog '%s': relo #%d: parsing [%u] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), str_is_empty(local_name) ? "" : local_name, spec_str ?: "", err); @@ -1346,7 +1346,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, /* libbpf doesn't support candidate search for anonymous types */ if (str_is_empty(local_name)) { - pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", + pr_warn("prog '%s': relo #%d: <%s> (%u) relocation doesn't support anonymous types\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); return -EOPNOTSUPP; } @@ -1697,7 +1697,7 @@ int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const st goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_t), local_id, targ_id); return 0; } diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 57fb82bb81b5..2e56e3ab5b6c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) /* * Detect kernel support for uprobe() syscall, it's presence means we can - * take advantage of faster nop5 uprobe handling. + * take advantage of faster nop10 uprobe handling. * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") */ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); @@ -327,7 +327,7 @@ static int sanity_check_usdt_elf(Elf *elf, const char *path) int endianness; if (elf_kind(elf) != ELF_K_ELF) { - pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); + pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path); return -EBADF; } @@ -438,8 +438,9 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz } pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", - i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, - (long)phdr.p_type, (long)phdr.p_flags); + i, path, + (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset, + (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags); if (phdr.p_type != PT_LOAD) continue; @@ -604,14 +605,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, #if defined(__x86_64__) static bool has_nop_combo(int fd, long off) { - unsigned char nop_combo[6] = { - 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; - unsigned char buf[6]; + unsigned char buf[11]; - if (pread(fd, buf, 6, off) != 6) + if (pread(fd, buf, 11, off) != 11) return false; - return memcmp(buf, nop_combo, 6) == 0; + return memcmp(buf, nop_combo, 11) == 0; } #else static bool has_nop_combo(int fd, long off) @@ -719,14 +720,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_abs_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip); goto err_out; } if (!seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - usdt_abs_ip); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)usdt_abs_ip); goto err_out; } /* translate from virtual address to file offset */ @@ -766,7 +767,7 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_rel_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip); goto err_out; } @@ -775,8 +776,10 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, - note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, - seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); + (unsigned long)note.loc_addr, (unsigned long)note.base_addr, + (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args, + (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0), + (unsigned long)(seg ? seg->offset : 0)); /* Adjust semaphore address to be a file offset */ if (note.sema_addr) { @@ -791,14 +794,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", - usdt_provider, usdt_name, path, note.sema_addr); + usdt_provider, usdt_name, path, (unsigned long)note.sema_addr); goto err_out; } if (seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - note.sema_addr); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)note.sema_addr); goto err_out; } @@ -806,8 +809,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", - path, note.sema_addr, note.base_addr, usdt_sema_off, - seg->start, seg->end, seg->offset); + path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off, + (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset); } /* Record adjusted addresses and offsets and parse USDT spec */ @@ -822,8 +825,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, memset(target, 0, sizeof(*target)); /* - * We have uprobe syscall and usdt with nop,nop5 instructions combo, - * so we can place the uprobe directly on nop5 (+1) and get this probe + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe * optimized. */ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { @@ -1117,7 +1120,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct spec_id, usdt_provider, usdt_name, path); } else { pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", - target->abs_ip, spec_id, usdt_provider, usdt_name, + (unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name, path, errstr(err)); } goto err_out; diff --git a/tools/lib/perf/TODO b/tools/lib/perf/TODO new file mode 100644 index 000000000000..e179728697d8 --- /dev/null +++ b/tools/lib/perf/TODO @@ -0,0 +1,30 @@ +Future ABI changes +================== + +This file collects items that require a libperf ABI bump. Each entry +should describe the current limitation, the desired end state, and the +scope of the change so that a future ABI revision can batch them +together. + +1. Widen struct perf_cpu.cpu from int16_t to int + - Current limit: 32767 CPUs. No architecture exceeds this today + (x86_64 max is 8192, arm64 is 4096), but NR_CPUS limits keep + growing. perf clamps to INT16_MAX in set_max_cpu_num() as a + safety net. + - Code simplification: the int16_t forces defensive truncation + checks at every boundary where a wider CPU index (int from + sample->cpu, al->cpu, etc.) is narrowed into struct perf_cpu. + Without these checks, values > 32767 silently wrap to negative + numbers (two's complement), bypassing bounds validation. + Widening to int eliminates this entire class of silent + truncation bugs and removes the need for the INT16_MAX clamp + in set_max_cpu_num(). + - Scope: struct perf_cpu is embedded everywhere — perf_cpu_map__cpu(), + perf_cpu_map__min(), perf_cpu_map__max(), perf_cpu_map__has(), the + for_each_cpu macros, and all internal callers. The perf_cpu_map + internal array (RC_CHK_ACCESS(map)->map[]) stores struct perf_cpu + directly. Widening changes the struct layout and every function + that returns or accepts struct perf_cpu by value. + - Migration: bump LIBPERF version in libperf.map, audit all + sizeof(struct perf_cpu) assumptions, update perf.data + serialization if needed. diff --git a/tools/lib/perf/include/perf/cpumap.h b/tools/lib/perf/include/perf/cpumap.h index a1dd25db65b6..e1a0b0d27210 100644 --- a/tools/lib/perf/include/perf/cpumap.h +++ b/tools/lib/perf/include/perf/cpumap.h @@ -6,7 +6,13 @@ #include #include -/** A wrapper around a CPU to avoid confusion with the perf_cpu_map's map's indices. */ +/** + * struct perf_cpu - wrapper around a CPU number. + * @cpu: CPU number, -1 for the "any CPU"/dummy value. + * + * int16_t limits this to 32767 CPUs. Widening to int requires a libperf + * ABI bump — see tools/lib/perf/TODO for the full scope. + */ struct perf_cpu { int16_t cpu; }; diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index 9043dc72b5d6..fdced574c889 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -8,7 +8,14 @@ #include #include /* pid_t */ -#define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem)) +/* + * Verify the full field fits within the event, not just its start offset. + * Only valid for fixed-size scalar fields — for trailing arrays like + * filename[PATH_MAX], sizeof() evaluates to the declared maximum, not + * the actual string length, so this would spuriously return false. + */ +#define event_contains(obj, mem) \ + ((obj).header.size >= offsetof(typeof(obj), mem) + sizeof((obj).mem)) struct perf_record_mmap { struct perf_event_header header; diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index de107ab4a281..618b0d765ef5 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -624,7 +624,7 @@ class ManFormat(OutputFormat): ``manual`` Defaults to ``Kernel API Manual``. - The above controls the output of teh corresponding fields on troff + The above controls the output of the corresponding fields on troff title headers, which will be filled like this:: .TH "{name}" {section} "{date}" "{modulename}" "{manual}" diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py index aab70e5eaa6f..4251f7c6673a 100644 --- a/tools/lib/python/kdoc/xforms_lists.py +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -49,6 +49,7 @@ class CTransforms: (CMatch("DEFINE_DMA_UNMAP_ADDR"), r"dma_addr_t \1"), (CMatch("DEFINE_DMA_UNMAP_LEN"), r"__u32 \1"), (CMatch("VIRTIO_DECLARE_FEATURES"), r"union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }"), + (CMatch("__SYSFS_FUNCTION_ALTERNATIVE"), r"union { \1+ }"), (CMatch("__attribute__"), ""), # @@ -102,6 +103,8 @@ class CTransforms: (CMatch("__no_context_analysis"), ""), (CMatch("__attribute_const__"), ""), (CMatch("__attribute__"), ""), + (CMatch("STATIC_IFN_KUNIT"), ""), + (CMatch("INLINE_IFN_KUNIT"), ""), # # HACK: this is similar to process_export() hack. It is meant to @@ -118,6 +121,7 @@ class CTransforms: (CMatch("__guarded_by"), ""), (CMatch("__pt_guarded_by"), ""), (CMatch("LIST_HEAD"), r"struct list_head \1"), + (CMatch("DECLARE_PER_CPU"), r"\1 \2[PER_CPU]; }"), (KernRe(r"(?://.*)$"), ""), (KernRe(r"(?:/\*.*\*/)"), ""), diff --git a/tools/lib/subcmd/run-command.c b/tools/lib/subcmd/run-command.c index b7510f83209a..bd21b8bfd58b 100644 --- a/tools/lib/subcmd/run-command.c +++ b/tools/lib/subcmd/run-command.c @@ -169,8 +169,18 @@ int start_command(struct child_process *cmd) static int wait_or_whine(struct child_process *cmd, bool block) { - bool finished = cmd->finished; - int result = cmd->finish_result; + bool finished; + int result; + + if (cmd->pid <= 0) { + cmd->finished = 1; + if (cmd->pid < 0 && cmd->finish_result == 0) + cmd->finish_result = -ERR_RUN_COMMAND_FORK; + return cmd->finish_result; + } + + finished = cmd->finished; + result = cmd->finish_result; while (!finished) { int status, code; @@ -233,7 +243,18 @@ int check_if_command_finished(struct child_process *cmd) char filename[6 + MAX_STRLEN_TYPE(typeof(cmd->pid)) + 7 + 1]; char status_line[256]; FILE *status_file; +#endif + if (cmd->finished) + return 1; + if (cmd->pid <= 0) { + cmd->finished = 1; + if (cmd->pid < 0 && cmd->finish_result == 0) + cmd->finish_result = -ERR_RUN_COMMAND_FORK; + return 1; + } + +#ifdef __linux__ /* * Check by reading /proc//status as calling waitpid causes * stdout/stderr to be closed and data lost. @@ -241,8 +262,48 @@ int check_if_command_finished(struct child_process *cmd) sprintf(filename, "/proc/%u/status", cmd->pid); status_file = fopen(filename, "r"); if (status_file == NULL) { - /* Open failed assume finish_command was called. */ - return true; + int status; + pid_t waiting; + + /* + * fopen() can fail with ENOENT if the process has been reaped. + * It can also fail with EMFILE/ENFILE if RLIMIT_NOFILE is reached. + * In those cases, use waitpid(..., WNOHANG) to robustly check + * and reap the process if it has exited. + */ + if (errno == ENOENT) + return 1; + + waiting = waitpid(cmd->pid, &status, WNOHANG); + if (waiting == cmd->pid) { + int result; + int code; + + cmd->finished = 1; + if (WIFSIGNALED(status)) { + result = -ERR_RUN_COMMAND_WAITPID_SIGNAL; + } else if (!WIFEXITED(status)) { + result = -ERR_RUN_COMMAND_WAITPID_NOEXIT; + } else { + code = WEXITSTATUS(status); + switch (code) { + case 127: + result = -ERR_RUN_COMMAND_EXEC; + break; + case 0: + result = 0; + break; + default: + result = -code; + break; + } + } + cmd->finish_result = result; + return 1; + } + if (waiting < 0 && (errno == ECHILD || errno == ESRCH)) + return 1; + return 0; } while (fgets(status_line, sizeof(status_line), status_file) != NULL) { char *p; diff --git a/tools/lib/symbol/kallsyms.c b/tools/lib/symbol/kallsyms.c index e335ac2b9e19..d64bd9cc82a9 100644 --- a/tools/lib/symbol/kallsyms.c +++ b/tools/lib/symbol/kallsyms.c @@ -60,7 +60,7 @@ int kallsyms__parse(const char *filename, void *arg, read_to_eol(&io); continue; } - for (i = 0; i < sizeof(symbol_name); i++) { + for (i = 0; i < KSYM_NAME_LEN; i++) { ch = io__get_char(&io); if (ch < 0 || ch == '\n') break; @@ -68,6 +68,9 @@ int kallsyms__parse(const char *filename, void *arg, } symbol_name[i] = '\0'; + if (i == KSYM_NAME_LEN) + read_to_eol(&io); + err = process_symbol(arg, symbol_name, symbol_type, start); if (err) break; diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore index 922879f93fc8..1446a659e540 100644 --- a/tools/mm/.gitignore +++ b/tools/mm/.gitignore @@ -2,3 +2,4 @@ slabinfo page-types page_owner_sort +thp_swap_allocator_test diff --git a/tools/mm/Makefile b/tools/mm/Makefile index f5725b5c23aa..858186a6eefd 100644 --- a/tools/mm/Makefile +++ b/tools/mm/Makefile @@ -3,7 +3,7 @@ # include ../scripts/Makefile.include -BUILD_TARGETS=page-types slabinfo page_owner_sort thp_swap_allocator_test +BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps LIB_DIR = ../lib/api @@ -23,7 +23,7 @@ $(LIBS): $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) clean: - $(RM) page-types slabinfo page_owner_sort thp_swap_allocator_test + $(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test make -C $(LIB_DIR) clean sbindir ?= /usr/sbin diff --git a/tools/mm/page_owner_filter.c b/tools/mm/page_owner_filter.c new file mode 100644 index 000000000000..1d1f0a38678a --- /dev/null +++ b/tools/mm/page_owner_filter.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * User-space helper to filter page_owner output per-fd + * + * Example use: + * ./page_owner_filter -m handle + * ./page_owner_filter -m stack_handle + * ./page_owner_filter -n 0,1,2 + * + * See Documentation/mm/page_owner.rst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_CMD_LEN 512 + +static void usage(const char *prog) +{ + fprintf(stderr, "Usage: %s [OPTIONS]\n", prog); + fprintf(stderr, "\nOptions:\n"); + fprintf(stderr, " -m, --mode MODE : print_mode (stack, handle, or stack_handle)\n"); + fprintf(stderr, " -n, --nid NID_LIST : NUMA node IDs (comma-separated or ranges)\n"); + fprintf(stderr, " -o, --output FILE : output file (default: stdout)\n"); + fprintf(stderr, " -h, --help : show this help message\n"); + fprintf(stderr, "\nExamples:\n"); + fprintf(stderr, " %s -m stack\n", prog); + fprintf(stderr, " %s -m handle\n", prog); + fprintf(stderr, " %s -m stack_handle\n", prog); + fprintf(stderr, " %s -m stack -o output.txt\n", prog); + fprintf(stderr, " %s -n 0,1,2\n", prog); + fprintf(stderr, " %s -m stack -n 0\n", prog); +} + +static int validate_mode(const char *mode) +{ + if (strcmp(mode, "stack") == 0 || + strcmp(mode, "handle") == 0 || + strcmp(mode, "stack_handle") == 0) + return 0; + + fprintf(stderr, "Error: Invalid mode '%s'\n", mode); + fprintf(stderr, "Valid modes: stack, handle, stack_handle\n"); + return -1; +} + +static int validate_nid_list(const char *nid_list) +{ + const char *p; + int i = 0; + int has_digit = 0; + int in_range = 0; + int prev_num = 0; + int curr_num = 0; + + if (!nid_list || strlen(nid_list) == 0) + return 0; + + for (p = nid_list; *p; p++) { + if (*p == ',') { + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + i = 0; + has_digit = 0; + in_range = 0; + prev_num = 0; + curr_num = 0; + continue; + } + + if (*p == '-') { + if (!has_digit) { + fprintf(stderr, + "Error: Invalid nid_list format "); + fprintf(stderr, + "(dash without preceding number)\n"); + return -1; + } + if (in_range) { + fprintf(stderr, "Error: Multiple dashes in nid_list\n"); + return -1; + } + prev_num = curr_num; + curr_num = 0; + i = 0; + has_digit = 0; + in_range = 1; + continue; + } + + if (!isdigit((unsigned char)*p)) { + fprintf(stderr, "Error: Invalid character '%c' in nid_list\n", *p); + return -1; + } + + if (i > 5) { + fprintf(stderr, "Error: NID too long (max 65536)\n"); + return -1; + } + curr_num = curr_num * 10 + (*p - '0'); + i++; + has_digit = 1; + } + + if (!has_digit) { + fprintf(stderr, "Error: Invalid nid_list format\n"); + return -1; + } + + if (in_range && prev_num > curr_num) { + fprintf(stderr, + "Error: Invalid range %d-%d (start must be <= end)\n", + prev_num, curr_num); + return -1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *output_file = NULL; + char filter_cmd[MAX_CMD_LEN]; + FILE *output = NULL; + int fd = -1; + ssize_t ret; + char buf[4096]; + int opt; + size_t cmd_len = 0; + + signal(SIGPIPE, SIG_IGN); + + static struct option long_options[] = { + {"mode", required_argument, 0, 'm'}, + {"nid", required_argument, 0, 'n'}, + {"output", required_argument, 0, 'o'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + filter_cmd[0] = '\0'; + + if (argc > 1) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(argv[0]); + return 0; + } + } + } + + /* Check if page_owner exists and is readable */ + if (access("/sys/kernel/debug/page_owner", F_OK) != 0) { + if (errno == ENOENT) + fprintf(stderr, "Error: /sys/kernel/debug/page_owner does not exist\n"); + else + perror("Error accessing /sys/kernel/debug/page_owner"); + fprintf(stderr, "Make sure page_owner is enabled in kernel\n"); + return 1; + } + + while ((opt = getopt_long(argc, argv, "m:n:o:h", long_options, NULL)) != -1) { + int len; + + switch (opt) { + case 'm': { + const char *mode = optarg; + + if (validate_mode(mode) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%smode=%s", cmd_len > 0 ? " " : "", mode); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'n': { + const char *nid_list = optarg; + + if (validate_nid_list(nid_list) < 0) + return 1; + len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len, + "%snid=%s", cmd_len > 0 ? " " : "", nid_list); + if (len < 0 || cmd_len + len >= MAX_CMD_LEN) { + fprintf(stderr, "Error: Command too long\n"); + return 1; + } + cmd_len += len; + break; + } + case 'o': + output_file = optarg; + break; + case 'h': + /* Already handled above */ + break; + default: + usage(argv[0]); + return 1; + } + } + + /* At least one filter must be specified */ + if (cmd_len == 0) { + fprintf(stderr, "Error: At least one filter (-m or -n) must be specified\n\n"); + usage(argv[0]); + return 1; + } + + /* Open page_owner for read-write - this will fail if kernel doesn't support write */ + fd = open("/sys/kernel/debug/page_owner", O_RDWR); + if (fd < 0) { + if (errno == EACCES || errno == EPERM) { + fprintf(stderr, "Error: /sys/kernel/debug/page_owner "); + fprintf(stderr, "does not support write access\n"); + fprintf(stderr, "This kernel does not support "); + fprintf(stderr, "per-fd filtering.\n"); + fprintf(stderr, "Please ensure you have a kernel with "); + fprintf(stderr, "per-fd filtering support.\n"); + } else { + perror("Error opening /sys/kernel/debug/page_owner"); + } + return 1; + } + + if (output_file) { + output = fopen(output_file, "w"); + if (!output) { + perror("open output file"); + close(fd); + return 1; + } + } else { + output = stdout; + } + + ret = write(fd, filter_cmd, strlen(filter_cmd)); + + if (ret < 0) { + if (errno == EINVAL) { + fprintf(stderr, "Error: Kernel rejected the filter command.\n"); + fprintf(stderr, "Possible causes:\n"); + fprintf(stderr, " - Kernel does not support per-fd filtering\n"); + fprintf(stderr, " - NUMA node has no memory\n"); + fprintf(stderr, " - Unknown reason\n"); + } else { + perror("write filter command"); + } + goto out; + } + + if ((size_t)ret != strlen(filter_cmd)) + fprintf(stderr, "Warning: Partial write (%zd/%zu)\n", ret, strlen(filter_cmd)); + + /* Read and display filtered output */ + ret = 0; + while ((ret = read(fd, buf, sizeof(buf))) > 0) { + size_t written = fwrite(buf, 1, ret, output); + + if (written != (size_t)ret) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + goto out; + } + perror("write output"); + ret = -1; + goto out; + } + } + + if (ret < 0) { + perror("read page_owner"); + goto out; + } + + if (fflush(output)) { + if (errno == EPIPE) { + /* Pipe closed, treat as success */ + ret = 0; + } else { + perror("flush output"); + ret = -1; + } + } + +out: + close(fd); + if (output != stdout) + fclose(output); + return ret < 0 ? 1 : 0; +} diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c index e6954909401c..35d3d254941c 100644 --- a/tools/mm/page_owner_sort.c +++ b/tools/mm/page_owner_sort.c @@ -43,6 +43,13 @@ enum FILTER_BIT { FILTER_TGID = 1<<2, FILTER_COMM = 1<<3 }; + +enum FILTER_RESULT { + FILTER_ERROR, + FILTER_SKIP, + FILTER_MATCH +}; + enum CULL_BIT { CULL_PID = 1<<1, CULL_TGID = 1<<2, @@ -230,7 +237,8 @@ static int remove_pattern(regex_t *pattern, char *buf, int len) return len - (pmatch[1].rm_eo - pmatch[1].rm_so); } -static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) +static int search_pattern(regex_t *pattern, char *pattern_str, + size_t pattern_str_size, char *buf) { int err, val_len; regmatch_t pmatch[2]; @@ -242,8 +250,14 @@ static int search_pattern(regex_t *pattern, char *pattern_str, char *buf) return -1; } val_len = pmatch[1].rm_eo - pmatch[1].rm_so; + if ((size_t)val_len >= pattern_str_size) { + if (debug_on) + fprintf(stderr, "pattern too long in %s\n", buf); + return -1; + } memcpy(pattern_str, buf + pmatch[1].rm_so, val_len); + pattern_str[val_len] = '\0'; return 0; } @@ -300,7 +314,8 @@ static int get_page_num(char *buf) char order_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&order_pattern, order_str, buf); + if (search_pattern(&order_pattern, order_str, sizeof(order_str), buf) < 0) + return 0; errno = 0; order_val = strtol(order_str, &endptr, 10); if (order_val > 64 || errno != 0 || endptr == order_str || *endptr != '\0') { @@ -318,7 +333,8 @@ static pid_t get_pid(char *buf) char pid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&pid_pattern, pid_str, buf); + if (search_pattern(&pid_pattern, pid_str, sizeof(pid_str), buf) < 0) + return -1; errno = 0; pid = strtol(pid_str, &endptr, 10); if (errno != 0 || endptr == pid_str || *endptr != '\0') { @@ -337,7 +353,8 @@ static pid_t get_tgid(char *buf) char tgid_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&tgid_pattern, tgid_str, buf); + if (search_pattern(&tgid_pattern, tgid_str, sizeof(tgid_str), buf) < 0) + return -1; errno = 0; tgid = strtol(tgid_str, &endptr, 10); if (errno != 0 || endptr == tgid_str || *endptr != '\0') { @@ -356,7 +373,9 @@ static __u64 get_ts_nsec(char *buf) char ts_nsec_str[FIELD_BUFF] = {0}; char *endptr; - search_pattern(&ts_nsec_pattern, ts_nsec_str, buf); + if (search_pattern(&ts_nsec_pattern, ts_nsec_str, + sizeof(ts_nsec_str), buf) < 0) + return -1; errno = 0; ts_nsec = strtoull(ts_nsec_str, &endptr, 10); if (errno != 0 || endptr == ts_nsec_str || *endptr != '\0') { @@ -372,9 +391,15 @@ static char *get_comm(char *buf) { char *comm_str = malloc(TASK_COMM_LEN); + if (!comm_str) + return NULL; + memset(comm_str, 0, TASK_COMM_LEN); - search_pattern(&comm_pattern, comm_str, buf); + if (search_pattern(&comm_pattern, comm_str, TASK_COMM_LEN, buf) < 0) { + free(comm_str); + return NULL; + } errno = 0; if (errno != 0) { if (debug_on) @@ -386,6 +411,12 @@ static char *get_comm(char *buf) return comm_str; } +static void free_block_list(struct block_list *block) +{ + free(block->comm); + free(block->txt); +} + static int get_arg_type(const char *arg) { if (!strcmp(arg, "pid") || !strcmp(arg, "p")) @@ -450,39 +481,56 @@ static bool match_str_list(const char *str, char **list, int list_size) return false; } -static bool is_need(char *buf) +static enum FILTER_RESULT filter_record(char *buf) { + char *comm; + if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) - return false; + return FILTER_SKIP; if ((filter & FILTER_TGID) && !match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size)) - return false; + return FILTER_SKIP; + if (!(filter & FILTER_COMM)) + return FILTER_MATCH; - char *comm = get_comm(buf); + comm = get_comm(buf); + if (!comm) + return FILTER_ERROR; - if ((filter & FILTER_COMM) && - !match_str_list(comm, fc.comms, fc.comms_size)) { + if (!match_str_list(comm, fc.comms, fc.comms_size)) { free(comm); - return false; + return FILTER_SKIP; } free(comm); - return true; + return FILTER_MATCH; } static bool add_list(char *buf, int len, char *ext_buf) { + enum FILTER_RESULT filter_result; + if (list_size == max_size) { fprintf(stderr, "max_size too small??\n"); return false; } - if (!is_need(buf)) + filter_result = filter_record(buf); + if (filter_result == FILTER_ERROR) { + fprintf(stderr, "Out of memory\n"); + return false; + } + if (filter_result == FILTER_SKIP) return true; list[list_size].pid = get_pid(buf); list[list_size].tgid = get_tgid(buf); list[list_size].comm = get_comm(buf); - list[list_size].txt = malloc(len+1); + if (!list[list_size].comm) { + fprintf(stderr, "Out of memory\n"); + return false; + } + list[list_size].txt = malloc(len + 1); if (!list[list_size].txt) { fprintf(stderr, "Out of memory\n"); + free(list[list_size].comm); return false; } memcpy(list[list_size].txt, buf, len); @@ -841,8 +889,10 @@ int main(int argc, char **argv) } else { list[count-1].num += list[i].num; list[count-1].page_num += list[i].page_num; + free_block_list(&list[i]); } } + list_size = count; qsort(list, count, sizeof(list[0]), compare_sort_condition); @@ -876,8 +926,11 @@ int main(int argc, char **argv) free(ext_buf); if (buf) free(buf); - if (list) + if (list) { + for (i = 0; i < list_size; i++) + free_block_list(&list[i]); free(list); + } out_ts: regfree(&ts_nsec_pattern); out_comm: diff --git a/tools/mm/slabinfo.c b/tools/mm/slabinfo.c index 87570c22b151..48d1ee8b0e81 100644 --- a/tools/mm/slabinfo.c +++ b/tools/mm/slabinfo.c @@ -33,7 +33,7 @@ struct slabinfo { unsigned int hwcache_align, object_size, objs_per_slab; unsigned int sanity_checks, slab_size, store_user, trace; int order, poison, reclaim_account, red_zone; - unsigned long partial, objects, slabs, objects_partial, objects_total; + unsigned long partial, objects, slabs, objects_partial, total_objects; unsigned long alloc_fastpath, alloc_slowpath; unsigned long free_fastpath, free_slowpath; unsigned long free_frozen, free_add_partial, free_remove_partial; @@ -1262,7 +1262,7 @@ static void read_slab_dir(void) slab->object_size = get_obj("object_size"); slab->objects = get_obj("objects"); slab->objects_partial = get_obj("objects_partial"); - slab->objects_total = get_obj("objects_total"); + slab->total_objects = get_obj("total_objects"); slab->objs_per_slab = get_obj("objs_per_slab"); slab->order = get_obj("order"); slab->partial = get_obj_and_str("partial", &t); diff --git a/tools/net/ynl/Makefile b/tools/net/ynl/Makefile index d514a48dae27..3cefe4ed96cb 100644 --- a/tools/net/ynl/Makefile +++ b/tools/net/ynl/Makefile @@ -22,7 +22,7 @@ tests: | lib generated libynl.a ynltool: | lib generated libynl.a libynl.a: | lib generated @echo -e "\tAR $@" - @ar rcs $@ lib/ynl.o generated/*-user.o + @$(AR) rcs $@ lib/ynl.o generated/*-user.o $(SUBDIRS): @if [ -f "$@/Makefile" ] ; then \ diff --git a/tools/net/ynl/Makefile.deps b/tools/net/ynl/Makefile.deps index cc53b2f21c44..43d06ecbae93 100644 --- a/tools/net/ynl/Makefile.deps +++ b/tools/net/ynl/Makefile.deps @@ -14,10 +14,12 @@ UAPI_PATH:=../../../../include/uapi/ get_hdr_inc=-D$(1) -include $(UAPI_PATH)/linux/$(2) get_hdr_inc2=-D$(1) -D$(2) -include $(UAPI_PATH)/linux/$(3) +get_hdr_inc_drm=-D$(1) -include $(UAPI_PATH)/drm/$(2) CFLAGS_dev-energymodel:=$(call get_hdr_inc,_LINUX_DEV_ENERGYMODEL_H,dev_energymodel.h) CFLAGS_devlink:=$(call get_hdr_inc,_LINUX_DEVLINK_H_,devlink.h) CFLAGS_dpll:=$(call get_hdr_inc,_LINUX_DPLL_H,dpll.h) +CFLAGS_drm_ras:=$(call get_hdr_inc_drm,_LINUX_DRM_RAS_H,drm_ras.h) CFLAGS_ethtool:=$(call get_hdr_inc,_LINUX_TYPELIMITS_H,typelimits.h) \ $(call get_hdr_inc,_LINUX_ETHTOOL_H,ethtool.h) \ $(call get_hdr_inc,_LINUX_ETHTOOL_NETLINK_H_,ethtool_netlink.h) \ diff --git a/tools/net/ynl/generated/Makefile b/tools/net/ynl/generated/Makefile index 86e1e4a959a7..ea4128f612d6 100644 --- a/tools/net/ynl/generated/Makefile +++ b/tools/net/ynl/generated/Makefile @@ -37,7 +37,7 @@ all: protos.a $(HDRS) $(SRCS) $(KHDRS) $(KSRCS) $(UAPI) $(RSTS) protos.a: $(OBJS) @echo -e "\tAR $@" - @ar rcs $@ $(OBJS) + @$(AR) rcs $@ $(OBJS) %-user.h: $(SPECS_DIR)/%.yaml $(TOOL) @echo -e "\tGEN $@" diff --git a/tools/net/ynl/lib/Makefile b/tools/net/ynl/lib/Makefile index 4b2b98704ff9..9b98c0599600 100644 --- a/tools/net/ynl/lib/Makefile +++ b/tools/net/ynl/lib/Makefile @@ -15,7 +15,7 @@ all: ynl.a ynl.a: $(OBJS) @echo -e "\tAR $@" - @ar rcs $@ $(OBJS) + @$(AR) rcs $@ $(OBJS) clean: rm -f *.o *.d *~ diff --git a/tools/net/ynl/pyynl/__init__.py b/tools/net/ynl/pyynl/__init__.py index e69de29bb2d1..d8f59c132ab7 100644 --- a/tools/net/ynl/pyynl/__init__.py +++ b/tools/net/ynl/pyynl/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +""" Python YNL (YAML Netlink) library. """ + +# Re-export the public library API so it can be imported straight from the +# package, e.g. `from pyynl import YnlFamily`. +# pylint: disable=wildcard-import,unused-wildcard-import +from .lib import * +from .lib import __all__ diff --git a/tools/net/ynl/pyynl/cli.py b/tools/net/ynl/pyynl/cli.py index 8275a806cf73..b6a6ce12b4a7 100755 --- a/tools/net/ynl/pyynl/cli.py +++ b/tools/net/ynl/pyynl/cli.py @@ -17,9 +17,7 @@ # pylint: disable=no-name-in-module,wrong-import-position sys.path.append(pathlib.Path(__file__).resolve().parent.as_posix()) from lib import YnlFamily, Netlink, NlError, SpecFamily, SpecException, YnlException - -SYS_SCHEMA_DIR='/usr/share/ynl' -RELATIVE_SCHEMA_DIR='../../../../Documentation/netlink' +from lib import list_families # pylint: disable=too-few-public-methods,too-many-locals class Colors: @@ -48,30 +46,6 @@ def term_width(): """ Get terminal width in columns (80 if stdout is not a terminal) """ return shutil.get_terminal_size().columns -def schema_dir(): - """ - Return the effective schema directory, preferring in-tree before - system schema directory. - """ - script_dir = os.path.dirname(os.path.abspath(__file__)) - schema_dir_ = os.path.abspath(f"{script_dir}/{RELATIVE_SCHEMA_DIR}") - if not os.path.isdir(schema_dir_): - schema_dir_ = SYS_SCHEMA_DIR - if not os.path.isdir(schema_dir_): - raise YnlException(f"Schema directory {schema_dir_} does not exist") - return schema_dir_ - -def spec_dir(): - """ - Return the effective spec directory, relative to the effective - schema directory. - """ - spec_dir_ = schema_dir() + '/specs' - if not os.path.isdir(spec_dir_): - raise YnlException(f"Spec directory {spec_dir_} does not exist") - return spec_dir_ - - class YnlEncoder(json.JSONEncoder): """A custom encoder for emitting JSON with ynl-specific instance types""" def default(self, o): @@ -272,9 +246,8 @@ def output(msg): pprint.pprint(msg, width=term_width(), compact=True) if args.list_families: - for filename in sorted(os.listdir(spec_dir())): - if filename.endswith('.yaml'): - print(filename.removesuffix('.yaml')) + for family in list_families(): + print(family) return if args.no_schema: @@ -284,28 +257,23 @@ def output(msg): if args.json_text: attrs = json.loads(args.json_text) - if args.family: - spec = f"{spec_dir()}/{args.family}.yaml" - else: - spec = args.spec - if not os.path.isfile(spec): - raise YnlException(f"Spec file {spec} does not exist") + if args.spec and not os.path.isfile(args.spec): + raise YnlException(f"Spec file {args.spec} does not exist") + # Spec/YnlFamily will raise if both or neither spec and family are given if args.validate: + # Force validation even for installed specs (schema=True), unless the + # user explicitly picked a schema or opted out with --no-schema. + schema = True if args.schema is None else args.schema try: - SpecFamily(spec, args.schema) + SpecFamily(args.spec, schema_path=schema, family=args.family) except SpecException as error: print(error) sys.exit(1) return - if args.family: # set behaviour when using installed specs - if args.schema is None and spec.startswith(SYS_SCHEMA_DIR): - args.schema = '' # disable schema validation when installed - if args.process_unknown is None: - args.process_unknown = True - - ynl = YnlFamily(spec, args.schema, args.process_unknown, + ynl = YnlFamily(args.spec, schema=args.schema, family=args.family, + process_unknown=args.process_unknown, recv_size=args.dbg_small_recv) if args.dbg_small_recv: ynl.set_recv_dbg(True) diff --git a/tools/net/ynl/pyynl/lib/__init__.py b/tools/net/ynl/pyynl/lib/__init__.py index be741985ae4e..aa4263c8cba9 100644 --- a/tools/net/ynl/pyynl/lib/__init__.py +++ b/tools/net/ynl/pyynl/lib/__init__.py @@ -5,12 +5,13 @@ from .nlspec import SpecAttr, SpecAttrSet, SpecEnumEntry, SpecEnumSet, \ SpecFamily, SpecOperation, SpecSubMessage, SpecSubMessageFormat, \ SpecException +from .specdir import list_families from .ynl import YnlFamily, Netlink, NlError, NlPolicy, YnlException from .doc_generator import YnlDocGenerator __all__ = ["SpecAttr", "SpecAttrSet", "SpecEnumEntry", "SpecEnumSet", "SpecFamily", "SpecOperation", "SpecSubMessage", "SpecSubMessageFormat", - "SpecException", + "SpecException", "list_families", "YnlFamily", "Netlink", "NlError", "NlPolicy", "YnlException", "YnlDocGenerator"] diff --git a/tools/net/ynl/pyynl/lib/nlspec.py b/tools/net/ynl/pyynl/lib/nlspec.py index 0469a0e270d0..b4ec59814ab1 100644 --- a/tools/net/ynl/pyynl/lib/nlspec.py +++ b/tools/net/ynl/pyynl/lib/nlspec.py @@ -12,6 +12,8 @@ import os import yaml as pyyaml +from .specdir import find_spec, SYS_SCHEMA_DIR + class SpecException(Exception): """Netlink spec exception. @@ -444,7 +446,23 @@ class SpecFamily(SpecElement): except AttributeError: _yaml_loader = pyyaml.SafeLoader - def __init__(self, spec_path, schema_path=None, exclude_ops=None): + def __init__(self, spec_path=None, schema_path=None, exclude_ops=None, + family=None): + # schema_path selects how the spec is validated: + # None -- no preference: validate against the default schema, + # but trust (skip) installed specs selected by family= + # True -- always validate against the default schema + # path -- validate against this schema + # '' -- do not validate + if (spec_path is None) == (family is None): + raise ValueError("Specify exactly one of spec path or family name") + if family is not None: + spec_path = find_spec(family) + # Installed specs are assumed correct, so skip schema validation + # to save cycles unless the caller asked to validate. + if schema_path is None and spec_path.startswith(SYS_SCHEMA_DIR): + schema_path = '' + with open(spec_path, "r", encoding='utf-8') as stream: prefix = '# SPDX-License-Identifier: ' first = stream.readline().strip() @@ -465,7 +483,7 @@ def __init__(self, spec_path, schema_path=None, exclude_ops=None): self.proto = self.yaml.get('protocol', 'genetlink') self.msg_id_model = self.yaml['operations'].get('enum-model', 'unified') - if schema_path is None: + if schema_path is None or schema_path is True: schema_path = os.path.dirname(os.path.dirname(spec_path)) + f'/{self.proto}.yaml' if schema_path: with open(schema_path, "r", encoding='utf-8') as stream: diff --git a/tools/net/ynl/pyynl/lib/specdir.py b/tools/net/ynl/pyynl/lib/specdir.py new file mode 100644 index 000000000000..fcea9b9fb7b0 --- /dev/null +++ b/tools/net/ynl/pyynl/lib/specdir.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +""" +Locating YNL spec and schema files on disk. + +Resolves the directory holding the YAML specs (preferring an in-tree copy +over the installed system path) and maps family names to spec files. +""" + +import os + +SYS_SCHEMA_DIR='/usr/share/ynl' +RELATIVE_SCHEMA_DIR='../../../../../Documentation/netlink' + + +def schema_dir(): + """ + Return the effective schema directory, preferring in-tree before + system schema directory. + """ + script_dir = os.path.dirname(os.path.abspath(__file__)) + schema_dir_ = os.path.abspath(f"{script_dir}/{RELATIVE_SCHEMA_DIR}") + if not os.path.isdir(schema_dir_): + schema_dir_ = SYS_SCHEMA_DIR + if not os.path.isdir(schema_dir_): + raise FileNotFoundError(f"Schema directory {schema_dir_} does not exist") + return schema_dir_ + +def spec_dir(): + """ + Return the effective spec directory, relative to the effective + schema directory. + """ + spec_dir_ = schema_dir() + '/specs' + if not os.path.isdir(spec_dir_): + raise FileNotFoundError(f"Spec directory {spec_dir_} does not exist") + return spec_dir_ + + +def find_spec(family): + """ Return the path to the YAML spec file for a family by name. """ + spec = f"{spec_dir()}/{family}.yaml" + if not os.path.isfile(spec): + raise FileNotFoundError(f"Spec for family '{family}' not found at {spec}") + return spec + + +def list_families(): + """ Return the sorted names of all families with an installed spec. """ + return sorted(f.removesuffix('.yaml') + for f in os.listdir(spec_dir()) if f.endswith('.yaml')) diff --git a/tools/net/ynl/pyynl/lib/ynl.py b/tools/net/ynl/pyynl/lib/ynl.py index 092d132edec1..8682bf588e1f 100644 --- a/tools/net/ynl/pyynl/lib/ynl.py +++ b/tools/net/ynl/pyynl/lib/ynl.py @@ -661,6 +661,14 @@ class YnlFamily(SpecFamily): """ YNL family -- a Netlink interface built from a YAML spec. + The spec can be selected either by file path (def_path=) or, when it + ships in a well-known location, by family name (family="xyz"); exactly + one of the two must be given. For example: + + from pyynl import YnlFamily + + ynl = YnlFamily(family="netdev") + Primary use of the class is to execute Netlink commands: ynl.(attrs, ...) @@ -691,11 +699,16 @@ class YnlFamily(SpecFamily): ynl.get_policy(op_name, mode) -- query kernel policy for an op """ - def __init__(self, def_path, schema=None, process_unknown=False, - recv_size=0): - super().__init__(def_path, schema) + def __init__(self, def_path=None, schema=None, process_unknown=None, + recv_size=0, family=None): + super().__init__(def_path, schema, family=family) self.include_raw = False + # Specs from /usr (selected by family=) have a higher chance of being + # stale, default to ignoring unknown attrs. In-tree users, and users + # who bundle the spec need to make a conscious decision. + if process_unknown is None: + process_unknown = family is not None self.process_unknown = process_unknown try: diff --git a/tools/net/ynl/tests/ethtool.py b/tools/net/ynl/tests/ethtool.py index db3b62c652e7..0ee0c8e87686 100755 --- a/tools/net/ynl/tests/ethtool.py +++ b/tools/net/ynl/tests/ethtool.py @@ -11,12 +11,10 @@ import pprint import sys import re -import os # pylint: disable=no-name-in-module,wrong-import-position sys.path.append(pathlib.Path(__file__).resolve().parent.parent.joinpath('pyynl').as_posix()) # pylint: disable=import-error -from cli import schema_dir, spec_dir from lib import YnlFamily @@ -173,10 +171,7 @@ def main(): args = parser.parse_args() - spec = os.path.join(spec_dir(), 'ethtool.yaml') - schema = os.path.join(schema_dir(), 'genetlink-legacy.yaml') - - ynl = YnlFamily(spec, schema, recv_size=args.dbg_small_recv) + ynl = YnlFamily(family='ethtool', recv_size=args.dbg_small_recv) if args.dbg_small_recv: ynl.set_recv_dbg(True) diff --git a/tools/perf/.clang-format b/tools/perf/.clang-format new file mode 100644 index 000000000000..902b2f7456f6 --- /dev/null +++ b/tools/perf/.clang-format @@ -0,0 +1,20 @@ +BasedOnStyle: InheritParentConfig +SortIncludes: true +IncludeBlocks: Regroup +IncludeCategories: + # Implicitly the corresponding header for the C file has Priority 0 + # C Standard Library Headers + - Regex: '^<(assert|complex|ctype|errno|fenv|float|inttypes|iso646|limits|locale|math|setjmp|signal|stdalign|stdarg|stdatomic|stdbool|stddef|stdint|stdio|stdlib|stdnoreturn|string|tgmath|threads|time|uchar|wchar|wctype)\.h>' + Priority: 1 + # OS/System-Specific Headers (directories) + - Regex: '^<(sys|linux|asm|arpa|net|netinet|x86_64|machine)/.*>' + Priority: 2 + # OS/System-Specific Headers (POSIX/System flat headers) + - Regex: '^<(unistd|pthread|fcntl|dirent|dlfcn|poll|sched|semaphore|spawn|syslog|termios|pwd|grp|netdb|sysexits|err|paths|pty|utmp|resolv|ifaddrs|elf|libelf|gelf)\.h>' + Priority: 2 + # Third-Party Library Headers + - Regex: '^<.*>' + Priority: 3 + # Your Project's Other Headers + - Regex: '^".*"' + Priority: 4 diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index 0f9451a6e39c..3b968c5158b8 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -38,6 +38,7 @@ arch/*/include/generated/ trace/beauty/generated/ pmu-events/arch/common/common/legacy-cache.json pmu-events/pmu-events.c +pmu-events/pmu-events-string.c pmu-events/jevents pmu-events/metric_test.log pmu-events/empty-pmu-events.log diff --git a/tools/perf/Build b/tools/perf/Build index b03cc59dabf8..e18c80a5c1bc 100644 --- a/tools/perf/Build +++ b/tools/perf/Build @@ -34,6 +34,8 @@ ifeq ($(CONFIG_LIBTRACEEVENT),y) perf-$(CONFIG_TRACE) += trace/beauty/ endif +perf-util-y += trace/beauty/ + perf-$(CONFIG_LIBELF) += builtin-probe.o perf-bench-y += bench/ diff --git a/tools/perf/Documentation/perf-amd-ibs.txt b/tools/perf/Documentation/perf-amd-ibs.txt index 548549935760..253a7375c88a 100644 --- a/tools/perf/Documentation/perf-amd-ibs.txt +++ b/tools/perf/Documentation/perf-amd-ibs.txt @@ -69,6 +69,14 @@ Per-cpu profile (cpu10), cycles event, sampling period: 100000 # perf record -e ibs_op// -c 100000 -C 10 +Userspace only, per-cpu profile (cpu10), cycles event, sampling period: 100000 + + Zen6 onward (See NOTES): + # perf record -e ibs_op//u -c 100000 -C 10 + + Until Zen5: + # perf record -e ibs_op/swfilt=1/u -c 100000 -C 10 + Per-cpu profile (cpu10), cycles event, sampling freq: 1000 # perf record -e ibs_op// -F 1000 -C 10 @@ -94,6 +102,11 @@ onward) Latency value which is a multiple of 128 incurs a little less profiling overhead compared to other values. +System-wide profile, cycles event, sampling period: 100000, streaming store +filter (Zen6 onward) + + # perf record -e ibs_op/strmst=1/ -c 100000 -a + Per process(upstream v6.2 onward), uOps event, sampling period: 100000 # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -p 1234 @@ -150,6 +163,14 @@ System-wide profile, fetch ops event, sampling period: 100000 # perf record -e ibs_fetch// -c 100000 -a +Userspace only, system-wide profile, fetch ops event, sampling period: 100000 + + Zen6 onward (See NOTES): + # perf record -e ibs_fetch//u -c 100000 -a + + Until Zen5: + # perf record -e ibs_fetch/swfilt=1/u -c 100000 -a + System-wide profile, fetch ops event, sampling period: 100000, Random enable # perf record -e ibs_fetch/rand_en=1/ -c 100000 -a @@ -158,6 +179,15 @@ System-wide profile, fetch ops event, sampling period: 100000, Random enable helps in cases like long running loops where PMU is tagging the same instruction over and over because of fixed sample period. +System-wide profile, fetch ops event, sampling period: 10000, fetch latency +filter (Zen6 onward) + + # perf record -e ibs_fetch/fetchlat=128/ -c 10000 -a + + Supported fetch latency threshold values are 128 to 1920 (both inclusive). + Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + etc. PERF MEM AND PERF C2C @@ -216,6 +246,15 @@ sort keys. For example: Please refer to their man page for more detail. +NOTES +----- +Hardware privilege filtering uses bit 63 to distinguish between kernel +and userspace addresses. Hardware privilege filtering is not supported +on 32-bit systems. Also, the bit 63 convention is not universal and can +fail in specific environments, such as, using 64-bit host IBS to profile +a 32-bit guest, using 64-bit host IBS to profile non-Linux 64-bit guests +that do not adhere to the bit 63 privilege standard etc. + SEE ALSO -------- diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt index 32da0d1fa86a..859df74e62ef 100644 --- a/tools/perf/Documentation/perf-test.txt +++ b/tools/perf/Documentation/perf-test.txt @@ -55,17 +55,35 @@ OPTIONS -w:: --workload=:: - Run a built-in workload, to list them use '--list-workloads', current ones include: - noploop, thloop, leafloop, sqrtloop, brstack, datasym and landlock. + Run a built-in workload, to list them use '--list-workloads', current + ones include: noploop, thloop, leafloop, sqrtloop, brstack, datasym, + context_switch_loop, deterministic, named_threads, landlock and + callchain. Used with the shell script regression tests. Some accept an extra parameter: seconds: leafloop, noploop, sqrtloop, thloop - nrloops: brstack + nrloops: brstack, context_switch_loop - The datasym and landlock workloads don't accept any. + 'named_threads' accepts the number of threads and the number of loops to + do in each thread. + + The datasym, landlock, deterministic and callchain workloads don't accept + any. --list-workloads:: List the available workloads to use with -w/--workload. + +--record-ctl=fifo:ctl-fifo[,ack-fifo]:: + This option is used to communicate with a perf record session in order + to control the recording scope to only the workload and avoid recording + setup and teardown code. When specifying this option, the same FIFO path + must be specified in the record session via: + + perf record -D -1 --control=fifo:ctl-fifo[,ack-fifo] ... + + Perf test sends 'enable' and 'disable' commands through ctl-fifo to + control event recording. If 'ack-fifo' is provided, the workload runner + waits for an 'ack' response after each command. diff --git a/tools/perf/Documentation/perf-trace.txt b/tools/perf/Documentation/perf-trace.txt index 892c82a9bf40..d0b6c771a1b9 100644 --- a/tools/perf/Documentation/perf-trace.txt +++ b/tools/perf/Documentation/perf-trace.txt @@ -199,6 +199,9 @@ the thread executes on the designated CPUs. Default is to monitor all CPUs. --show-on-off-events:: Show the --switch-on/off events too. +--show-cpu:: + Show cpu id. + --max-stack:: Set the stack depth limit when parsing the callchain, anything beyond the specified depth will be ignored. Note that at this point diff --git a/tools/perf/Documentation/perf.data-file-format.txt b/tools/perf/Documentation/perf.data-file-format.txt index 0e4d0ecc9e12..b90cba9168f8 100644 --- a/tools/perf/Documentation/perf.data-file-format.txt +++ b/tools/perf/Documentation/perf.data-file-format.txt @@ -464,7 +464,21 @@ struct cpu_domain_info { struct domain_info domains[]; }; - other bits are reserved and should ignored for now + HEADER_E_MACHINE = 33, + +ELF machine and flags data. e_machine is expanded from 16 to 32 bits +for alignment. Format: + + u32 e_machine; + u32 e_flags; + + HEADER_CLN_SIZE = 34, + +The size of the cacheline in bytes. Format: + + unsigned int cln_size; + + other bits are reserved and should be ignored for now HEADER_FEAT_BITS = 256, Attributes diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 333ddd0e4bd8..0ba307e78fe1 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -65,95 +65,85 @@ $(call detected_var,SRCARCH) CFLAGS += -I$(OUTPUT)arch/$(SRCARCH)/include/generated -# Additional ARCH settings for ppc -ifeq ($(SRCARCH),powerpc) - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS := -lunwind -lunwind-ppc64 - endif -endif - # Additional ARCH settings for x86 ifeq ($(SRCARCH),x86) $(call detected,CONFIG_X86) ifeq (${IS_64_BIT}, 1) CFLAGS += -DHAVE_ARCH_X86_64_SUPPORT ARCH_INCLUDE = ../../arch/x86/lib/memcpy_64.S ../../arch/x86/lib/memset_64.S - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind-x86_64 -lunwind -llzma - endif $(call detected,CONFIG_X86_64) - else - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind-x86 -llzma -lunwind - endif endif endif -ifeq ($(SRCARCH),arm) - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind -lunwind-arm - endif +ifeq ($(ARCH),s390) + CFLAGS += -fPIC + CXXFLAGS += -fPIC endif +# Unconditionally set up the libunwind feature build flags as a +# feature-dump build doesn't specify LIBUNWIND=1. This means that +# dumping the libunwind features will be broken that can impact later +# builds that use the feature dump. +ifeq ($(SRCARCH),arm) + LIBUNWIND_LIBS = -lunwind -lunwind-arm +endif ifeq ($(SRCARCH),arm64) - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind -lunwind-aarch64 - endif + LIBUNWIND_LIBS = -lunwind -lunwind-aarch64 endif - ifeq ($(SRCARCH),loongarch) - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind -lunwind-loongarch64 - endif + LIBUNWIND_LIBS = -lunwind -lunwind-loongarch64 endif - -ifeq ($(ARCH),s390) - CFLAGS += -fPIC -endif - ifeq ($(ARCH),mips) - ifndef NO_LIBUNWIND - LIBUNWIND_LIBS = -lunwind -lunwind-mips + LIBUNWIND_LIBS = -lunwind -lunwind-mips +endif +ifeq ($(SRCARCH),powerpc) + LIBUNWIND_LIBS := -lunwind -lunwind-ppc64 +endif +ifeq ($(SRCARCH),riscv) + LIBUNWIND_LIBS := -lunwind -lunwind-riscv +endif +ifeq ($(SRCARCH),s390) + LIBUNWIND_LIBS := -lunwind -lunwind-s390x +endif +ifeq ($(SRCARCH),x86) + ifeq (${IS_64_BIT}, 1) + LIBUNWIND_LIBS = -lunwind-x86_64 -lunwind -llzma + else + LIBUNWIND_LIBS = -lunwind-x86 -lunwind -llzma endif endif - ifneq ($(LIBUNWIND),1) NO_LIBUNWIND := 1 endif - ifeq ($(LIBUNWIND_LIBS),) NO_LIBUNWIND := 1 endif + # # For linking with debug library, run like: # # make DEBUG=1 LIBUNWIND_DIR=/opt/libunwind/ # +LIBUNWIND_ARCHS:=aarch64 arm loongarch64 mips ppc32 ppc64 riscv s390x x86 x86_64 -libunwind_arch_set_flags = $(eval $(libunwind_arch_set_flags_code)) -define libunwind_arch_set_flags_code - FEATURE_CHECK_CFLAGS-libunwind-$(1) = -I$(LIBUNWIND_DIR)/include - FEATURE_CHECK_LDFLAGS-libunwind-$(1) = -L$(LIBUNWIND_DIR)/lib -endef +# "Local" (no arch specified) feature test flags. +FEATURE_CHECK_CFLAGS-libunwind = $(LIBUNWIND_CFLAGS) +FEATURE_CHECK_LDFLAGS-libunwind = $(LIBUNWIND_LDFLAGS) $(LIBUNWIND_LIBS) +FEATURE_CHECK_CFLAGS-libunwind-debug-frame = $(LIBUNWIND_CFLAGS) +FEATURE_CHECK_LDFLAGS-libunwind-debug-frame = $(LIBUNWIND_LDFLAGS) $(LIBUNWIND_LIBS) +# Add directory into the "remote" (build for a a specific arch) feature tests. ifdef LIBUNWIND_DIR LIBUNWIND_CFLAGS = -I$(LIBUNWIND_DIR)/include LIBUNWIND_LDFLAGS = -L$(LIBUNWIND_DIR)/lib - LIBUNWIND_ARCHS = x86 x86_64 arm aarch64 debug-frame-arm debug-frame-aarch64 loongarch - $(foreach libunwind_arch,$(LIBUNWIND_ARCHS),$(call libunwind_arch_set_flags,$(libunwind_arch))) -endif -ifndef NO_LIBUNWIND - # Set per-feature check compilation flags - FEATURE_CHECK_CFLAGS-libunwind = $(LIBUNWIND_CFLAGS) - FEATURE_CHECK_LDFLAGS-libunwind = $(LIBUNWIND_LDFLAGS) $(LIBUNWIND_LIBS) - FEATURE_CHECK_CFLAGS-libunwind-debug-frame = $(LIBUNWIND_CFLAGS) - FEATURE_CHECK_LDFLAGS-libunwind-debug-frame = $(LIBUNWIND_LDFLAGS) $(LIBUNWIND_LIBS) - - FEATURE_CHECK_LDFLAGS-libunwind-arm += -lunwind -lunwind-arm - FEATURE_CHECK_LDFLAGS-libunwind-aarch64 += -lunwind -lunwind-aarch64 - FEATURE_CHECK_LDFLAGS-libunwind-x86 += -lunwind -llzma -lunwind-x86 - FEATURE_CHECK_LDFLAGS-libunwind-x86_64 += -lunwind -llzma -lunwind-x86_64 + define libunwind_arch_set_flags + FEATURE_CHECK_CFLAGS-libunwind-$(1) = -I$(LIBUNWIND_DIR)/include + FEATURE_CHECK_LDFLAGS-libunwind-$(1) = -L$(LIBUNWIND_DIR)/lib -lunwind -lunwind-$(1) + endef + $(foreach arch,$(LIBUNWIND_ARCHS), \ + $(eval $(call libunwind_arch_set_flags,$(arch))) \ + ) endif ifdef CSINCLUDES @@ -195,14 +185,11 @@ endif FEATURE_CHECK_CFLAGS-libdw := $(LIBDW_CFLAGS) FEATURE_CHECK_LDFLAGS-libdw := $(LIBDW_LDFLAGS) $(DWARFLIBS) -# for linking with debug library, run like: -# make DEBUG=1 LIBBABELTRACE_DIR=/opt/libbabeltrace/ -ifdef LIBBABELTRACE_DIR - LIBBABELTRACE_CFLAGS := -I$(LIBBABELTRACE_DIR)/include - LIBBABELTRACE_LDFLAGS := -L$(LIBBABELTRACE_DIR)/lib +ifneq ($(NO_BABELTRACE2),1) + ifeq ($(call get-executable,$(PKG_CONFIG)),) + $(error Error: $(PKG_CONFIG) needed by babeltrace2 is missing on this system, please install it) + endif endif -FEATURE_CHECK_CFLAGS-libbabeltrace := $(LIBBABELTRACE_CFLAGS) -FEATURE_CHECK_LDFLAGS-libbabeltrace := $(LIBBABELTRACE_LDFLAGS) -lbabeltrace-ctf # for linking with debug library, run like: # make DEBUG=1 LIBCAPSTONE_DIR=/opt/capstone/ @@ -587,10 +574,8 @@ ifndef NO_LIBELF ifndef NO_LIBBPF ifeq ($(feature-bpf), 1) - # detecting libbpf without LIBBPF_DYNAMIC, so make VF=1 shows libbpf detection status - $(call feature_check,libbpf) - ifdef LIBBPF_DYNAMIC + $(call feature_check,libbpf) ifeq ($(feature-libbpf), 1) EXTLIBS += -lbpf CFLAGS += -DHAVE_LIBBPF_SUPPORT @@ -638,49 +623,6 @@ ifeq ($(SRCARCH),powerpc) endif endif -ifndef NO_LIBUNWIND - have_libunwind := - - $(call feature_check,libunwind) - - $(call feature_check,libunwind-x86) - ifeq ($(feature-libunwind-x86), 1) - $(call detected,CONFIG_LIBUNWIND_X86) - CFLAGS += -DHAVE_LIBUNWIND_X86_SUPPORT - LDFLAGS += -lunwind-x86 - EXTLIBS_LIBUNWIND += -lunwind-x86 - have_libunwind = 1 - endif - - $(call feature_check,libunwind-aarch64) - ifeq ($(feature-libunwind-aarch64), 1) - $(call detected,CONFIG_LIBUNWIND_AARCH64) - CFLAGS += -DHAVE_LIBUNWIND_AARCH64_SUPPORT - LDFLAGS += -lunwind-aarch64 - EXTLIBS_LIBUNWIND += -lunwind-aarch64 - have_libunwind = 1 - $(call feature_check,libunwind-debug-frame-aarch64) - ifneq ($(feature-libunwind-debug-frame-aarch64), 1) - $(warning No debug_frame support found in libunwind-aarch64) - CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME_AARCH64 - endif - endif - - ifneq ($(feature-libunwind), 1) - $(warning No libunwind found. Please install libunwind-dev[el] >= 1.1 and/or set LIBUNWIND_DIR and set LIBUNWIND=1 in the make command line as it is opt-in now) - NO_LOCAL_LIBUNWIND := 1 - else - have_libunwind := 1 - $(call detected,CONFIG_LOCAL_LIBUNWIND) - endif - - ifneq ($(have_libunwind), 1) - NO_LIBUNWIND := 1 - endif -else - NO_LOCAL_LIBUNWIND := 1 -endif - ifndef NO_LIBBPF ifneq ($(feature-bpf), 1) $(warning BPF API too old. Please install recent kernel headers. BPF support in 'perf record' is disabled.) @@ -736,7 +678,61 @@ ifeq ($(BUILD_BPF_SKEL),1) endif ifndef GEN_VMLINUX_H - VMLINUX_H=$(src-perf)/util/bpf_skel/vmlinux/vmlinux.h + VMLINUX_H_FILE=$(src-perf)/util/bpf_skel/vmlinux/vmlinux.h +endif + +ifndef NO_LIBUNWIND + have_libunwind := + + $(call feature_check,libunwind) + ifneq ($(feature-libunwind), 1) + $(warning No libunwind found. Please install libunwind-dev[el] >= 1.1 and/or set LIBUNWIND_DIR and set LIBUNWIND=1 in the make command line as it is opt-in now) + NO_LOCAL_LIBUNWIND := 1 + else + have_libunwind := 1 + $(call detected,CONFIG_LOCAL_LIBUNWIND) + CFLAGS += -DHAVE_LIBUNWIND_SUPPORT + CFLAGS += $(LIBUNWIND_CFLAGS) + LDFLAGS += $(LIBUNWIND_LDFLAGS) + EXTLIBS_LIBUNWIND := $(LIBUNWIND_LIBS) + $(call feature_check,libunwind-debug-frame) + ifneq ($(feature-libunwind-debug-frame), 1) + CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME + endif + endif + + define PROCESS_REMOTE_LIBUNWIND_ARCH + $(call feature_check,libunwind-$(1)) + + ifeq ($$(feature-libunwind-$(1)), 1) + upper_arch := $$(shell echo $(1) | tr '[:lower:]' '[:upper:]') + $$(call detected,CONFIG_LIBUNWIND_$$(upper_arch)) + + CFLAGS += -DHAVE_LIBUNWIND_$$(upper_arch)_SUPPORT + LDFLAGS += -lunwind-$(1) + EXTLIBS_LIBUNWIND += -lunwind-$(1) + have_libunwind := 1 + + $$(call feature_check,libunwind-debug-frame-$(1)) + ifneq ($$(feature-libunwind-debug-frame-$(1)), 1) + CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME_$$(upper_arch) + endif + endif + endef + $(foreach arch,$(LIBUNWIND_ARCHS), \ + $(eval $(call PROCESS_REMOTE_LIBUNWIND_ARCH,$(arch))) \ + ) + + EXTLIBS += $(EXTLIBS_LIBUNWIND) + + ifeq ($(findstring -static,${LDFLAGS}),-static) + # gcc -static links libgcc_eh which contains piece of libunwind + LIBUNWIND_LDFLAGS += -Wl,--allow-multiple-definition + endif + + ifneq ($(have_libunwind), 1) + NO_LIBUNWIND := 1 + endif endif dwarf-post-unwind := 1 @@ -761,31 +757,6 @@ ifeq ($(dwarf-post-unwind),1) $(call detected,CONFIG_DWARF_UNWIND) endif -ifndef NO_LIBUNWIND - ifndef NO_LOCAL_LIBUNWIND - ifeq ($(SRCARCH),$(filter $(SRCARCH),arm arm64)) - $(call feature_check,libunwind-debug-frame) - ifneq ($(feature-libunwind-debug-frame), 1) - $(warning No debug_frame support found in libunwind) - CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME - endif - else - # non-ARM has no dwarf_find_debug_frame() function: - CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME - endif - EXTLIBS += $(LIBUNWIND_LIBS) - LDFLAGS += $(LIBUNWIND_LIBS) - endif - ifeq ($(findstring -static,${LDFLAGS}),-static) - # gcc -static links libgcc_eh which contans piece of libunwind - LIBUNWIND_LDFLAGS += -Wl,--allow-multiple-definition - endif - CFLAGS += -DHAVE_LIBUNWIND_SUPPORT - CFLAGS += $(LIBUNWIND_CFLAGS) - LDFLAGS += $(LIBUNWIND_LDFLAGS) - EXTLIBS += $(EXTLIBS_LIBUNWIND) -endif - ifneq ($(NO_LIBTRACEEVENT),1) $(call detected,CONFIG_TRACE) endif @@ -923,7 +894,7 @@ ifdef BUILD_NONDISTRO $(call feature_check,libbfd-liberty-z) ifneq ($(feature-libbfd-threadsafe), 1) - $(error binutils 2.42 or later is required for non-distro builds) + $(error binutils-dev(el) 2.42 or later is required for non-distro builds) endif # we may be on a system that requires -liberty and (maybe) -lz @@ -953,11 +924,14 @@ ifndef NO_LIBLLVM $(call feature_check,llvm-perf) ifeq ($(feature-llvm-perf), 1) CFLAGS += -DHAVE_LIBLLVM_SUPPORT - CFLAGS += $(shell $(LLVM_CONFIG) --cflags) - CXXFLAGS += -DHAVE_LIBLLVM_SUPPORT - CXXFLAGS += $(shell $(LLVM_CONFIG) --cxxflags) - LIBLLVM = $(shell $(LLVM_CONFIG) --libs all) $(shell $(LLVM_CONFIG) --system-libs) - EXTLIBS += -L$(shell $(LLVM_CONFIG) --libdir) $(LIBLLVM) + LLVM_CFLAGS := $(shell $(LLVM_CONFIG) --cflags 2>/dev/null) + LLVM_CXXFLAGS := $(shell $(LLVM_CONFIG) --cxxflags 2>/dev/null) + LLVM_LIBLLVM := $(shell $(LLVM_CONFIG) --libs all 2>/dev/null) $(shell $(LLVM_CONFIG) --system-libs 2>/dev/null) + LLVM_LIBDIR := $(shell $(LLVM_CONFIG) --libdir 2>/dev/null) + CFLAGS += $(LLVM_CFLAGS) + CXXFLAGS += -DHAVE_LIBLLVM_SUPPORT $(LLVM_CXXFLAGS) + LIBLLVM := $(LLVM_LIBLLVM) + EXTLIBS += -L$(LLVM_LIBDIR) $(LIBLLVM) EXTLIBS += -lstdc++ $(call detected,CONFIG_LIBLLVM) else @@ -1059,15 +1033,15 @@ else NO_PERF_READ_VDSOX32 := 1 endif -ifndef NO_LIBBABELTRACE - $(call feature_check,libbabeltrace) - ifeq ($(feature-libbabeltrace), 1) - CFLAGS += -DHAVE_LIBBABELTRACE_SUPPORT $(LIBBABELTRACE_CFLAGS) - LDFLAGS += $(LIBBABELTRACE_LDFLAGS) - EXTLIBS += -lbabeltrace-ctf - $(call detected,CONFIG_LIBBABELTRACE) +ifndef NO_BABELTRACE2 + $(call feature_check,babeltrace2-ctf-writer) + ifeq ($(feature-babeltrace2-ctf-writer), 1) + CFLAGS += -DHAVE_BABELTRACE2_CTF_WRITER_SUPPORT $(shell $(PKG_CONFIG) --cflags babeltrace2-ctf-writer) + LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-L babeltrace2-ctf-writer) + EXTLIBS += $(shell $(PKG_CONFIG) --libs-only-l babeltrace2-ctf-writer) + $(call detected,CONFIG_BABELTRACE2_CTF_WRITER) else - $(warning No libbabeltrace found, disables 'perf data' CTF format support, please install libbabeltrace-dev[el]/libbabeltrace-ctf-dev) + $(warning No babeltrace2 found, disables 'perf data' CTF format support, please install libbabeltrace2-dev[el]) endif endif diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 76b35ac19acb..2b53d9e0c73b 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -79,7 +79,7 @@ include ../scripts/utilities.mak # # Define NO_ZLIB if you do not want to support compressed kernel modules # -# Define NO_LIBBABELTRACE if you do not want libbabeltrace support +# Define NO_BABELTRACE2 if you do not want babeltrace2-ctf-writer support # for CTF data format. # # Define NO_CAPSTONE if you do not want libcapstone support @@ -274,8 +274,8 @@ ifeq ($(PYLINT),1) PYLINT := $(shell which pylint 2> /dev/null) endif -export srctree OUTPUT RM CC CXX RUSTC LD AR CFLAGS CXXFLAGS RUST_FLAGS V BISON FLEX AWK -export HOSTCC HOSTLD HOSTAR HOSTCFLAGS SHELLCHECK MYPY PYLINT +export srctree OUTPUT RM CC CXX RUSTC CLANG LD AR CFLAGS CXXFLAGS RUST_FLAGS V BISON FLEX AWK LIBBPF READELF +export HOSTCC HOSTLD HOSTAR HOSTCFLAGS SHELLCHECK MYPY PYLINT CONFIG_PERF_BPF_SKEL include $(srctree)/tools/build/Makefile.include @@ -323,7 +323,7 @@ else FEATURE_DUMP_EXPORT := $(realpath $(FEATURES_DUMP)) endif -export prefix bindir sharedir sysconfdir DESTDIR +export prefix bindir sharedir sysconfdir DESTDIR VMLINUX_H_FILE # sparse is architecture-neutral, which means that we need to tell it # explicitly what architecture to check for. Fix this up for yours.. @@ -401,10 +401,14 @@ export PYTHON_EXTBUILD_LIB PYTHON_EXTBUILD_TMP python-clean := $(call QUIET_CLEAN, python) $(RM) -r $(PYTHON_EXTBUILD) $(OUTPUT)python/perf*.so +ifneq ($(quiet),) +python_setup_quiet=--quiet +endif + # Use the detected configuration -include $(OUTPUT).config-detected -SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) +SCRIPTS = $(addprefix $(OUTPUT),$(patsubst %.sh,%,$(SCRIPT_SH))) PROGRAMS += $(OUTPUT)perf @@ -508,252 +512,20 @@ arm64-sysreg-defs-clean: $(Q)$(MAKE) -C $(arm64_gen_sysreg_dir) O=$(arm64_gen_sysreg_outdir) \ prefix= subdir= clean > /dev/null -beauty_linux_dir := $(srctree)/tools/perf/trace/beauty/include/linux/ -beauty_uapi_linux_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/linux/ -beauty_uapi_sound_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/sound/ -beauty_arch_asm_dir := $(srctree)/tools/perf/trace/beauty/arch/x86/include/asm/ -beauty_x86_arch_asm_uapi_dir := $(srctree)/tools/perf/trace/beauty/arch/x86/include/uapi/asm/ - -linux_uapi_dir := $(srctree)/tools/include/uapi/linux -asm_generic_uapi_dir := $(srctree)/tools/include/uapi/asm-generic -arch_asm_uapi_dir := $(srctree)/tools/arch/$(SRCARCH)/include/uapi/asm/ -x86_arch_asm_dir := $(srctree)/tools/arch/x86/include/asm/ - -beauty_outdir := $(OUTPUT)trace/beauty/generated -beauty_ioctl_outdir := $(beauty_outdir)/ioctl - -# Create output directory if not already present -$(shell [ -d '$(beauty_ioctl_outdir)' ] || mkdir -p '$(beauty_ioctl_outdir)') - -syscall_array := $(beauty_outdir)/syscalltbl.c -syscall_tbl := $(srctree)/tools/perf/trace/beauty/syscalltbl.sh -syscall_tbl_data := $(srctree)/tools/scripts/syscall.tbl \ - $(wildcard $(srctree)/tools/perf/arch/*/entry/syscalls/syscall*.tbl) - -$(syscall_array): $(syscall_tbl) $(syscall_tbl_data) - $(Q)$(SHELL) '$(syscall_tbl)' $(srctree)/tools $@ - -fs_at_flags_array := $(beauty_outdir)/fs_at_flags_array.c -fs_at_flags_tbl := $(srctree)/tools/perf/trace/beauty/fs_at_flags.sh - -$(fs_at_flags_array): $(beauty_uapi_linux_dir)/fcntl.h $(fs_at_flags_tbl) - $(Q)$(SHELL) '$(fs_at_flags_tbl)' $(beauty_uapi_linux_dir) > $@ - -clone_flags_array := $(beauty_outdir)/clone_flags_array.c -clone_flags_tbl := $(srctree)/tools/perf/trace/beauty/clone.sh - -$(clone_flags_array): $(beauty_uapi_linux_dir)/sched.h $(clone_flags_tbl) - $(Q)$(SHELL) '$(clone_flags_tbl)' $(beauty_uapi_linux_dir) > $@ - -drm_ioctl_array := $(beauty_ioctl_outdir)/drm_ioctl_array.c -drm_hdr_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/drm -drm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/drm_ioctl.sh - -$(drm_ioctl_array): $(drm_hdr_dir)/drm.h $(drm_hdr_dir)/i915_drm.h $(drm_ioctl_tbl) - $(Q)$(SHELL) '$(drm_ioctl_tbl)' $(drm_hdr_dir) > $@ - -fadvise_advice_array := $(beauty_outdir)/fadvise_advice_array.c -fadvise_advice_tbl := $(srctree)/tools/perf/trace/beauty/fadvise.sh - -$(fadvise_advice_array): $(beauty_uapi_linux_dir)/fadvise.h $(fadvise_advice_tbl) - $(Q)$(SHELL) '$(fadvise_advice_tbl)' $(beauty_uapi_linux_dir) > $@ - -fsmount_arrays := $(beauty_outdir)/fsmount_arrays.c -fsmount_tbls := $(srctree)/tools/perf/trace/beauty/fsmount.sh - -$(fsmount_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_tbls) - $(Q)$(SHELL) '$(fsmount_tbls)' $(beauty_uapi_linux_dir) > $@ - -fsmount_attr_arrays := $(beauty_outdir)/fsmount_attr_arrays.c -fsmount_attr_tbls := $(srctree)/tools/perf/trace/beauty/fsmount_attr.sh - -$(fsmount_attr_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_attr_tbls) - $(Q)$(SHELL) '$(fsmount_attr_tbls)' $(beauty_uapi_linux_dir) > $@ - -fspick_arrays := $(beauty_outdir)/fspick_arrays.c -fspick_tbls := $(srctree)/tools/perf/trace/beauty/fspick.sh - -$(fspick_arrays): $(beauty_uapi_linux_dir)/mount.h $(fspick_tbls) - $(Q)$(SHELL) '$(fspick_tbls)' $(beauty_uapi_linux_dir) > $@ - -fsconfig_arrays := $(beauty_outdir)/fsconfig_arrays.c -fsconfig_tbls := $(srctree)/tools/perf/trace/beauty/fsconfig.sh - -$(fsconfig_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsconfig_tbls) - $(Q)$(SHELL) '$(fsconfig_tbls)' $(beauty_uapi_linux_dir) > $@ - -pkey_alloc_access_rights_array := $(beauty_outdir)/pkey_alloc_access_rights_array.c -asm_generic_hdr_dir := $(srctree)/tools/include/uapi/asm-generic/ -pkey_alloc_access_rights_tbl := $(srctree)/tools/perf/trace/beauty/pkey_alloc_access_rights.sh - -$(pkey_alloc_access_rights_array): $(asm_generic_hdr_dir)/mman-common.h $(pkey_alloc_access_rights_tbl) - $(Q)$(SHELL) '$(pkey_alloc_access_rights_tbl)' $(asm_generic_hdr_dir) > $@ - -sndrv_ctl_ioctl_array := $(beauty_ioctl_outdir)/sndrv_ctl_ioctl_array.c -sndrv_ctl_hdr_dir := $(srctree)/tools/include/uapi/sound -sndrv_ctl_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/sndrv_ctl_ioctl.sh - -$(sndrv_ctl_ioctl_array): $(beauty_uapi_sound_dir)/asound.h $(sndrv_ctl_ioctl_tbl) - $(Q)$(SHELL) '$(sndrv_ctl_ioctl_tbl)' $(beauty_uapi_sound_dir) > $@ - -sndrv_pcm_ioctl_array := $(beauty_ioctl_outdir)/sndrv_pcm_ioctl_array.c -sndrv_pcm_hdr_dir := $(srctree)/tools/include/uapi/sound -sndrv_pcm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/sndrv_pcm_ioctl.sh - -$(sndrv_pcm_ioctl_array): $(beauty_uapi_sound_dir)/asound.h $(sndrv_pcm_ioctl_tbl) - $(Q)$(SHELL) '$(sndrv_pcm_ioctl_tbl)' $(beauty_uapi_sound_dir) > $@ - -kcmp_type_array := $(beauty_outdir)/kcmp_type_array.c -kcmp_hdr_dir := $(srctree)/tools/include/uapi/linux/ -kcmp_type_tbl := $(srctree)/tools/perf/trace/beauty/kcmp_type.sh - -$(kcmp_type_array): $(kcmp_hdr_dir)/kcmp.h $(kcmp_type_tbl) - $(Q)$(SHELL) '$(kcmp_type_tbl)' $(kcmp_hdr_dir) > $@ - -kvm_ioctl_array := $(beauty_ioctl_outdir)/kvm_ioctl_array.c -kvm_hdr_dir := $(srctree)/tools/include/uapi/linux -kvm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/kvm_ioctl.sh - -$(kvm_ioctl_array): $(kvm_hdr_dir)/kvm.h $(kvm_ioctl_tbl) - $(Q)$(SHELL) '$(kvm_ioctl_tbl)' $(kvm_hdr_dir) > $@ - -socket_arrays := $(beauty_outdir)/socket.c -socket_tbl := $(srctree)/tools/perf/trace/beauty/socket.sh - -$(socket_arrays): $(linux_uapi_dir)/in.h $(beauty_linux_dir)/socket.h $(socket_tbl) - $(Q)$(SHELL) '$(socket_tbl)' $(linux_uapi_dir) $(beauty_linux_dir) > $@ - -sockaddr_arrays := $(beauty_outdir)/sockaddr.c -sockaddr_tbl := $(srctree)/tools/perf/trace/beauty/sockaddr.sh - -$(sockaddr_arrays): $(beauty_linux_dir)/socket.h $(sockaddr_tbl) - $(Q)$(SHELL) '$(sockaddr_tbl)' $(beauty_linux_dir) > $@ - -vhost_virtio_ioctl_array := $(beauty_ioctl_outdir)/vhost_virtio_ioctl_array.c -vhost_virtio_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/vhost_virtio_ioctl.sh - -$(vhost_virtio_ioctl_array): $(beauty_uapi_linux_dir)/vhost.h $(vhost_virtio_ioctl_tbl) - $(Q)$(SHELL) '$(vhost_virtio_ioctl_tbl)' $(beauty_uapi_linux_dir) > $@ - -perf_ioctl_array := $(beauty_ioctl_outdir)/perf_ioctl_array.c -perf_hdr_dir := $(srctree)/tools/include/uapi/linux -perf_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/perf_ioctl.sh - -$(perf_ioctl_array): $(perf_hdr_dir)/perf_event.h $(perf_ioctl_tbl) - $(Q)$(SHELL) '$(perf_ioctl_tbl)' $(perf_hdr_dir) > $@ - -madvise_behavior_array := $(beauty_outdir)/madvise_behavior_array.c -madvise_hdr_dir := $(srctree)/tools/include/uapi/asm-generic/ -madvise_behavior_tbl := $(srctree)/tools/perf/trace/beauty/madvise_behavior.sh - -$(madvise_behavior_array): $(madvise_hdr_dir)/mman-common.h $(madvise_behavior_tbl) - $(Q)$(SHELL) '$(madvise_behavior_tbl)' $(madvise_hdr_dir) > $@ - -mmap_flags_array := $(beauty_outdir)/mmap_flags_array.c -mmap_flags_tbl := $(srctree)/tools/perf/trace/beauty/mmap_flags.sh - -$(mmap_flags_array): $(linux_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman-common.h $(mmap_flags_tbl) - $(Q)$(SHELL) '$(mmap_flags_tbl)' $(linux_uapi_dir) $(asm_generic_uapi_dir) $(arch_asm_uapi_dir) > $@ - -mremap_flags_array := $(beauty_outdir)/mremap_flags_array.c -mremap_flags_tbl := $(srctree)/tools/perf/trace/beauty/mremap_flags.sh - -$(mremap_flags_array): $(linux_uapi_dir)/mman.h $(mremap_flags_tbl) - $(Q)$(SHELL) '$(mremap_flags_tbl)' $(linux_uapi_dir) > $@ - -mount_flags_array := $(beauty_outdir)/mount_flags_array.c -mount_flags_tbl := $(srctree)/tools/perf/trace/beauty/mount_flags.sh - -$(mount_flags_array): $(beauty_uapi_linux_dir)/mount.h $(mount_flags_tbl) - $(Q)$(SHELL) '$(mount_flags_tbl)' $(beauty_uapi_linux_dir) > $@ - -move_mount_flags_array := $(beauty_outdir)/move_mount_flags_array.c -move_mount_flags_tbl := $(srctree)/tools/perf/trace/beauty/move_mount_flags.sh - -$(move_mount_flags_array): $(beauty_uapi_linux_dir)/mount.h $(move_mount_flags_tbl) - $(Q)$(SHELL) '$(move_mount_flags_tbl)' $(beauty_uapi_linux_dir) > $@ - -mmap_prot_array := $(beauty_outdir)/mmap_prot_array.c -mmap_prot_tbl := $(srctree)/tools/perf/trace/beauty/mmap_prot.sh - -$(mmap_prot_array): $(asm_generic_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman-common.h $(mmap_prot_tbl) - $(Q)$(SHELL) '$(mmap_prot_tbl)' $(asm_generic_uapi_dir) $(arch_asm_uapi_dir) > $@ - -prctl_option_array := $(beauty_outdir)/prctl_option_array.c -prctl_option_tbl := $(srctree)/tools/perf/trace/beauty/prctl_option.sh - -$(prctl_option_array): $(beauty_uapi_linux_dir)/prctl.h $(prctl_option_tbl) - $(Q)$(SHELL) '$(prctl_option_tbl)' $(beauty_uapi_linux_dir) > $@ - -usbdevfs_ioctl_array := $(beauty_ioctl_outdir)/usbdevfs_ioctl_array.c -usbdevfs_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/usbdevfs_ioctl.sh - -$(usbdevfs_ioctl_array): $(beauty_uapi_linux_dir)/usbdevice_fs.h $(usbdevfs_ioctl_tbl) - $(Q)$(SHELL) '$(usbdevfs_ioctl_tbl)' $(beauty_uapi_linux_dir) > $@ - -x86_arch_prctl_code_array := $(beauty_outdir)/x86_arch_prctl_code_array.c -x86_arch_prctl_code_tbl := $(srctree)/tools/perf/trace/beauty/x86_arch_prctl.sh - -$(x86_arch_prctl_code_array): $(beauty_x86_arch_asm_uapi_dir)/prctl.h $(x86_arch_prctl_code_tbl) - $(Q)$(SHELL) '$(x86_arch_prctl_code_tbl)' $(beauty_x86_arch_asm_uapi_dir) > $@ - -x86_arch_irq_vectors_array := $(beauty_outdir)/x86_arch_irq_vectors_array.c -x86_arch_irq_vectors_tbl := $(srctree)/tools/perf/trace/beauty/tracepoints/x86_irq_vectors.sh - -$(x86_arch_irq_vectors_array): $(beauty_arch_asm_dir)/irq_vectors.h $(x86_arch_irq_vectors_tbl) - $(Q)$(SHELL) '$(x86_arch_irq_vectors_tbl)' $(beauty_arch_asm_dir) > $@ - -x86_arch_MSRs_array := $(beauty_outdir)/x86_arch_MSRs_array.c -x86_arch_MSRs_tbl := $(srctree)/tools/perf/trace/beauty/tracepoints/x86_msr.sh - -$(x86_arch_MSRs_array): $(x86_arch_asm_dir)/msr-index.h $(x86_arch_MSRs_tbl) - $(Q)$(SHELL) '$(x86_arch_MSRs_tbl)' $(x86_arch_asm_dir) > $@ - -rename_flags_array := $(beauty_outdir)/rename_flags_array.c -rename_flags_tbl := $(srctree)/tools/perf/trace/beauty/rename_flags.sh - -$(rename_flags_array): $(beauty_uapi_linux_dir)/fs.h $(rename_flags_tbl) - $(Q)$(SHELL) '$(rename_flags_tbl)' $(beauty_uapi_linux_dir) > $@ - -arch_errno_name_array := $(beauty_outdir)/arch_errno_name_array.c -arch_errno_hdr_dir := $(srctree)/tools -arch_errno_tbl := $(srctree)/tools/perf/trace/beauty/arch_errno_names.sh - -$(arch_errno_name_array): $(arch_errno_tbl) - $(Q)$(SHELL) '$(arch_errno_tbl)' '$(patsubst -%,,$(CC))' $(arch_errno_hdr_dir) > $@ - -statx_mask_array := $(beauty_outdir)/statx_mask_array.c -statx_mask_tbl := $(srctree)/tools/perf/trace/beauty/statx_mask.sh - -$(statx_mask_array): $(beauty_uapi_linux_dir)/stat.h $(statx_mask_tbl) - $(Q)$(SHELL) '$(statx_mask_tbl)' $(beauty_uapi_linux_dir) > $@ - -sync_file_range_arrays := $(beauty_outdir)/sync_file_range_arrays.c -sync_file_range_tbls := $(srctree)/tools/perf/trace/beauty/sync_file_range.sh - -$(sync_file_range_arrays): $(beauty_uapi_linux_dir)/fs.h $(sync_file_range_tbls) - $(Q)$(SHELL) '$(sync_file_range_tbls)' $(beauty_uapi_linux_dir) > $@ - -TESTS_CORESIGHT_DIR := $(srctree)/tools/perf/tests/shell/coresight - -tests-coresight-targets: FORCE - $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR) - -tests-coresight-targets-clean: - $(call QUIET_CLEAN, coresight) - $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR) O=$(OUTPUT) clean >/dev/null - -all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) tests-coresight-targets +all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) # Create python binding output directory if not already present $(shell [ -d '$(OUTPUT)python' ] || mkdir -p '$(OUTPUT)python') -$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py $(PERFLIBS_PY) +$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py python/perf.pyi $(PERFLIBS_PY) $(QUIET_GEN)LDSHARED="$(CC) -pthread -shared" \ CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS) $(LIBS_PY)' \ $(PYTHON_WORD) util/setup.py \ - --quiet build_ext; \ - cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/ + $(python_setup_quiet) build_ext; \ + cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/; \ + if [ "$(abspath python/perf.pyi)" != "$(abspath $(OUTPUT)python/perf.pyi)" ]; then \ + cp python/perf.pyi $(OUTPUT)python/; \ + fi python_perf_target: @echo "Target is: $(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX)" @@ -774,13 +546,15 @@ build := -f $(srctree)/tools/build/Makefile.build dir=. obj $(PERF_IN): prepare FORCE $(Q)$(MAKE) $(build)=perf -$(LIBPMU_EVENTS_IN): FORCE prepare +$(LIBPMU_EVENTS_IN): FORCE $(LIBPERF) $(Q)$(MAKE) -f $(srctree)/tools/build/Makefile.build dir=pmu-events obj=pmu-events $(LIBPMU_EVENTS): $(LIBPMU_EVENTS_IN) $(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $< -$(LIBPERF_BENCH_IN): FORCE prepare +# The $(LIBPERF_UTIL) dependency is to ensure bpftool and vmlinux.h +# aren't racily built for bench/bpf_skel/bench_uprobe.bpf.c +$(LIBPERF_BENCH_IN): FORCE prepare $(LIBSYMBOL) | $(LIBPERF_UTIL) $(Q)$(MAKE) $(build)=perf-bench $(LIBPERF_BENCH): $(LIBPERF_BENCH_IN) @@ -798,7 +572,7 @@ $(LIBPERF_UI_IN): FORCE prepare $(LIBPERF_UI): $(LIBPERF_UI_IN) $(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $< -$(LIBPERF_UTIL_IN): FORCE prepare +$(LIBPERF_UTIL_IN): FORCE prepare $(LIBSYMBOL) $(Q)$(MAKE) $(build)=perf-util $(LIBPERF_UTIL): $(LIBPERF_UTIL_IN) @@ -814,8 +588,8 @@ $(GTK_IN): FORCE prepare $(OUTPUT)libperf-gtk.so: $(GTK_IN) $(PERFLIBS) $(QUIET_LINK)$(CC) -o $@ -shared $(LDFLAGS) $(filter %.o,$^) $(GTK_LIBS) -$(SCRIPTS) : % : %.sh - $(QUIET_GEN)$(INSTALL) '$@.sh' '$(OUTPUT)$@' +$(SCRIPTS) : $(OUTPUT)% : %.sh + $(QUIET_GEN)$(INSTALL) '$<' '$@' $(OUTPUT)PERF-VERSION-FILE: .FORCE-PERF-VERSION-FILE $(Q)$(SHELL_PATH) util/PERF-VERSION-GEN $(OUTPUT) @@ -851,46 +625,15 @@ endif __build-dir = $(subst $(OUTPUT),,$(dir $@)) build-dir = $(or $(__build-dir),.) -prepare: $(OUTPUT)PERF-VERSION-FILE archheaders \ +bpf-skel-prepare: + $(Q)$(MAKE) -f bpf_skel.mak bpf-skel-prepare + +prepare: $(OUTPUT)PERF-VERSION-FILE \ arm64-sysreg-defs \ - $(syscall_array) \ - $(fs_at_flags_array) \ - $(clone_flags_array) \ - $(drm_ioctl_array) \ - $(fadvise_advice_array) \ - $(fsconfig_arrays) \ - $(fsmount_arrays) \ - $(fsmount_attr_arrays) \ - $(fspick_arrays) \ - $(pkey_alloc_access_rights_array) \ - $(sndrv_pcm_ioctl_array) \ - $(sndrv_ctl_ioctl_array) \ - $(kcmp_type_array) \ - $(kvm_ioctl_array) \ - $(socket_arrays) \ - $(sockaddr_arrays) \ - $(vhost_virtio_ioctl_array) \ - $(madvise_behavior_array) \ - $(mmap_flags_array) \ - $(mmap_prot_array) \ - $(mremap_flags_array) \ - $(mount_flags_array) \ - $(move_mount_flags_array) \ - $(perf_ioctl_array) \ - $(prctl_option_array) \ - $(usbdevfs_ioctl_array) \ - $(x86_arch_irq_vectors_array) \ - $(x86_arch_MSRs_array) \ - $(x86_arch_prctl_code_array) \ - $(rename_flags_array) \ - $(arch_errno_name_array) \ - $(statx_mask_array) \ - $(sync_file_range_arrays) \ + bpf-skel-prepare \ $(LIBAPI) \ $(LIBPERF) \ - $(LIBSUBCMD) \ - $(LIBSYMBOL) \ - bpf-skel + $(LIBSUBCMD) ifdef LIBBPF_STATIC prepare: $(LIBBPF) @@ -937,7 +680,7 @@ $(OUTPUT)dlfilters/%.o: dlfilters/%.c include/perf/perf_dlfilter.h .SECONDARY: $(DLFILTERS:.so=.o) $(OUTPUT)dlfilters/%.so: $(OUTPUT)dlfilters/%.o - $(QUIET_LINK)$(CC) $(EXTRA_CFLAGS) -shared -o $@ $< + $(QUIET_LINK)$(CC) $(LDFLAGS) $(EXTRA_CFLAGS) -shared -o $@ $< ifndef NO_JVMTI LIBJVMTI_IN := $(OUTPUT)jvmti/jvmti-in.o @@ -1151,14 +894,13 @@ install-tests: all install-gtk $(INSTALL) tests/shell/base_report/*.txt '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/base_report'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight' ; \ $(INSTALL) tests/shell/coresight/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight' - $(Q)$(MAKE) -C tests/shell/coresight install-tests install-bin: install-tools install-tests install: install-bin try-install-man install-python_ext: - $(PYTHON_WORD) util/setup.py --quiet install --root='/$(DESTDIR_SQ)' + $(PYTHON_WORD) util/setup.py $(python_setup_quiet) install --root='/$(DESTDIR_SQ)' # 'make install-doc' should call 'make -C Documentation install' $(INSTALL_DOC_TARGETS): @@ -1169,114 +911,16 @@ $(INSTALL_DOC_TARGETS): python-clean: $(python-clean) -SKEL_OUT := $(abspath $(OUTPUT)util/bpf_skel) -SKEL_TMP_OUT := $(abspath $(SKEL_OUT)/.tmp) -SKELETONS := $(SKEL_OUT)/bpf_prog_profiler.skel.h -SKELETONS += $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h -SKELETONS += $(SKEL_OUT)/bperf_cgroup.skel.h $(SKEL_OUT)/func_latency.skel.h -SKELETONS += $(SKEL_OUT)/off_cpu.skel.h $(SKEL_OUT)/lock_contention.skel.h -SKELETONS += $(SKEL_OUT)/kwork_trace.skel.h $(SKEL_OUT)/sample_filter.skel.h -SKELETONS += $(SKEL_OUT)/kwork_top.skel.h $(SKEL_OUT)/syscall_summary.skel.h -SKELETONS += $(SKEL_OUT)/bench_uprobe.skel.h -SKELETONS += $(SKEL_OUT)/augmented_raw_syscalls.skel.h - -$(SKEL_TMP_OUT) $(LIBAPI_OUTPUT) $(LIBBPF_OUTPUT) $(LIBPERF_OUTPUT) $(LIBSUBCMD_OUTPUT) $(LIBSYMBOL_OUTPUT): +$(LIBAPI_OUTPUT) $(LIBBPF_OUTPUT) $(LIBPERF_OUTPUT) $(LIBSUBCMD_OUTPUT) $(LIBSYMBOL_OUTPUT): $(Q)$(MKDIR) -p $@ -ifeq ($(CONFIG_PERF_BPF_SKEL),y) -BPFTOOL := $(SKEL_TMP_OUT)/bootstrap/bpftool -# Get Clang's default includes on this system, as opposed to those seen by -# '--target=bpf'. This fixes "missing" files on some architectures/distros, -# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. -# -# Use '-idirafter': Don't interfere with include mechanics except where the -# build would have failed anyways. -define get_sys_includes -$(shell $(1) $(2) -v -E - &1 \ - | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ -$(shell $(1) $(2) -dM -E - $@ -else - $(Q)cp "$(VMLINUX_H)" $@ -endif - -$(SKEL_TMP_OUT)/%.bpf.o: $(OUTPUT)PERF-VERSION-FILE util/bpf_skel/perf_version.h | $(SKEL_TMP_OUT) -$(SKEL_TMP_OUT)/%.bpf.o: util/bpf_skel/%.bpf.c $(LIBBPF) $(SKEL_OUT)/vmlinux.h - $(QUIET_CLANG)$(CLANG) -g -O2 -fno-stack-protector --target=bpf \ - $(CLANG_OPTIONS) $(EXTRA_BPF_FLAGS) $(BPF_INCLUDE) $(TOOLS_UAPI_INCLUDE) \ - -include $(OUTPUT)PERF-VERSION-FILE -include util/bpf_skel/perf_version.h \ - -c $(filter util/bpf_skel/%.bpf.c,$^) -o $@ - -$(SKEL_OUT)/%.skel.h: $(SKEL_TMP_OUT)/%.bpf.o | $(BPFTOOL) - $(QUIET_GENSKEL)$(BPFTOOL) gen skeleton $< > $@ - -bpf-skel: $(SKELETONS) - -.PRECIOUS: $(SKEL_TMP_OUT)/%.bpf.o - -else # CONFIG_PERF_BPF_SKEL - -bpf-skel: - -endif # CONFIG_PERF_BPF_SKEL - bpf-skel-clean: - $(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TMP_OUT) $(SKELETONS) $(SKEL_OUT)/vmlinux.h + $(Q)$(MAKE) -f bpf_skel.mak clean pmu-events-clean: ifeq ($(OUTPUT),) $(call QUIET_CLEAN, pmu-events) $(RM) \ - pmu-events/pmu-events.c \ + pmu-events/pmu-events*.c \ pmu-events/metric_test.log \ pmu-events/test-empty-pmu-events.c \ pmu-events/empty-pmu-events.log @@ -1284,7 +928,7 @@ ifeq ($(OUTPUT),) -name 'extra-metricgroups.json' -delete else # When an OUTPUT directory is present, clean up the copied pmu-events/arch directory. $(call QUIET_CLEAN, pmu-events) $(RM) -r $(OUTPUT)pmu-events/arch \ - $(OUTPUT)pmu-events/pmu-events.c \ + $(OUTPUT)pmu-events/pmu-events*.c \ $(OUTPUT)pmu-events/metric_test.log \ $(OUTPUT)pmu-events/test-empty-pmu-events.c \ $(OUTPUT)pmu-events/empty-pmu-events.log @@ -1292,7 +936,7 @@ endif clean:: $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBSYMBOL)-clean $(LIBPERF)-clean \ arm64-sysreg-defs-clean fixdep-clean python-clean bpf-skel-clean \ - tests-coresight-targets-clean pmu-events-clean + pmu-events-clean $(call QUIET_CLEAN, core-objs) $(RM) $(LIBPERF_A) $(OUTPUT)perf-archive \ $(OUTPUT)perf-iostat $(LANG_BINDINGS) $(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '*.a' -delete -o \ @@ -1304,36 +948,8 @@ clean:: $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBSYMBOL)-clean $( TAGS tags cscope* $(OUTPUT)PERF-VERSION-FILE \ $(OUTPUT)FEATURE-DUMP $(OUTPUT)util/*-bison* $(OUTPUT)util/*-flex* \ $(OUTPUT)util/intel-pt-decoder/inat-tables.c \ - $(OUTPUT)tests/llvm-src-{base,kbuild,prologue,relocation}.c \ - $(OUTPUT)$(fadvise_advice_array) \ - $(OUTPUT)$(fsconfig_arrays) \ - $(OUTPUT)$(fsmount_arrays) \ - $(OUTPUT)$(fsmount_attr_arrays) \ - $(OUTPUT)$(fspick_arrays) \ - $(OUTPUT)$(madvise_behavior_array) \ - $(OUTPUT)$(mmap_flags_array) \ - $(OUTPUT)$(mmap_prot_array) \ - $(OUTPUT)$(mremap_flags_array) \ - $(OUTPUT)$(mount_flags_array) \ - $(OUTPUT)$(move_mount_flags_array) \ - $(OUTPUT)$(drm_ioctl_array) \ - $(OUTPUT)$(pkey_alloc_access_rights_array) \ - $(OUTPUT)$(sndrv_ctl_ioctl_array) \ - $(OUTPUT)$(sndrv_pcm_ioctl_array) \ - $(OUTPUT)$(kvm_ioctl_array) \ - $(OUTPUT)$(kcmp_type_array) \ - $(OUTPUT)$(socket_arrays) \ - $(OUTPUT)$(sockaddr_arrays) \ - $(OUTPUT)$(vhost_virtio_ioctl_array) \ - $(OUTPUT)$(perf_ioctl_array) \ - $(OUTPUT)$(prctl_option_array) \ - $(OUTPUT)$(usbdevfs_ioctl_array) \ - $(OUTPUT)$(x86_arch_irq_vectors_array) \ - $(OUTPUT)$(x86_arch_MSRs_array) \ - $(OUTPUT)$(x86_arch_prctl_code_array) \ - $(OUTPUT)$(rename_flags_array) \ - $(OUTPUT)$(arch_errno_name_array) \ - $(OUTPUT)$(sync_file_range_arrays) + $(OUTPUT)tests/llvm-src-{base,kbuild,prologue,relocation}.c + $(Q)$(RM) -r $(OUTPUT)trace/beauty/generated $(call QUIET_CLEAN, Documentation) \ $(MAKE) -C $(DOC_DIR) O=$(OUTPUT) clean >/dev/null @@ -1353,8 +969,8 @@ FORCE: .PHONY: all install clean config-clean strip install-gtk .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell -.PHONY: .FORCE-PERF-VERSION-FILE TAGS tags cscope FORCE prepare -.PHONY: archheaders python_perf_target +.PHONY: .FORCE-PERF-VERSION-FILE TAGS tags cscope FORCE prepare bpf-skel-prepare +.PHONY: python_perf_target endif # force_fixdep diff --git a/tools/perf/arch/arm/util/Build b/tools/perf/arch/arm/util/Build index b94bf3c5279a..768ae5d16553 100644 --- a/tools/perf/arch/arm/util/Build +++ b/tools/perf/arch/arm/util/Build @@ -1,3 +1 @@ -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o - perf-util-y += pmu.o auxtrace.o cs-etm.o diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index b7a839de8707..d2861d66a661 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -3,10 +3,13 @@ * Copyright(C) 2015 Linaro Limited. All rights reserved. * Author: Mathieu Poirier */ +#include "../../../util/cs-etm.h" + +#include +#include -#include -#include #include +#include #include #include #include @@ -14,25 +17,24 @@ #include #include #include +#include + +#include +#include // page_size -#include "cs-etm.h" -#include "../../../util/debug.h" -#include "../../../util/record.h" #include "../../../util/auxtrace.h" #include "../../../util/cpumap.h" +#include "../../../util/debug.h" #include "../../../util/event.h" #include "../../../util/evlist.h" #include "../../../util/evsel.h" -#include "../../../util/perf_api_probe.h" #include "../../../util/evsel_config.h" +#include "../../../util/perf_api_probe.h" +#include "../../../util/pmu.h" #include "../../../util/pmus.h" -#include "../../../util/cs-etm.h" -#include // page_size +#include "../../../util/record.h" #include "../../../util/session.h" - -#include -#include -#include +#include "cs-etm.h" struct cs_etm_recording { struct auxtrace_record itr; @@ -199,7 +201,7 @@ static int cs_etm_validate_config(struct perf_pmu *cs_etm_pmu, { unsigned int idx; int err = 0; - struct perf_cpu_map *event_cpus = evsel->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evsel->evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; @@ -323,7 +325,7 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, container_of(itr, struct cs_etm_recording, itr); struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu; struct evsel *evsel, *cs_etm_evsel = NULL; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); int err = 0; @@ -549,7 +551,7 @@ cs_etm_info_priv_size(struct auxtrace_record *itr, { unsigned int idx; int etmv3 = 0, etmv4 = 0, ete = 0; - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *intersect_cpus; struct perf_cpu cpu; struct perf_pmu *cs_etm_pmu = cs_etm_get_pmu(itr); @@ -788,7 +790,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, u32 offset; u64 nr_cpu, type; struct perf_cpu_map *cpu_map; - struct perf_cpu_map *event_cpus = session->evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(session->evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct cs_etm_recording *ptr = container_of(itr, struct cs_etm_recording, itr); @@ -798,7 +800,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr, if (priv_size != cs_etm_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; /* If the cpu_map has the "any" CPU all online CPUs are involved */ diff --git a/tools/perf/arch/arm/util/unwind-libunwind.c b/tools/perf/arch/arm/util/unwind-libunwind.c deleted file mode 100644 index 438906bf0014..000000000000 --- a/tools/perf/arch/arm/util/unwind-libunwind.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "perf_regs.h" -#include "../../../util/unwind.h" -#include "../../../util/debug.h" - -int libunwind__arch_reg_id(int regnum) -{ - switch (regnum) { - case UNW_ARM_R0: - return PERF_REG_ARM_R0; - case UNW_ARM_R1: - return PERF_REG_ARM_R1; - case UNW_ARM_R2: - return PERF_REG_ARM_R2; - case UNW_ARM_R3: - return PERF_REG_ARM_R3; - case UNW_ARM_R4: - return PERF_REG_ARM_R4; - case UNW_ARM_R5: - return PERF_REG_ARM_R5; - case UNW_ARM_R6: - return PERF_REG_ARM_R6; - case UNW_ARM_R7: - return PERF_REG_ARM_R7; - case UNW_ARM_R8: - return PERF_REG_ARM_R8; - case UNW_ARM_R9: - return PERF_REG_ARM_R9; - case UNW_ARM_R10: - return PERF_REG_ARM_R10; - case UNW_ARM_R11: - return PERF_REG_ARM_FP; - case UNW_ARM_R12: - return PERF_REG_ARM_IP; - case UNW_ARM_R13: - return PERF_REG_ARM_SP; - case UNW_ARM_R14: - return PERF_REG_ARM_LR; - case UNW_ARM_R15: - return PERF_REG_ARM_PC; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } - - return -EINVAL; -} diff --git a/tools/perf/arch/arm64/util/Build b/tools/perf/arch/arm64/util/Build index 4e06a08d281a..15b7340b70f7 100644 --- a/tools/perf/arch/arm64/util/Build +++ b/tools/perf/arch/arm64/util/Build @@ -1,11 +1,9 @@ -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o perf-util-y += ../../arm/util/auxtrace.o perf-util-y += ../../arm/util/cs-etm.o perf-util-y += ../../arm/util/pmu.o perf-util-y += arm-spe.o perf-util-y += header.o perf-util-y += hisi-ptt.o -perf-util-y += machine.o perf-util-y += mem-events.o perf-util-y += pmu.o perf-util-y += tsc.o diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index f00d72d087fc..1ba803a8d9b4 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -60,7 +60,7 @@ static bool arm_spe_is_set_freq(struct evsel *evsel) */ static struct perf_cpu_map *arm_spe_find_cpus(struct evlist *evlist) { - struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus; struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus(); struct perf_cpu_map *intersect_cpus; @@ -157,7 +157,7 @@ static int arm_spe_info_fill(struct auxtrace_record *itr, if (priv_size != arm_spe_info_priv_size(itr, session->evlist)) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; cpu_map = arm_spe_find_cpus(session->evlist); @@ -363,7 +363,7 @@ static int arm_spe_setup_tracking_event(struct evlist *evlist, { int err; struct evsel *tracking_evsel; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; /* Add dummy event to keep tracking */ err = parse_event(evlist, "dummy:u"); @@ -396,7 +396,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, struct arm_spe_recording *sper = container_of(itr, struct arm_spe_recording, itr); struct evsel *evsel, *tmp; - struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool discard = false; int err; u64 discard_bit; @@ -428,7 +428,8 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, evlist__for_each_entry_safe(evlist, tmp, evsel) { if (evsel__is_aux_event(evsel)) { arm_spe_setup_evsel(evsel, cpus); - if (!evsel__get_config_val(evsel, "discard", &discard_bit)) + if (evsel__config_exists(evsel, "discard") && + !evsel__get_config_val(evsel, "discard", &discard_bit)) discard = !!discard_bit; } } diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c index fe457fd58c9e..52257715d2b7 100644 --- a/tools/perf/arch/arm64/util/hisi-ptt.c +++ b/tools/perf/arch/arm64/util/hisi-ptt.c @@ -53,7 +53,7 @@ static int hisi_ptt_info_fill(struct auxtrace_record *itr, if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; auxtrace_info->type = PERF_AUXTRACE_HISI_PTT; diff --git a/tools/perf/arch/arm64/util/machine.c b/tools/perf/arch/arm64/util/machine.c deleted file mode 100644 index 80fb13c958d9..000000000000 --- a/tools/perf/arch/arm64/util/machine.c +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include "callchain.h" // prototype of arch__add_leaf_frame_record_opts -#include "perf_regs.h" -#include "record.h" - -#define SMPL_REG_MASK(b) (1ULL << (b)) - -void arch__add_leaf_frame_record_opts(struct record_opts *opts) -{ - opts->sample_user_regs |= SMPL_REG_MASK(PERF_REG_ARM64_LR); -} diff --git a/tools/perf/arch/arm64/util/unwind-libunwind.c b/tools/perf/arch/arm64/util/unwind-libunwind.c deleted file mode 100644 index 871af5992298..000000000000 --- a/tools/perf/arch/arm64/util/unwind-libunwind.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#ifndef REMOTE_UNWIND_LIBUNWIND -#include -#include "perf_regs.h" -#include "../../../util/unwind.h" -#endif -#include "../../../util/debug.h" - -int LIBUNWIND__ARCH_REG_ID(int regnum) -{ - if (regnum < 0 || regnum >= PERF_REG_ARM64_EXTENDED_MAX) - return -EINVAL; - - return regnum; -} diff --git a/tools/perf/arch/common.c b/tools/perf/arch/common.c index 21836f70f231..5ad50e331c55 100644 --- a/tools/perf/arch/common.c +++ b/tools/perf/arch/common.c @@ -1,13 +1,18 @@ // SPDX-License-Identifier: GPL-2.0 +#include "common.h" + #include #include #include #include + +#include #include -#include "common.h" -#include "../util/env.h" + +#include + #include "../util/debug.h" -#include +#include "../util/env.h" static const char *const arc_triplets[] = { "arc-linux-", @@ -141,11 +146,40 @@ static int lookup_triplets(const char *const *triplets, const char *name) return -1; } +static bool is_native_compatible(struct perf_env *env, uint16_t target, uint16_t host) +{ + if (target != host) { + /* A 64-bit host can natively disassemble its 32-bit compat architecture */ + if (host == EM_X86_64 && target == EM_386) + return true; + if (host == EM_AARCH64 && target == EM_ARM) + return true; + if (host == EM_PPC64 && target == EM_PPC) + return true; + if (host == EM_SPARCV9 && target == EM_SPARC) + return true; + return false; + } + + /* target == host case */ + if (target == EM_RISCV) { + bool target_is_64 = perf_env__kernel_is_64_bit(env); + bool host_is_64 = (sizeof(void *) == 8); + + /* 32-bit host cannot natively disassemble 64-bit target */ + if (!host_is_64 && target_is_64) + return false; + } + + return true; +} + static int perf_env__lookup_binutils_path(struct perf_env *env, const char *name, char **path) { int idx; - const char *arch = perf_env__arch(env), *cross_env; + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + const char *cross_env; const char *const *path_list; char *buf = NULL; @@ -153,7 +187,7 @@ static int perf_env__lookup_binutils_path(struct perf_env *env, * We don't need to try to find objdump path for native system. * Just use default binutils path (e.g.: "objdump"). */ - if (!strcmp(perf_env__arch(NULL), arch)) + if (is_native_compatible(env, e_machine, EM_HOST)) goto out; cross_env = getenv("CROSS_COMPILE"); @@ -170,30 +204,42 @@ static int perf_env__lookup_binutils_path(struct perf_env *env, zfree(&buf); } - if (!strcmp(arch, "arc")) + switch (e_machine) { + case EM_ARC: path_list = arc_triplets; - else if (!strcmp(arch, "arm")) + break; + case EM_ARM: path_list = arm_triplets; - else if (!strcmp(arch, "arm64")) + break; + case EM_AARCH64: path_list = arm64_triplets; - else if (!strcmp(arch, "powerpc")) + break; + case EM_PPC: + case EM_PPC64: path_list = powerpc_triplets; - else if (!strcmp(arch, "riscv32")) - path_list = riscv32_triplets; - else if (!strcmp(arch, "riscv64")) - path_list = riscv64_triplets; - else if (!strcmp(arch, "sh")) + break; + case EM_RISCV: + path_list = perf_env__kernel_is_64_bit(env) ? riscv64_triplets : riscv32_triplets; + break; + case EM_SH: path_list = sh_triplets; - else if (!strcmp(arch, "s390")) + break; + case EM_S390: path_list = s390_triplets; - else if (!strcmp(arch, "sparc")) + break; + case EM_SPARC: + case EM_SPARCV9: path_list = sparc_triplets; - else if (!strcmp(arch, "x86")) + break; + case EM_X86_64: + case EM_386: path_list = x86_triplets; - else if (!strcmp(arch, "mips")) + break; + case EM_MIPS: path_list = mips_triplets; - else { - ui__error("binutils for %s not supported.\n", arch); + break; + default: + ui__error("binutils for %s not supported.\n", perf_env__arch(env)); goto out_error; } @@ -202,7 +248,7 @@ static int perf_env__lookup_binutils_path(struct perf_env *env, ui__error("Please install %s for %s.\n" "You can add it to PATH, set CROSS_COMPILE or " "override the default using --%s.\n", - name, arch, name); + name, perf_env__arch(env), name); goto out_error; } @@ -237,5 +283,7 @@ int perf_env__lookup_objdump(struct perf_env *env, char **path) */ bool perf_env__single_address_space(struct perf_env *env) { - return strcmp(perf_env__arch(env), "sparc"); + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + + return e_machine != EM_SPARC && e_machine != EM_SPARCV9 && e_machine != EM_S390; } diff --git a/tools/perf/arch/loongarch/util/Build b/tools/perf/arch/loongarch/util/Build index 8d91e78d31c9..2328fb9a30a3 100644 --- a/tools/perf/arch/loongarch/util/Build +++ b/tools/perf/arch/loongarch/util/Build @@ -1,3 +1 @@ perf-util-y += header.o - -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o diff --git a/tools/perf/arch/loongarch/util/unwind-libunwind.c b/tools/perf/arch/loongarch/util/unwind-libunwind.c deleted file mode 100644 index f693167b86ef..000000000000 --- a/tools/perf/arch/loongarch/util/unwind-libunwind.c +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "perf_regs.h" -#include "../../util/unwind.h" -#include "util/debug.h" - -int libunwind__arch_reg_id(int regnum) -{ - switch (regnum) { - case UNW_LOONGARCH64_R1: - return PERF_REG_LOONGARCH_R1; - case UNW_LOONGARCH64_R2: - return PERF_REG_LOONGARCH_R2; - case UNW_LOONGARCH64_R3: - return PERF_REG_LOONGARCH_R3; - case UNW_LOONGARCH64_R4: - return PERF_REG_LOONGARCH_R4; - case UNW_LOONGARCH64_R5: - return PERF_REG_LOONGARCH_R5; - case UNW_LOONGARCH64_R6: - return PERF_REG_LOONGARCH_R6; - case UNW_LOONGARCH64_R7: - return PERF_REG_LOONGARCH_R7; - case UNW_LOONGARCH64_R8: - return PERF_REG_LOONGARCH_R8; - case UNW_LOONGARCH64_R9: - return PERF_REG_LOONGARCH_R9; - case UNW_LOONGARCH64_R10: - return PERF_REG_LOONGARCH_R10; - case UNW_LOONGARCH64_R11: - return PERF_REG_LOONGARCH_R11; - case UNW_LOONGARCH64_R12: - return PERF_REG_LOONGARCH_R12; - case UNW_LOONGARCH64_R13: - return PERF_REG_LOONGARCH_R13; - case UNW_LOONGARCH64_R14: - return PERF_REG_LOONGARCH_R14; - case UNW_LOONGARCH64_R15: - return PERF_REG_LOONGARCH_R15; - case UNW_LOONGARCH64_R16: - return PERF_REG_LOONGARCH_R16; - case UNW_LOONGARCH64_R17: - return PERF_REG_LOONGARCH_R17; - case UNW_LOONGARCH64_R18: - return PERF_REG_LOONGARCH_R18; - case UNW_LOONGARCH64_R19: - return PERF_REG_LOONGARCH_R19; - case UNW_LOONGARCH64_R20: - return PERF_REG_LOONGARCH_R20; - case UNW_LOONGARCH64_R21: - return PERF_REG_LOONGARCH_R21; - case UNW_LOONGARCH64_R22: - return PERF_REG_LOONGARCH_R22; - case UNW_LOONGARCH64_R23: - return PERF_REG_LOONGARCH_R23; - case UNW_LOONGARCH64_R24: - return PERF_REG_LOONGARCH_R24; - case UNW_LOONGARCH64_R25: - return PERF_REG_LOONGARCH_R25; - case UNW_LOONGARCH64_R26: - return PERF_REG_LOONGARCH_R26; - case UNW_LOONGARCH64_R27: - return PERF_REG_LOONGARCH_R27; - case UNW_LOONGARCH64_R28: - return PERF_REG_LOONGARCH_R28; - case UNW_LOONGARCH64_R29: - return PERF_REG_LOONGARCH_R29; - case UNW_LOONGARCH64_R30: - return PERF_REG_LOONGARCH_R30; - case UNW_LOONGARCH64_R31: - return PERF_REG_LOONGARCH_R31; - case UNW_LOONGARCH64_PC: - return PERF_REG_LOONGARCH_PC; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } - - return -EINVAL; -} diff --git a/tools/perf/arch/mips/Build b/tools/perf/arch/mips/Build deleted file mode 100644 index e63eabc2c8f4..000000000000 --- a/tools/perf/arch/mips/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-y += util/ diff --git a/tools/perf/arch/mips/util/Build b/tools/perf/arch/mips/util/Build deleted file mode 100644 index 818b808a8247..000000000000 --- a/tools/perf/arch/mips/util/Build +++ /dev/null @@ -1 +0,0 @@ -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o diff --git a/tools/perf/arch/mips/util/unwind-libunwind.c b/tools/perf/arch/mips/util/unwind-libunwind.c deleted file mode 100644 index 0d8c99c29da6..000000000000 --- a/tools/perf/arch/mips/util/unwind-libunwind.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "perf_regs.h" -#include "../../util/unwind.h" -#include "util/debug.h" - -int libunwind__arch_reg_id(int regnum) -{ - switch (regnum) { - case UNW_MIPS_R1 ... UNW_MIPS_R25: - return regnum - UNW_MIPS_R1 + PERF_REG_MIPS_R1; - case UNW_MIPS_R28 ... UNW_MIPS_R31: - return regnum - UNW_MIPS_R28 + PERF_REG_MIPS_R28; - case UNW_MIPS_PC: - return PERF_REG_MIPS_PC; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } -} diff --git a/tools/perf/arch/powerpc/include/dwarf-regs-table.h b/tools/perf/arch/powerpc/include/dwarf-regs-table.h index 66dc015a733d..7e746cb31b66 100644 --- a/tools/perf/arch/powerpc/include/dwarf-regs-table.h +++ b/tools/perf/arch/powerpc/include/dwarf-regs-table.h @@ -7,6 +7,7 @@ * http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi-1.9.html * http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf */ +#undef REG_DWARFNUM_NAME #define REG_DWARFNUM_NAME(reg, idx) [idx] = "%" #reg static const char * const powerpc_regstr_tbl[] = { diff --git a/tools/perf/arch/powerpc/util/Build b/tools/perf/arch/powerpc/util/Build index d66574cbb9a9..ae928050e07a 100644 --- a/tools/perf/arch/powerpc/util/Build +++ b/tools/perf/arch/powerpc/util/Build @@ -6,5 +6,4 @@ perf-util-y += evsel.o perf-util-$(CONFIG_LIBDW) += skip-callchain-idx.o -perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o perf-util-y += auxtrace.o diff --git a/tools/perf/arch/powerpc/util/auxtrace.c b/tools/perf/arch/powerpc/util/auxtrace.c index e39deff6c857..4600a1661b4f 100644 --- a/tools/perf/arch/powerpc/util/auxtrace.c +++ b/tools/perf/arch/powerpc/util/auxtrace.c @@ -71,6 +71,12 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, struct evsel *pos; int found = 0; + /* + * Set err value to zero here. Any fail later + * will set appropriate return code to err. + */ + *err = 0; + evlist__for_each_entry(evlist, pos) { if (strstarts(pos->name, "vpa_dtl")) { found = 1; diff --git a/tools/perf/arch/powerpc/util/unwind-libunwind.c b/tools/perf/arch/powerpc/util/unwind-libunwind.c deleted file mode 100644 index 90a6beda20de..000000000000 --- a/tools/perf/arch/powerpc/util/unwind-libunwind.c +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2016 Chandan Kumar, IBM Corporation. - */ - -#include -#include -#include -#include "../../util/unwind.h" -#include "../../util/debug.h" - -int libunwind__arch_reg_id(int regnum) -{ - switch (regnum) { - case UNW_PPC64_R0: - return PERF_REG_POWERPC_R0; - case UNW_PPC64_R1: - return PERF_REG_POWERPC_R1; - case UNW_PPC64_R2: - return PERF_REG_POWERPC_R2; - case UNW_PPC64_R3: - return PERF_REG_POWERPC_R3; - case UNW_PPC64_R4: - return PERF_REG_POWERPC_R4; - case UNW_PPC64_R5: - return PERF_REG_POWERPC_R5; - case UNW_PPC64_R6: - return PERF_REG_POWERPC_R6; - case UNW_PPC64_R7: - return PERF_REG_POWERPC_R7; - case UNW_PPC64_R8: - return PERF_REG_POWERPC_R8; - case UNW_PPC64_R9: - return PERF_REG_POWERPC_R9; - case UNW_PPC64_R10: - return PERF_REG_POWERPC_R10; - case UNW_PPC64_R11: - return PERF_REG_POWERPC_R11; - case UNW_PPC64_R12: - return PERF_REG_POWERPC_R12; - case UNW_PPC64_R13: - return PERF_REG_POWERPC_R13; - case UNW_PPC64_R14: - return PERF_REG_POWERPC_R14; - case UNW_PPC64_R15: - return PERF_REG_POWERPC_R15; - case UNW_PPC64_R16: - return PERF_REG_POWERPC_R16; - case UNW_PPC64_R17: - return PERF_REG_POWERPC_R17; - case UNW_PPC64_R18: - return PERF_REG_POWERPC_R18; - case UNW_PPC64_R19: - return PERF_REG_POWERPC_R19; - case UNW_PPC64_R20: - return PERF_REG_POWERPC_R20; - case UNW_PPC64_R21: - return PERF_REG_POWERPC_R21; - case UNW_PPC64_R22: - return PERF_REG_POWERPC_R22; - case UNW_PPC64_R23: - return PERF_REG_POWERPC_R23; - case UNW_PPC64_R24: - return PERF_REG_POWERPC_R24; - case UNW_PPC64_R25: - return PERF_REG_POWERPC_R25; - case UNW_PPC64_R26: - return PERF_REG_POWERPC_R26; - case UNW_PPC64_R27: - return PERF_REG_POWERPC_R27; - case UNW_PPC64_R28: - return PERF_REG_POWERPC_R28; - case UNW_PPC64_R29: - return PERF_REG_POWERPC_R29; - case UNW_PPC64_R30: - return PERF_REG_POWERPC_R30; - case UNW_PPC64_R31: - return PERF_REG_POWERPC_R31; - case UNW_PPC64_LR: - return PERF_REG_POWERPC_LINK; - case UNW_PPC64_CTR: - return PERF_REG_POWERPC_CTR; - case UNW_PPC64_XER: - return PERF_REG_POWERPC_XER; - case UNW_PPC64_NIP: - return PERF_REG_POWERPC_NIP; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } - return -EINVAL; -} diff --git a/tools/perf/arch/riscv/include/dwarf-regs-table.h b/tools/perf/arch/riscv/include/dwarf-regs-table.h index a45b63a6d5a8..8c42806d34f1 100644 --- a/tools/perf/arch/riscv/include/dwarf-regs-table.h +++ b/tools/perf/arch/riscv/include/dwarf-regs-table.h @@ -2,7 +2,8 @@ #ifdef DEFINE_DWARF_REGSTR_TABLE /* This is included in perf/util/dwarf-regs.c */ -#define REG_DWARFNUM_NAME(reg, idx) [idx] = "%" #reg +#undef REG_DWARFNUM_NAME +#define REG_DWARFNUM_NAME(reg, idx) [idx] = reg static const char * const riscv_regstr_tbl[] = { REG_DWARFNUM_NAME("%zero", 0), diff --git a/tools/perf/arch/riscv/util/header.c b/tools/perf/arch/riscv/util/header.c index 4b839203d4a5..891984e909bd 100644 --- a/tools/perf/arch/riscv/util/header.c +++ b/tools/perf/arch/riscv/util/header.c @@ -19,7 +19,7 @@ static char *_get_field(const char *line) { - char *line2, *nl; + const char *line2, *nl; line2 = strrchr(line, ' '); if (!line2) diff --git a/tools/perf/arch/s390/include/dwarf-regs-table.h b/tools/perf/arch/s390/include/dwarf-regs-table.h index 671553525f41..e90b63157702 100644 --- a/tools/perf/arch/s390/include/dwarf-regs-table.h +++ b/tools/perf/arch/s390/include/dwarf-regs-table.h @@ -2,6 +2,7 @@ #ifndef S390_DWARF_REGS_TABLE_H #define S390_DWARF_REGS_TABLE_H +#undef REG_DWARFNUM_NAME #define REG_DWARFNUM_NAME(reg, idx) [idx] = "%" #reg /* diff --git a/tools/perf/arch/x86/tests/amd-ibs-period.c b/tools/perf/arch/x86/tests/amd-ibs-period.c index cee9e11c05e0..6a92b3a23ed7 100644 --- a/tools/perf/arch/x86/tests/amd-ibs-period.c +++ b/tools/perf/arch/x86/tests/amd-ibs-period.c @@ -932,7 +932,7 @@ static bool kernel_v6_15_or_newer(void) endptr++; minor = strtol(endptr, NULL, 10); - return major >= 6 && minor >= 15; + return major > 6 || (major == 6 && minor >= 15); } int test__amd_ibs_period(struct test_suite *test __maybe_unused, diff --git a/tools/perf/arch/x86/tests/hybrid.c b/tools/perf/arch/x86/tests/hybrid.c index e221ea104174..0477e17b8e53 100644 --- a/tools/perf/arch/x86/tests/hybrid.c +++ b/tools/perf/arch/x86/tests/hybrid.c @@ -26,7 +26,7 @@ static int test__hybrid_hw_event_with_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -38,7 +38,7 @@ static int test__hybrid_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -57,7 +57,7 @@ static int test__hybrid_sw_hw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); @@ -74,7 +74,7 @@ static int test__hybrid_hw_sw_group_event(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -91,7 +91,7 @@ static int test__hybrid_group_modifier1(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -113,7 +113,7 @@ static int test__hybrid_raw1(struct evlist *evlist) { struct perf_evsel *evsel; - perf_evlist__for_each_evsel(&evlist->core, evsel) { + perf_evlist__for_each_evsel(evlist__core(evlist), evsel) { struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->attr.type); TEST_ASSERT_VAL("missing pmu", pmu); @@ -127,7 +127,7 @@ static int test__hybrid_raw2(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", test_config(evsel, 0x1a)); return TEST_OK; @@ -137,7 +137,7 @@ static int test__hybrid_cache_event(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 0x2 == (evsel->core.attr.config & 0xffffffff)); return TEST_OK; @@ -148,7 +148,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type); TEST_ASSERT_VAL("wrong config", 10 == evsel->core.attr.config); TEST_ASSERT_VAL("wrong config1", 1 == evsel->core.attr.config1); @@ -168,7 +168,7 @@ static int test__hybrid_hw_group_event_2(struct evlist *evlist) struct evsel *evsel, *leader; evsel = leader = evlist__first(evlist); - TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist)); TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type); TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW)); TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES)); @@ -268,7 +268,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/arch/x86/tests/topdown.c b/tools/perf/arch/x86/tests/topdown.c index 3ee4e5e71be3..2b6f47ce4932 100644 --- a/tools/perf/arch/x86/tests/topdown.c +++ b/tools/perf/arch/x86/tests/topdown.c @@ -56,7 +56,7 @@ static int event_cb(void *state, struct pmu_event_info *info) *ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); return 0; } @@ -75,4 +75,168 @@ static int test__x86_topdown(struct test_suite *test __maybe_unused, int subtest return ret; } -DEFINE_SUITE("x86 topdown", x86_topdown); +#define CHECK_COND(cond, text) \ +do { \ + if (!(cond)) { \ + pr_debug("FAILED %s:%d %s\n", __FILE__, __LINE__, text); \ + ret = TEST_FAIL; \ + goto out_err; \ + } \ +} while (0) + +#define CHECK_EQUAL(val, expected, text) \ +do { \ + if ((val) != (expected)) { \ + pr_debug("FAILED %s:%d %s (%d != %d)\n", \ + __FILE__, __LINE__, text, (val), (expected)); \ + ret = TEST_FAIL; \ + goto out_err; \ + } \ +} while (0) + +static int test_sort(const char *str, int expected_slots_group_size, + int expected_instructions_group_size) +{ + struct evlist *evlist = NULL; + struct parse_events_error err; + struct evsel *evsel; + int ret = TEST_FAIL; + bool slots_seen = false; + + parse_events_error__init(&err); + + evlist = evlist__new(); + if (!evlist) + goto out_err; + + if (parse_events(evlist, str, &err)) { + pr_debug("parse_events failed for %s\n", str); + goto out_err; + } + + evlist__for_each_entry(evlist, evsel) { + if (!evsel__is_group_leader(evsel)) + continue; + + if (strstr(evsel__name(evsel), "slots")) { + /* + * Slots as a leader means the PMU is for a perf metric + * group as the slots event isn't present when not. + */ + slots_seen = true; + CHECK_EQUAL(evsel->core.nr_members, expected_slots_group_size, + "slots group size"); + if (expected_slots_group_size == 3) { + struct evsel *next = evsel__next(evsel); + struct evsel *next2 = evsel__next(next); + + CHECK_COND(strstr(evsel__name(next), "instructions") != NULL, + "slots second event is instructions"); + CHECK_COND(strstr(evsel__name(next2), "topdown-retiring") != NULL, + "slots third event is topdown-retiring"); + } else if (expected_slots_group_size == 2) { + struct evsel *next = evsel__next(evsel); + + CHECK_COND(strstr(evsel__name(next), "topdown-retiring") != NULL, + "slots second event is topdown-retiring"); + } + } else if (strstr(evsel__name(evsel), "instructions")) { + CHECK_EQUAL(evsel->core.nr_members, expected_instructions_group_size, + "instructions group size"); + if (expected_instructions_group_size == 2) { + /* + * On Intel hybrid CPUs (e.g., Alder Lake/ + * Raptor Lake), E-cores (cpu_atom) do not + * support/enforce the slots event. When + * parsing event groups containing slots + * across all PMUs, slots is automatically + * filtered out from cpu_atom, leaving + * {cpu_atom/instructions/, + * cpu_atom/topdown-retiring/}. On cpu_atom, + * instructions correctly leads this group of + * 2 without slots reordering. + */ + struct evsel *next = evsel__next(evsel); + + CHECK_COND(strstr(evsel__name(next), "topdown-retiring") != NULL, + "instructions second event is topdown-retiring"); + } + } else if (strstr(evsel__name(evsel), "topdown-retiring")) { + /* + * A perf metric event where the PMU doesn't require + * slots as a leader. + */ + CHECK_EQUAL(evsel->core.nr_members, 1, "topdown-retiring group size"); + } else if (strstr(evsel__name(evsel), "cycles")) { + CHECK_EQUAL(evsel->core.nr_members, 1, "cycles group size"); + } + } + CHECK_COND(slots_seen, "slots seen"); + ret = TEST_OK; +out_err: + evlist__put(evlist); + parse_events_error__exit(&err); + return ret; +} + +static int test__x86_topdown_sorting(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + int ret; + + if (!topdown_sys_has_perf_metrics()) + return TEST_OK; + + ret = test_sort("{instructions,topdown-retiring,slots}", 3, 2); + TEST_ASSERT_EQUAL("all events in a group", ret, TEST_OK); + ret = test_sort("instructions,topdown-retiring,slots", 2, 1); + TEST_ASSERT_EQUAL("all events not in a group", ret, TEST_OK); + ret = test_sort("{instructions,slots},topdown-retiring", 2, 1); + TEST_ASSERT_EQUAL("slots event in a group but topdown metrics events outside the group", + ret, TEST_OK); + ret = test_sort("{instructions,slots},{topdown-retiring}", 2, 1); + TEST_ASSERT_EQUAL("slots event and topdown metrics events in two groups", + ret, TEST_OK); + ret = test_sort("{instructions,slots},cycles,topdown-retiring", 2, 1); + TEST_ASSERT_EQUAL("slots event and metrics event are not in a group and not adjacent", + ret, TEST_OK); + + return TEST_OK; +} + +static int test__x86_topdown_slots_injection(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + int ret; + + if (!topdown_sys_has_perf_metrics()) + return TEST_OK; + + ret = test_sort("{instructions,topdown-retiring}", 3, 2); + TEST_ASSERT_EQUAL("all events in a group", ret, TEST_OK); + ret = test_sort("instructions,topdown-retiring", 2, 1); + TEST_ASSERT_EQUAL("all events not in a group", ret, TEST_OK); + ret = test_sort("{instructions},topdown-retiring", 2, 1); + TEST_ASSERT_EQUAL("event in a group but topdown metrics events outside the group", + ret, TEST_OK); + ret = test_sort("{instructions},{topdown-retiring}", 2, 1); + TEST_ASSERT_EQUAL("event and topdown metrics events in two groups", + ret, TEST_OK); + ret = test_sort("{instructions},cycles,topdown-retiring", 2, 1); + TEST_ASSERT_EQUAL("event and metrics event are not in a group and not adjacent", + ret, TEST_OK); + + return TEST_OK; +} + +static struct test_case x86_topdown_tests[] = { + TEST_CASE("topdown events", x86_topdown), + TEST_CASE("topdown sorting", x86_topdown_sorting), + TEST_CASE("topdown slots injection", x86_topdown_slots_injection), + { .name = NULL, } +}; + +struct test_suite suite__x86_topdown = { + .desc = "x86 topdown", + .test_cases = x86_topdown_tests, +}; diff --git a/tools/perf/arch/x86/util/Build b/tools/perf/arch/x86/util/Build index b94c91984c66..7f89fffe4615 100644 --- a/tools/perf/arch/x86/util/Build +++ b/tools/perf/arch/x86/util/Build @@ -8,9 +8,6 @@ perf-util-y += evlist.o perf-util-y += mem-events.o perf-util-y += evsel.o perf-util-y += iostat.o - -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o - perf-util-y += auxtrace.o perf-util-y += intel-pt.o perf-util-y += intel-bts.o diff --git a/tools/perf/arch/x86/util/auxtrace.c b/tools/perf/arch/x86/util/auxtrace.c index ecbf61a7eb3a..84fce0b51ccf 100644 --- a/tools/perf/arch/x86/util/auxtrace.c +++ b/tools/perf/arch/x86/util/auxtrace.c @@ -55,7 +55,7 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { char buffer[64]; - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); int ret; *err = 0; diff --git a/tools/perf/arch/x86/util/intel-bts.c b/tools/perf/arch/x86/util/intel-bts.c index 85c8186300c8..d44d568a6d21 100644 --- a/tools/perf/arch/x86/util/intel-bts.c +++ b/tools/perf/arch/x86/util/intel-bts.c @@ -4,26 +4,30 @@ * Copyright (c) 2013-2015, Intel Corporation. */ +#include "../../../util/intel-bts.h" + #include -#include -#include + #include +#include #include +#include #include +#include // page_size + +#include "../../../util/auxtrace.h" #include "../../../util/cpumap.h" +#include "../../../util/debug.h" #include "../../../util/event.h" -#include "../../../util/evsel.h" #include "../../../util/evlist.h" +#include "../../../util/evsel.h" #include "../../../util/mmap.h" -#include "../../../util/session.h" +#include "../../../util/pmu.h" #include "../../../util/pmus.h" -#include "../../../util/debug.h" #include "../../../util/record.h" +#include "../../../util/session.h" #include "../../../util/tsc.h" -#include "../../../util/auxtrace.h" -#include "../../../util/intel-bts.h" -#include // page_size #define KiB(x) ((x) * 1024) #define MiB(x) ((x) * 1024 * 1024) @@ -75,10 +79,10 @@ static int intel_bts_info_fill(struct auxtrace_record *itr, if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE) return -EINVAL; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -110,7 +114,7 @@ static int intel_bts_recording_options(struct auxtrace_record *itr, container_of(itr, struct intel_bts_recording, itr); struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu; struct evsel *evsel, *intel_bts_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); if (opts->auxtrace_sample_mode) { diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index c131a727774f..a533114c0048 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -3,36 +3,39 @@ * intel_pt.c: Intel Processor Trace support * Copyright (c) 2013-2015, Intel Corporation. */ +#include "../../../util/intel-pt.h" #include #include -#include -#include + #include +#include +#include #include +#include #include -#include -#include "../../../util/session.h" +#include +#include // page_size +#include + +#include "../../../util/auxtrace.h" +#include "../../../util/config.h" +#include "../../../util/cpumap.h" +#include "../../../util/debug.h" #include "../../../util/event.h" #include "../../../util/evlist.h" #include "../../../util/evsel.h" #include "../../../util/evsel_config.h" -#include "../../../util/config.h" -#include "../../../util/cpumap.h" #include "../../../util/mmap.h" -#include #include "../../../util/parse-events.h" -#include "../../../util/pmus.h" -#include "../../../util/debug.h" -#include "../../../util/auxtrace.h" #include "../../../util/perf_api_probe.h" +#include "../../../util/pmu.h" +#include "../../../util/pmus.h" #include "../../../util/record.h" +#include "../../../util/session.h" #include "../../../util/target.h" #include "../../../util/tsc.h" -#include // page_size -#include "../../../util/intel-pt.h" -#include #include "cpuid.h" #define KiB(x) ((x) * 1024) @@ -357,10 +360,10 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu); filter_str_len = filter ? strlen(filter) : 0; - if (!session->evlist->core.nr_mmaps) + if (!evlist__core(session->evlist)->nr_mmaps) return -EINVAL; - pc = session->evlist->mmap[0].core.base; + pc = evlist__mmap(session->evlist)[0].core.base; if (pc) { err = perf_read_tsc_conversion(pc, &tc); if (err) { @@ -373,7 +376,8 @@ static int intel_pt_info_fill(struct auxtrace_record *itr, ui__warning("Intel Processor Trace: TSC not available\n"); } - per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty(session->evlist->core.user_requested_cpus); + per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(session->evlist)->user_requested_cpus); auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; @@ -618,7 +622,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr, struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; bool have_timing_info, need_immediate = false; struct evsel *evsel, *intel_pt_evsel = NULL; - const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus; + const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus; bool privileged = perf_event_paranoid_check(-1); u64 tsc_bit; int err; diff --git a/tools/perf/arch/x86/util/iostat.c b/tools/perf/arch/x86/util/iostat.c index 7442a2cd87ed..b13abea3a6f4 100644 --- a/tools/perf/arch/x86/util/iostat.c +++ b/tools/perf/arch/x86/util/iostat.c @@ -332,13 +332,15 @@ static int iostat_event_group(struct evlist *evl, return ret; } -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config) +int iostat_prepare(struct evlist **evlist_ptr, struct perf_stat_config *config) { - if (evlist->core.nr_entries > 0) { + struct evlist *evlist = *evlist_ptr; + + if (evlist__nr_entries(evlist) > 0) { pr_warning("The -e and -M options are not supported." "All chosen events/metrics will be dropped\n"); - evlist__delete(evlist); - evlist = evlist__new(); + evlist__put(evlist); + *evlist_ptr = evlist = evlist__new(); if (!evlist) return -ENOMEM; } @@ -400,7 +402,7 @@ void iostat_prefix(struct evlist *evlist, struct perf_stat_config *config, char *prefix, struct timespec *ts) { - struct iio_root_port *rp = evlist->selected->priv; + struct iio_root_port *rp = evlist__selected(evlist)->priv; if (rp) { /* @@ -463,7 +465,7 @@ void iostat_print_counters(struct evlist *evlist, iostat_prefix(evlist, config, prefix, ts); fprintf(config->output, "%s", prefix); evlist__for_each_entry(evlist, counter) { - perf_device = evlist->selected->priv; + perf_device = evlist__selected(evlist)->priv; if (perf_device && perf_device != counter->priv) { evlist__set_selected(evlist, counter); iostat_prefix(evlist, config, prefix, ts); diff --git a/tools/perf/arch/x86/util/unwind-libunwind.c b/tools/perf/arch/x86/util/unwind-libunwind.c deleted file mode 100644 index 47357973b55b..000000000000 --- a/tools/perf/arch/x86/util/unwind-libunwind.c +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include "../../util/debug.h" -#ifndef REMOTE_UNWIND_LIBUNWIND -#include -#include "perf_regs.h" -#include "../../util/unwind.h" -#endif - -#ifdef HAVE_ARCH_X86_64_SUPPORT -int LIBUNWIND__ARCH_REG_ID(int regnum) -{ - int id; - - switch (regnum) { - case UNW_X86_64_RAX: - id = PERF_REG_X86_AX; - break; - case UNW_X86_64_RDX: - id = PERF_REG_X86_DX; - break; - case UNW_X86_64_RCX: - id = PERF_REG_X86_CX; - break; - case UNW_X86_64_RBX: - id = PERF_REG_X86_BX; - break; - case UNW_X86_64_RSI: - id = PERF_REG_X86_SI; - break; - case UNW_X86_64_RDI: - id = PERF_REG_X86_DI; - break; - case UNW_X86_64_RBP: - id = PERF_REG_X86_BP; - break; - case UNW_X86_64_RSP: - id = PERF_REG_X86_SP; - break; - case UNW_X86_64_R8: - id = PERF_REG_X86_R8; - break; - case UNW_X86_64_R9: - id = PERF_REG_X86_R9; - break; - case UNW_X86_64_R10: - id = PERF_REG_X86_R10; - break; - case UNW_X86_64_R11: - id = PERF_REG_X86_R11; - break; - case UNW_X86_64_R12: - id = PERF_REG_X86_R12; - break; - case UNW_X86_64_R13: - id = PERF_REG_X86_R13; - break; - case UNW_X86_64_R14: - id = PERF_REG_X86_R14; - break; - case UNW_X86_64_R15: - id = PERF_REG_X86_R15; - break; - case UNW_X86_64_RIP: - id = PERF_REG_X86_IP; - break; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } - - return id; -} -#else -int LIBUNWIND__ARCH_REG_ID(int regnum) -{ - int id; - - switch (regnum) { - case UNW_X86_EAX: - id = PERF_REG_X86_AX; - break; - case UNW_X86_EDX: - id = PERF_REG_X86_DX; - break; - case UNW_X86_ECX: - id = PERF_REG_X86_CX; - break; - case UNW_X86_EBX: - id = PERF_REG_X86_BX; - break; - case UNW_X86_ESI: - id = PERF_REG_X86_SI; - break; - case UNW_X86_EDI: - id = PERF_REG_X86_DI; - break; - case UNW_X86_EBP: - id = PERF_REG_X86_BP; - break; - case UNW_X86_ESP: - id = PERF_REG_X86_SP; - break; - case UNW_X86_EIP: - id = PERF_REG_X86_IP; - break; - default: - pr_err("unwind: invalid reg id %d\n", regnum); - return -EINVAL; - } - - return id; -} -#endif /* HAVE_ARCH_X86_64_SUPPORT */ diff --git a/tools/perf/bench/Build b/tools/perf/bench/Build index b558ab98719f..67b76fe20ba6 100644 --- a/tools/perf/bench/Build +++ b/tools/perf/bench/Build @@ -24,3 +24,9 @@ perf-bench-$(CONFIG_X86_64) += mem-memcpy-x86-64-asm.o perf-bench-$(CONFIG_X86_64) += mem-memset-x86-64-asm.o perf-bench-$(CONFIG_NUMA) += numa.o + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +include $(srctree)/tools/perf/bpf_skel.mak + +$(OUTPUT)bench/uprobe.o: $(SKEL_OUT)/bench_uprobe.skel.h +endif diff --git a/tools/perf/util/bpf_skel/bench_uprobe.bpf.c b/tools/perf/bench/bpf_skel/bench_uprobe.bpf.c similarity index 100% rename from tools/perf/util/bpf_skel/bench_uprobe.bpf.c rename to tools/perf/bench/bpf_skel/bench_uprobe.bpf.c diff --git a/tools/perf/bench/evlist-open-close.c b/tools/perf/bench/evlist-open-close.c index faf9c34b4a5d..748ebbe458f4 100644 --- a/tools/perf/bench/evlist-open-close.c +++ b/tools/perf/bench/evlist-open-close.c @@ -76,7 +76,7 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) parse_events_error__exit(&err); pr_err("Run 'perf list' for a list of valid events\n"); ret = 1; - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__exit(&err); if (uid_str) { @@ -85,24 +85,24 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str) if (uid == UINT_MAX) { pr_err("Invalid User: %s", uid_str); ret = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } ret = parse_uid_filter(evlist, uid); if (ret) - goto out_delete_evlist; + goto out_put_evlist; } ret = evlist__create_maps(evlist, &opts.target); if (ret < 0) { pr_err("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); return evlist; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return NULL; } @@ -116,7 +116,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) return err; } - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_err("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); return err; @@ -124,7 +124,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist) evlist__enable(evlist); evlist__disable(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); return 0; @@ -145,13 +145,14 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) init_stats(&time_stats); - printf(" Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.user_requested_cpus)); - printf(" Number of threads:\t%d\n", evlist->core.threads->nr); + printf(" Number of cpus:\t%d\n", + perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus)); + printf(" Number of threads:\t%d\n", evlist__core(evlist)->threads->nr); printf(" Number of events:\t%d (%d fds)\n", - evlist->core.nr_entries, evlist__count_evsel_fds(evlist)); + evlist__nr_entries(evlist), evlist__count_evsel_fds(evlist)); printf(" Number of iterations:\t%d\n", iterations); - evlist__delete(evlist); + evlist__put(evlist); for (i = 0; i < iterations; i++) { pr_debug("Started iteration %d\n", i); @@ -162,7 +163,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) gettimeofday(&start, NULL); err = bench__do_evlist_open_close(evlist); if (err) { - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -171,7 +172,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str) runtime_us = timeval2usec(&diff); update_stats(&time_stats, runtime_us); - evlist__delete(evlist); + evlist__put(evlist); pr_debug("Iteration %d took:\t%" PRIu64 "us\n", i, runtime_us); } diff --git a/tools/perf/bench/inject-buildid.c b/tools/perf/bench/inject-buildid.c index aad572a78d7f..bfd2c5ec9488 100644 --- a/tools/perf/bench/inject-buildid.c +++ b/tools/perf/bench/inject-buildid.c @@ -228,9 +228,12 @@ static ssize_t synthesize_sample(struct bench_data *data, struct bench_dso *dso, event.header.type = PERF_RECORD_SAMPLE; event.header.misc = PERF_RECORD_MISC_USER; - event.header.size = perf_event__sample_event_size(&sample, bench_sample_type, 0); - - perf_event__synthesize_sample(&event, bench_sample_type, 0, &sample); + event.header.size = perf_event__sample_event_size(&sample, bench_sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0); + perf_event__synthesize_sample(&event, bench_sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0, &sample); return writen(data->input_pipe[1], &event, event.header.size); } diff --git a/tools/perf/bench/sched-pipe.c b/tools/perf/bench/sched-pipe.c index 70139036d68f..eb20c6d73d06 100644 --- a/tools/perf/bench/sched-pipe.c +++ b/tools/perf/bench/sched-pipe.c @@ -22,7 +22,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -39,6 +41,7 @@ struct thread_data { int epoll_fd; bool cgroup_failed; pthread_t pthread; + char *buf; }; #define LOOPS_DEFAULT 1000000 @@ -48,6 +51,7 @@ static int loops = LOOPS_DEFAULT; static bool threaded; static bool nonblocking; +static unsigned int write_size = sizeof(int); static char *cgrp_names[2]; static struct cgroup *cgrps[2]; @@ -88,6 +92,8 @@ static const struct option options[] = { OPT_BOOLEAN('n', "nonblocking", &nonblocking, "Use non-blocking operations"), OPT_INTEGER('l', "loop", &loops, "Specify number of loops"), OPT_BOOLEAN('T', "threaded", &threaded, "Specify threads/process based task setup"), + OPT_UINTEGER('s', "write-size", &write_size, + "Bytes per ping-pong write (default 4-bytes). Use larger values to exercise the pipe page-allocation path."), OPT_CALLBACK('G', "cgroups", NULL, "SEND,RECV", "Put sender and receivers in given cgroups", parse_two_cgroups), @@ -170,25 +176,77 @@ static void exit_cgroup(int nr) free(cgrp_names[nr]); } +/* Sleep until @fd is writable, so we don't busy-spin on EWOULDBLOCK. */ +static inline void wait_writable(int fd) +{ + struct pollfd pfd = { + .fd = fd, + .events = POLLOUT, + }; + + poll(&pfd, 1, -1); +} + +/* + * Loop on short read()/write(): the kernel may return fewer bytes than + * requested, retry on EINTR, and in non-blocking mode wait via poll() + * when the writer transiently hits EWOULDBLOCK while the peer is still + * draining a full pipe (capacity is sized to write_size). + */ +static inline int write_pipe(struct thread_data *td) +{ + unsigned int done = 0; + int ret; + + while (done < write_size) { + ret = write(td->pipe_write, td->buf + done, write_size - done); + if (ret < 0) { + if (errno == EINTR) + continue; + if (nonblocking && errno == EWOULDBLOCK) { + wait_writable(td->pipe_write); + continue; + } + return ret; + } + done += ret; + } + return done; +} + static inline int read_pipe(struct thread_data *td) { - int ret, m; -retry: - if (nonblocking) { - ret = epoll_wait(td->epoll_fd, &td->epoll_ev, 1, -1); - if (ret < 0) + unsigned int done = 0; + int ret; + + while (done < write_size) { + if (nonblocking) { + ret = epoll_wait(td->epoll_fd, &td->epoll_ev, 1, -1); + if (ret < 0) { + if (errno == EINTR) + continue; + return ret; + } + } + ret = read(td->pipe_read, td->buf + done, write_size - done); + if (ret < 0) { + if (errno == EINTR) + continue; + if (nonblocking && errno == EWOULDBLOCK) + continue; return ret; + } + if (ret == 0) + return done; + done += ret; } - ret = read(td->pipe_read, &m, sizeof(int)); - if (nonblocking && ret < 0 && errno == EWOULDBLOCK) - goto retry; - return ret; + return done; } static void *worker_thread(void *__tdata) { struct thread_data *td = __tdata; - int i, ret, m = 0; + int i, ret; ret = enter_cgroup(td->nr); if (ret < 0) { @@ -204,15 +262,38 @@ static void *worker_thread(void *__tdata) } for (i = 0; i < loops; i++) { - ret = write(td->pipe_write, &m, sizeof(int)); - BUG_ON(ret != sizeof(int)); + ret = write_pipe(td); + BUG_ON(ret != (int)write_size); ret = read_pipe(td); - BUG_ON(ret != sizeof(int)); + BUG_ON(ret != (int)write_size); } return NULL; } +/* + * On a custom write_size, resize the pipes so a single payload fits. + */ +static int resize_pipes(int wfd1, int wfd2) +{ + int r1, r2; + + if (write_size <= sizeof(int)) + return 0; + + r1 = fcntl(wfd1, F_SETPIPE_SZ, write_size); + r2 = fcntl(wfd2, F_SETPIPE_SZ, write_size); + if (r1 < 0 || r2 < 0 || + (unsigned int)r1 < write_size || + (unsigned int)r2 < write_size) { + fprintf(stderr, + "--write-size %u exceeds /proc/sys/fs/pipe-max-size\n", + write_size); + return -1; + } + return 0; +} + int bench_sched_pipe(int argc, const char **argv) { struct thread_data threads[2] = {}; @@ -233,12 +314,31 @@ int bench_sched_pipe(int argc, const char **argv) argc = parse_options(argc, argv, options, bench_sched_pipe_usage, 0); + /* + * The error paths below return early without closing the pipes or + * freeing the cgroup state. That is fine: bench_sched_pipe() runs + * once and the process exits right after it returns, so these are + * not real leaks. + */ + if (write_size == 0 || write_size > INT_MAX) { + fprintf(stderr, "--write-size must be in 1..%d\n", INT_MAX); + return -1; + } + if (nonblocking) flags |= O_NONBLOCK; BUG_ON(pipe2(pipe_1, flags)); BUG_ON(pipe2(pipe_2, flags)); + if (resize_pipes(pipe_1[1], pipe_2[1]) < 0) + return -1; + + for (t = 0; t < nr_threads; t++) { + threads[t].buf = calloc(1, write_size); + BUG_ON(!threads[t].buf); + } + gettimeofday(&start, NULL); for (t = 0; t < nr_threads; t++) { @@ -287,6 +387,9 @@ int bench_sched_pipe(int argc, const char **argv) gettimeofday(&stop, NULL); timersub(&stop, &start, &diff); + for (t = 0; t < nr_threads; t++) + free(threads[t].buf); + exit_cgroup(0); exit_cgroup(1); diff --git a/tools/perf/bench/uprobe.c b/tools/perf/bench/uprobe.c index 89697ff788ef..616873bca243 100644 --- a/tools/perf/bench/uprobe.c +++ b/tools/perf/bench/uprobe.c @@ -44,7 +44,7 @@ static const char * const bench_uprobe_usage[] = { }; #ifdef HAVE_BPF_SKEL -#include "bpf_skel/bench_uprobe.skel.h" +#include "bench/bpf_skel/bench_uprobe.skel.h" #define bench_uprobe__attach_uprobe(prog) \ skel->links.prog = bpf_program__attach_uprobe_opts(/*prog=*/skel->progs.prog, \ diff --git a/tools/perf/bpf_skel.mak b/tools/perf/bpf_skel.mak new file mode 100644 index 000000000000..f2559de39f96 --- /dev/null +++ b/tools/perf/bpf_skel.mak @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: GPL-2.0 +# Shared BPF Skeleton Generator Rules + +include $(srctree)/tools/scripts/Makefile.include + +# Shared foundational tooling always lives in util/bpf_skel +SKEL_TOOL_OUT := $(abspath $(OUTPUT)util/bpf_skel) +SKEL_TOOL_TMP_OUT := $(abspath $(SKEL_TOOL_OUT)/.tmp) + +# Component specific output lives in $(dir)/bpf_skel +SKEL_OUT := $(abspath $(OUTPUT)$(dir)/bpf_skel) +SKEL_TMP_OUT := $(abspath $(SKEL_OUT)/.tmp) + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +BPFTOOL := $(SKEL_TOOL_TMP_OUT)/bootstrap/bpftool +VMLINUX_H := $(SKEL_TOOL_OUT)/vmlinux.h + +.PHONY: bpf-skel-prepare +bpf-skel-prepare: $(BPFTOOL) $(VMLINUX_H) + @: + +define get_sys_includes +$(shell $(1) $(2) -v -E - &1 \ + | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ +$(shell $(1) $(2) -dM -E - $@ +else + $(Q)cp "$(VMLINUX_H_FILE)" $@ +endif + +# Consolidated Pattern rule for $(dir)/bpf_skel/ +$(SKEL_TMP_OUT)/%.bpf.o: $(srctree)/tools/perf/$(dir)/bpf_skel/%.bpf.c $(LIBBPF) $(VMLINUX_H) $(OUTPUT)PERF-VERSION-FILE util/bpf_skel/perf_version.h + $(call rule_mkdir) + $(QUIET_CLANG) + $(Q)$(CLANG) -g -O2 -fno-stack-protector --target=bpf \ + $(CLANG_OPTIONS) $(EXTRA_BPF_FLAGS) $(BPF_INCLUDE) $(TOOLS_UAPI_INCLUDE) \ + -include $(OUTPUT)PERF-VERSION-FILE -include util/bpf_skel/perf_version.h \ + -fms-extensions -Wno-microsoft-anon-tag \ + -c $< -o $@ + +$(SKEL_OUT)/%.skel.h: $(SKEL_TMP_OUT)/%.bpf.o $(BPFTOOL) + $(call rule_mkdir) + $(QUIET_GENSKEL) + $(Q)$(BPFTOOL) gen skeleton $< > $@ + +.PRECIOUS: $(SKEL_TMP_OUT)/%.bpf.o + +else # CONFIG_PERF_BPF_SKEL +.PHONY: bpf-skel-prepare +bpf-skel-prepare: + @: +endif # CONFIG_PERF_BPF_SKEL + +clean: + $(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TOOL_TMP_OUT) $(OUTPUT)bench/bpf_skel/.tmp $(SKEL_TOOL_OUT)/*.skel.h $(OUTPUT)bench/bpf_skel/*.skel.h $(SKEL_TOOL_OUT)/vmlinux.h + +.PHONY: clean diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 5e57b78548f4..69cb72b2082a 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -176,29 +176,26 @@ static int hist_iter__branch_callback(struct hist_entry_iter *iter, struct hist_entry *he = iter->he; struct branch_info *bi; struct perf_sample *sample = iter->sample; - struct evsel *evsel = iter->evsel; int err; bi = he->branch_info; - err = addr_map_symbol__inc_samples(&bi->from, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->from, sample); if (err) goto out; - err = addr_map_symbol__inc_samples(&bi->to, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->to, sample); out: return err; } -static int process_branch_callback(struct evsel *evsel, - struct perf_sample *sample, +static int process_branch_callback(struct perf_sample *sample, struct addr_location *al, struct perf_annotate *ann, struct machine *machine) { struct hist_entry_iter iter = { - .evsel = evsel, .sample = sample, .add_entry_cb = hist_iter__branch_callback, .hide_unresolved = symbol_conf.hide_unresolved, @@ -221,8 +218,8 @@ static int process_branch_callback(struct evsel *evsel, if (a.map != NULL) dso__set_hit(map__dso(a.map)); - hist__account_cycles(sample->branch_stack, al, sample, false, - NULL, evsel); + hist__account_cycles(sample->branch_stack, al, sample, /*nonany_branch_mode=*/false, + /*total_cycles=*/NULL); ret = hist_entry_iter__add(&iter, &a, PERF_MAX_STACK_DEPTH, ann); out: @@ -235,11 +232,11 @@ static bool has_annotation(struct perf_annotate *ann) return ui__has_annotation() || ann->use_stdio2; } -static int evsel__add_sample(struct evsel *evsel, struct perf_sample *sample, - struct addr_location *al, struct perf_annotate *ann, - struct machine *machine) +static int add_sample(struct perf_sample *sample, + struct addr_location *al, struct perf_annotate *ann, + struct machine *machine) { - struct hists *hists = evsel__hists(evsel); + struct hists *hists = evsel__hists(sample->evsel); struct hist_entry *he; int ret; @@ -269,13 +266,13 @@ static int evsel__add_sample(struct evsel *evsel, struct perf_sample *sample, process_branch_stack(sample->branch_stack, al, sample); if (ann->has_br_stack && has_annotation(ann)) - return process_branch_callback(evsel, sample, al, ann, machine); + return process_branch_callback(sample, al, ann, machine); he = hists__add_entry(hists, al, NULL, NULL, NULL, NULL, sample, true); if (he == NULL) return -ENOMEM; - ret = hist_entry__inc_addr_samples(he, sample, evsel, al->addr); + ret = hist_entry__inc_addr_samples(he, sample, al->addr); hists__inc_nr_samples(hists, true); return ret; } @@ -283,7 +280,6 @@ static int evsel__add_sample(struct evsel *evsel, struct perf_sample *sample, static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_annotate *ann = container_of(tool, struct perf_annotate, tool); @@ -292,17 +288,19 @@ static int process_sample_event(const struct perf_tool *tool, addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_warning("problem processing %d event, skipping it.\n", - event->header.type); + pr_warning("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out_put; } - if (ann->cpu_list && !test_bit(sample->cpu, ann->cpu_bitmap)) + if (ann->cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, ann->cpu_bitmap))) goto out_put; if (!al.filtered && - evsel__add_sample(evsel, sample, &al, ann, machine)) { + add_sample(sample, &al, ann, machine)) { pr_warning("problem incrementing symbol count, " "skipping event\n"); ret = -1; @@ -563,6 +561,10 @@ static int __cmd_annotate(struct perf_annotate *ann) if (ret) goto out; + if ((use_browser == 1 || ann->use_stdio2) && ann->has_br_stack) + if (evlist__nr_br_cntr(session->evlist) > 0) + annotate_opts.show_br_cntr = true; + if (dump_trace) { perf_session__fprintf_nr_events(session, stdout); evlist__fprintf_nr_events(session->evlist, stdout); @@ -928,7 +930,7 @@ int cmd_annotate(int argc, const char **argv) */ if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) { sort__mode = SORT_MODE__BRANCH; - if (annotate.session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(annotate.session->evlist) > 0) annotate_opts.show_br_cntr = true; } diff --git a/tools/perf/builtin-buildid-list.c b/tools/perf/builtin-buildid-list.c index a91bbb34ac94..e0881b0ac38f 100644 --- a/tools/perf/builtin-buildid-list.c +++ b/tools/perf/builtin-buildid-list.c @@ -61,7 +61,7 @@ static int sysfs__fprintf_build_id(FILE *fp) int ret; ret = sysfs__snprintf_build_id("/", sbuild_id, sizeof(sbuild_id)); - if (ret != sizeof(sbuild_id)) + if (ret + 1 != sizeof(sbuild_id)) return ret < 0 ? ret : -EINVAL; return fprintf(fp, "%s\n", sbuild_id); @@ -73,7 +73,7 @@ static int filename__fprintf_build_id(const char *name, FILE *fp) int ret; ret = filename__snprintf_build_id(name, sbuild_id, sizeof(sbuild_id)); - if (ret != sizeof(sbuild_id)) + if (ret + 1 != sizeof(sbuild_id)) return ret < 0 ? ret : -EINVAL; return fprintf(fp, "%s\n", sbuild_id); diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index 72a7802775ee..c9584dbedf77 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -12,41 +12,45 @@ */ #include #include + +#include #include #include #include #include #include -#include #include -#include "debug.h" -#include "builtin.h" + +#include #include #include #include -#include "map_symbol.h" -#include "mem-events.h" -#include "session.h" -#include "hist.h" -#include "sort.h" -#include "tool.h" + +#include "builtin.h" #include "cacheline.h" #include "data.h" +#include "debug.h" #include "event.h" #include "evlist.h" #include "evsel.h" -#include "ui/browsers/hists.h" -#include "thread.h" -#include "mem2node.h" +#include "hist.h" +#include "map_symbol.h" +#include "mem-events.h" #include "mem-info.h" -#include "symbol.h" -#include "ui/ui.h" -#include "ui/progress.h" +#include "mem2node.h" #include "pmus.h" +#include "session.h" +#include "sort.h" #include "string2.h" -#include "util/util.h" -#include "util/symbol.h" +#include "symbol.h" +#include "thread.h" +#include "tool.h" +#include "ui/browsers/hists.h" +#include "ui/progress.h" +#include "ui/ui.h" #include "util/annotate.h" +#include "util/symbol.h" +#include "util/util.h" struct c2c_hists { struct hists hists; @@ -180,7 +184,8 @@ static void c2c_he_free(void *he) c2c_he = container_of(he, struct c2c_hist_entry, he); if (c2c_he->hists) { - hists__delete_entries(&c2c_he->hists->hists); + hists__delete_all_entries(&c2c_he->hists->hists); + perf_hpp__reset_output_field(&c2c_he->hists->list); zfree(&c2c_he->hists); } @@ -221,6 +226,8 @@ he__get_c2c_hists(struct hist_entry *he, ret = c2c_hists__init(hists, sort, nr_header_lines, env); if (ret) { + perf_hpp__reset_output_field(&hists->list); + c2c_he->hists = NULL; free(hists); return NULL; } @@ -241,6 +248,10 @@ static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he, "WARNING: no sample cpu value")) return; + /* cpuset bitmap has c2c.cpus_cnt bits from env->nr_cpus_avail */ + if (sample->cpu >= (unsigned int)c2c.cpus_cnt) + return; + __set_bit(sample->cpu, c2c_he->cpuset); } @@ -258,6 +269,10 @@ static void c2c_he__set_node(struct c2c_hist_entry *c2c_he, if (WARN_ONCE(node < 0, "WARNING: failed to find node\n")) return; + /* nodeset bitmap has c2c.nodes_cnt bits from env->nr_numa_nodes */ + if (node >= c2c.nodes_cnt) + return; + __set_bit(node, c2c_he->nodeset); if (c2c_he->paddr != sample->phys_addr) { @@ -314,9 +329,9 @@ static void perf_c2c__evsel_hists_inc_stats(struct evsel *evsel, static int process_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { + struct evsel *evsel = sample->evsel; struct c2c_hists *c2c_hists = &c2c.hists; struct c2c_hist_entry *c2c_he; struct c2c_stats stats = { .nr_entries = 0, }; @@ -328,8 +343,9 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_debug("problem processing %d event, skipping it.\n", - event->header.type); + pr_debug("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out; } @@ -339,7 +355,7 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, cursor = get_tls_callchain_cursor(); ret = sample__resolve_callchain(sample, cursor, NULL, - evsel, &al, sysctl_perf_event_max_stack); + &al, sysctl_perf_event_max_stack); if (ret) goto out; @@ -371,7 +387,7 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, if (perf_c2c__has_annotation(NULL)) { perf_c2c__evsel_hists_inc_stats(evsel, he, sample); - addr_map_symbol__inc_samples(mem_info__iaddr(mi), sample, evsel); + addr_map_symbol__inc_samples(mem_info__iaddr(mi), sample); } ret = hist_entry__append_callchain(he, sample); @@ -386,7 +402,12 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, * Doing node stats only for single callchain data. */ int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu; - int node = c2c.cpu2node[cpu]; + int node; + + /* cpu2node[] has c2c.cpus_cnt entries; large u32 wraps signed negative */ + if (cpu < 0 || cpu >= c2c.cpus_cnt) + cpu = 0; + node = c2c.cpu2node[cpu]; c2c_hists = he__get_c2c_hists(he, c2c.cl_sort, 2, machine->env); if (!c2c_hists) { @@ -405,7 +426,9 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, c2c_he = container_of(he, struct c2c_hist_entry, he); c2c_add_stats(&c2c_he->stats, &stats); c2c_add_stats(&c2c_hists->stats, &stats); - c2c_add_stats(&c2c_he->node_stats[node], &stats); + /* node_stats[] has c2c.nodes_cnt entries */ + if (node >= 0 && node < c2c.nodes_cnt) + c2c_add_stats(&c2c_he->node_stats[node], &stats); compute_stats(c2c_he, &stats, sample->weight); @@ -2351,6 +2374,10 @@ static int setup_nodes(struct perf_session *session) nodes[node] = set; perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) { + /* topology CPU IDs from perf.data may exceed nr_cpus_avail */ + if (cpu.cpu < 0 || cpu.cpu >= c2c.cpus_cnt) + continue; + __set_bit(cpu.cpu, set); if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug")) @@ -3202,7 +3229,7 @@ static int perf_c2c__report(int argc, const char **argv) * default display type. */ if (!display) { - if (!strcmp(perf_env__arch(env), "arm64")) + if (perf_env__e_machine(env, /*e_flags=*/NULL) == EM_AARCH64) display = "peer"; else display = "tot"; diff --git a/tools/perf/builtin-check.c b/tools/perf/builtin-check.c index 3641d263b345..60437650c50f 100644 --- a/tools/perf/builtin-check.c +++ b/tools/perf/builtin-check.c @@ -43,7 +43,7 @@ struct feature_status supported_features[] = { FEATURE_STATUS("dwarf_getlocations", HAVE_LIBDW_SUPPORT), FEATURE_STATUS("dwarf-unwind", HAVE_DWARF_UNWIND_SUPPORT), FEATURE_STATUS_TIP("libbfd", HAVE_LIBBFD_SUPPORT, "Deprecated, license incompatibility, use BUILD_NONDISTRO=1 and install binutils-dev[el]"), - FEATURE_STATUS("libbabeltrace", HAVE_LIBBABELTRACE_SUPPORT), + FEATURE_STATUS("babeltrace2-ctf-writer", HAVE_BABELTRACE2_CTF_WRITER_SUPPORT), FEATURE_STATUS("libbpf-strings", HAVE_LIBBPF_STRINGS_SUPPORT), FEATURE_STATUS("libcapstone", HAVE_LIBCAPSTONE_SUPPORT), FEATURE_STATUS("libdw-dwarf-unwind", HAVE_LIBDW_SUPPORT), diff --git a/tools/perf/builtin-data.c b/tools/perf/builtin-data.c index 4c08ccb8c06b..1dd73ed6bdcb 100644 --- a/tools/perf/builtin-data.c +++ b/tools/perf/builtin-data.c @@ -40,10 +40,8 @@ static const struct option data_options[] = { OPT_INCR('v', "verbose", &verbose, "be more verbose"), OPT_STRING('i', "input", &input_name, "file", "input file name"), OPT_STRING(0, "to-json", &to_json, NULL, "Convert to JSON format"), -#ifdef HAVE_LIBBABELTRACE_SUPPORT OPT_STRING(0, "to-ctf", &to_ctf, NULL, "Convert to CTF format"), OPT_BOOLEAN(0, "tod", &opts.tod, "Convert time to wall clock time"), -#endif OPT_BOOLEAN('f', "force", &opts.force, "don't complain, do it"), OPT_BOOLEAN(0, "all", &opts.all, "Convert all events"), OPT_STRING(0, "time", &opts.time_str, "str", @@ -65,29 +63,21 @@ static int cmd_data_convert(int argc, const char **argv) pr_err("You cannot specify both --to-ctf and --to-json.\n"); return -1; } -#ifdef HAVE_LIBBABELTRACE_SUPPORT if (!to_json && !to_ctf) { pr_err("You must specify one of --to-ctf or --to-json.\n"); return -1; } -#else - if (!to_json) { - pr_err("You must specify --to-json.\n"); - return -1; -} -#endif if (to_json) return bt_convert__perf2json(input_name, to_json, &opts); if (to_ctf) { -#if defined(HAVE_LIBBABELTRACE_SUPPORT) && defined(HAVE_LIBTRACEEVENT) +#if defined(HAVE_BABELTRACE2_CTF_WRITER_SUPPORT) && defined(HAVE_LIBTRACEEVENT) return bt_convert__perf2ctf(input_name, to_ctf, &opts); #else - pr_err("The libbabeltrace support is not compiled in. perf should be " - "compiled with environment variables LIBBABELTRACE=1 and " - "LIBBABELTRACE_DIR=/path/to/libbabeltrace/.\n" - "Check also if libbtraceevent devel files are available.\n"); + pr_err("The babeltrace2 ctf support is not compiled in. Ensure you have both\n" + "libbabeltrace2-dev[el] and libtraceevent-dev[el] installed or set\n" + "PKG_CONFIG_PATH to find a local installation of those libraries.\n"); return -1; #endif } diff --git a/tools/perf/builtin-diff.c b/tools/perf/builtin-diff.c index 1b3df868849a..9fa8e900637b 100644 --- a/tools/perf/builtin-diff.c +++ b/tools/perf/builtin-diff.c @@ -390,14 +390,13 @@ static struct hist_entry_ops block_hist_ops = { static int diff__process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_diff *pdiff = container_of(tool, struct perf_diff, tool); struct addr_location al; + struct evsel *evsel = sample->evsel; struct hists *hists = evsel__hists(evsel); struct hist_entry_iter iter = { - .evsel = evsel, .sample = sample, .ops = &hist_iter_normal, }; @@ -410,13 +409,15 @@ static int diff__process_sample_event(const struct perf_tool *tool, addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_warning("problem processing %d event, skipping it.\n", - event->header.type); + pr_warning("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out; } - if (cpu_list && !test_bit(sample->cpu, cpu_bitmap)) { + if (cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, cpu_bitmap))) { ret = 0; goto out; } @@ -431,13 +432,14 @@ static int diff__process_sample_event(const struct perf_tool *tool, } hist__account_cycles(sample->branch_stack, &al, sample, - false, NULL, evsel); + /*nonany_branch_mode=*/false, /*total_cycles=*/NULL); break; case COMPUTE_STREAM: if (hist_entry_iter__add(&iter, &al, PERF_MAX_STACK_DEPTH, NULL)) { - pr_debug("problem adding hist entry, skipping event\n"); + pr_debug("problem adding hist entry at offset %#" PRIx64 ", skipping event\n", + sample->file_offset); goto out; } break; diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 8a7dbfb14535..9e4c5220d43c 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -377,9 +377,9 @@ static int set_tracing_pid(struct perf_ftrace *ftrace) if (target__has_cpu(&ftrace->target)) return 0; - for (i = 0; i < perf_thread_map__nr(ftrace->evlist->core.threads); i++) { + for (i = 0; i < perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); i++) { scnprintf(buf, sizeof(buf), "%d", - perf_thread_map__pid(ftrace->evlist->core.threads, i)); + perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i)); if (append_tracing_file("set_ftrace_pid", buf) < 0) return -1; } @@ -413,7 +413,7 @@ static int set_tracing_cpumask(struct perf_cpu_map *cpumap) static int set_tracing_cpu(struct perf_ftrace *ftrace) { - struct perf_cpu_map *cpumap = ftrace->evlist->core.user_requested_cpus; + struct perf_cpu_map *cpumap = evlist__core(ftrace->evlist)->user_requested_cpus; if (!target__has_cpu(&ftrace->target)) return 0; @@ -1999,20 +1999,20 @@ int cmd_ftrace(int argc, const char **argv) ret = evlist__create_maps(ftrace.evlist, &ftrace.target); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; if (argc) { ret = evlist__prepare_workload(ftrace.evlist, &ftrace.target, argv, false, ftrace__workload_exec_failed_signal); if (ret < 0) - goto out_delete_evlist; + goto out_put_evlist; } ret = cmd_func(&ftrace); -out_delete_evlist: - evlist__delete(ftrace.evlist); +out_put_evlist: + evlist__put(ftrace.evlist); out_delete_filters: delete_filter_func(&ftrace.filters); diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c index f174bc69cec4..70bbfad5653e 100644 --- a/tools/perf/builtin-inject.c +++ b/tools/perf/builtin-inject.c @@ -8,6 +8,7 @@ */ #include "builtin.h" +#include "util/aslr.h" #include "util/color.h" #include "util/dso.h" #include "util/vdso.h" @@ -24,8 +25,10 @@ #include "util/string2.h" #include "util/symbol.h" #include "util/synthetic-events.h" +#include "util/pmus.h" #include "util/thread.h" #include "util/namespaces.h" +#include "util/unwind.h" #include "util/util.h" #include "util/tsc.h" @@ -123,6 +126,7 @@ struct perf_inject { bool in_place_update_dry_run; bool copy_kcore_dir; bool convert_callchain; + bool aslr; const char *input_name; struct perf_data output; u64 bytes_written; @@ -146,14 +150,12 @@ struct event_entry { static int tool__inject_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, - const struct evsel *evsel, __u16 misc, const char *filename, struct dso *dso, u32 flags); static int tool__inject_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -215,23 +217,96 @@ static int perf_event__repipe_op4_synth(const struct perf_tool *tool, return perf_event__repipe_synth(tool, event); } +static int perf_event__repipe_synth_cb(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + return perf_event__repipe_synth(tool, event); +} + static int perf_event__repipe_attr(const struct perf_tool *tool, union perf_event *event, struct evlist **pevlist) { struct perf_inject *inject = container_of(tool, struct perf_inject, tool); + struct perf_event_attr attr; + u32 raw_attr_size, attr_size; + size_t n_ids; + u64 *ids; int ret; + union perf_event *aslr_event = NULL; + ret = perf_event__process_attr(tool, event, pevlist); if (ret) return ret; + if (inject->aslr) { + aslr_event = malloc(event->header.size); + if (!aslr_event) + return -ENOMEM; + memcpy(aslr_event, event, event->header.size); + aslr_tool__strip_attr_event(aslr_event, *pevlist); + event = aslr_event; + } + /* If the output isn't a pipe then the attributes will be written as part of the header. */ - if (!inject->output.is_pipe) - return 0; + if (!inject->output.is_pipe) { + ret = 0; + goto out; + } - return perf_event__repipe_synth(tool, event); + if (!inject->itrace_synth_opts.set) { + ret = perf_event__repipe_synth(tool, event); + goto out; + } + + if (event->header.size < sizeof(struct perf_event_header) + PERF_ATTR_SIZE_VER0) { + pr_err("Attribute event size %u is too small\n", event->header.size); + ret = -EINVAL; + goto out; + } + + /* + * ABI0 pipe/inject events have attr.size == 0; default to + * PERF_ATTR_SIZE_VER0 (the ABI0 footprint) for the bounded + * copy and ID array position. Same pattern as + * perf_event__process_attr() in header.c. + */ + raw_attr_size = event->attr.attr.size; + attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; + + if (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || + raw_attr_size > event->header.size - sizeof(event->header))) { + pr_err("Attribute event size %u is too small for attr.size %u\n", + event->header.size, raw_attr_size); + ret = -EINVAL; + goto out; + } + + memset(&attr, 0, sizeof(attr)); + memcpy(&attr, &event->attr.attr, + min_t(size_t, sizeof(attr), attr_size)); + + n_ids = event->header.size - sizeof(event->header) - attr_size; + n_ids /= sizeof(u64); + ids = (void *)&event->attr.attr + attr_size; + + attr.size = sizeof(struct perf_event_attr); + attr.sample_type &= ~PERF_SAMPLE_AUX; + + + if (inject->itrace_synth_opts.add_last_branch) { + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + ret = perf_event__synthesize_attr(tool, &attr, (u32)n_ids, ids, + perf_event__repipe_synth_cb); +out: + free(aslr_event); + return ret; } static int perf_event__repipe_event_update(const struct perf_tool *tool, @@ -330,8 +405,8 @@ perf_inject__cut_auxtrace_sample(struct perf_inject *inject, union perf_event *event, struct perf_sample *sample) { - size_t sz1 = sample->aux_sample.data - (void *)event; - size_t sz2 = event->header.size - sample->aux_sample.size - sz1; + size_t sz1 = sample->aux_sample.data - (void *)event - sizeof(u64); + size_t sz2 = event->header.size - sample->aux_sample.size - (sz1 + sizeof(u64)); union perf_event *ev; if (inject->event_copy == NULL) { @@ -342,13 +417,12 @@ perf_inject__cut_auxtrace_sample(struct perf_inject *inject, ev = (union perf_event *)inject->event_copy; if (sz1 > event->header.size || sz2 > event->header.size || sz1 + sz2 > event->header.size || - sz1 < sizeof(struct perf_event_header) + sizeof(u64)) + sz1 < sizeof(struct perf_event_header)) return event; memcpy(ev, event, sz1); memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2); ev->header.size = sz1 + sz2; - ((u64 *)((void *)ev + sz1))[-1] = 0; return ev; } @@ -356,26 +430,87 @@ perf_inject__cut_auxtrace_sample(struct perf_inject *inject, typedef int (*inject_handler)(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine); static int perf_event__repipe_sample(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { - struct perf_inject *inject = container_of(tool, struct perf_inject, - tool); + struct perf_inject *inject = container_of(tool, struct perf_inject, tool); + struct evsel *evsel = sample->evsel; + + if (evsel == NULL) + return perf_event__repipe_synth(tool, event); - if (evsel && evsel->handler) { + if (evsel->handler) { inject_handler f = evsel->handler; - return f(tool, event, sample, evsel, machine); + return f(tool, event, sample, machine); } - build_id__mark_dso_hit(tool, event, sample, evsel, machine); + build_id__mark_dso_hit(tool, event, sample, machine); + + if (inject->itrace_synth_opts.set && + (inject->itrace_synth_opts.last_branch || + inject->itrace_synth_opts.add_last_branch)) { + union perf_event *event_copy = (void *)inject->event_copy; + struct branch_stack dummy_bs = { .nr = 0, .hw_idx = 0 }; + int err; + size_t sz; + u64 orig_type = evsel->core.attr.sample_type; + u64 orig_branch_type = evsel->core.attr.branch_sample_type; + + struct branch_stack *orig_bs = sample->branch_stack; - if (inject->itrace_synth_opts.set && sample->aux_sample.size) { + if (event_copy == NULL) { + inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE); + if (!inject->event_copy) + return -ENOMEM; + + event_copy = (void *)inject->event_copy; + } + + if (!sample->branch_stack) + sample->branch_stack = &dummy_bs; + + if (inject->itrace_synth_opts.add_last_branch) { + /* Temporarily add in type bits for synthesis. */ + evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; + + sz = perf_event__sample_event_size(sample, evsel->core.attr.sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type); + + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + evsel->core.attr.sample_type = orig_type; + evsel->core.attr.branch_sample_type = orig_branch_type; + sample->branch_stack = orig_bs; + return -EFAULT; + } + + event_copy->header.type = PERF_RECORD_SAMPLE; + event_copy->header.misc = event->header.misc; + event_copy->header.size = sz; + + err = perf_event__synthesize_sample(event_copy, evsel->core.attr.sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, sample); + + evsel->core.attr.sample_type = orig_type; + evsel->core.attr.branch_sample_type = orig_branch_type; + sample->branch_stack = orig_bs; + + if (err) { + pr_err("Failed to synthesize sample\n"); + return err; + } + event = event_copy; + } else if (inject->itrace_synth_opts.set && + (evsel->core.attr.sample_type & PERF_SAMPLE_AUX)) { event = perf_inject__cut_auxtrace_sample(inject, event, sample); if (IS_ERR(event)) return PTR_ERR(event); @@ -387,16 +522,16 @@ static int perf_event__repipe_sample(const struct perf_tool *tool, static int perf_event__convert_sample_callchain(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_inject *inject = container_of(tool, struct perf_inject, tool); + struct evsel *evsel = sample->evsel; struct callchain_cursor *cursor = get_tls_callchain_cursor(); union perf_event *event_copy = (void *)inject->event_copy; struct callchain_cursor_node *node; struct thread *thread; u64 sample_type = evsel->core.attr.sample_type; - u32 sample_size = event->header.size; + size_t sz; u64 i, k; int ret; @@ -418,7 +553,7 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, goto out; /* this will parse DWARF using stack and register data */ - ret = thread__resolve_callchain(thread, cursor, evsel, sample, + ret = thread__resolve_callchain(thread, cursor, sample, /*parent=*/NULL, /*root_al=*/NULL, PERF_MAX_STACK_DEPTH); thread__put(thread); @@ -438,12 +573,11 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, node = cursor->first; for (k = 0; k < cursor->nr && i < PERF_MAX_STACK_DEPTH; k++) { - if (machine__kernel_ip(machine, node->ip)) - /* kernel IPs were added already */; - else if (node->ms.sym && node->ms.sym->inlined) - /* we can't handle inlined callchains */; - else + if (!(machine->single_address_space && + machine__kernel_ip(machine, node->ip)) && + !(node->ms.sym && symbol__inlined(node->ms.sym))) { inject->raw_callchain->ips[i++] = node->ip; + } node = node->next; } @@ -454,17 +588,25 @@ static int perf_event__convert_sample_callchain(const struct perf_tool *tool, out: memcpy(event_copy, event, sizeof(event->header)); - /* adjust sample size for stack and regs */ - sample_size -= sample->user_stack.size; - sample_size -= (hweight64(evsel->core.attr.sample_regs_user) + 1) * sizeof(u64); - sample_size += (sample->callchain->nr + 1) * sizeof(u64); - event_copy->header.size = sample_size; - /* remove sample_type {STACK,REGS}_USER for synthesize */ sample_type &= ~(PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER); - perf_event__synthesize_sample(event_copy, sample_type, - evsel->core.attr.read_format, sample); + sz = perf_event__sample_event_size(sample, sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event_copy->header.size = sz; + + ret = perf_event__synthesize_sample(event_copy, sample_type, + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, sample); + if (ret) { + pr_err("Failed to synthesize sample\n"); + return ret; + } return perf_event__repipe_synth(tool, event_copy); } @@ -584,11 +726,15 @@ static int perf_event__repipe_common_mmap(const struct perf_tool *tool, } if (dso && !dso__hit(dso)) { - struct evsel *evsel = evlist__event2evsel(inject->session->evlist, event); + if (!sample->evsel) { + sample->evsel = evlist__event2evsel(inject->session->evlist, event); + if (sample->evsel) + evsel__get(sample->evsel); + } - if (evsel) { + if (sample->evsel) { dso__set_hit(dso); - tool__inject_build_id(tool, sample, machine, evsel, + tool__inject_build_id(tool, sample, machine, /*misc=*/sample->cpumode, filename, dso, flags); } @@ -615,23 +761,26 @@ static int perf_event__repipe_common_mmap(const struct perf_tool *tool, } if ((inject->build_id_style == BID_RWS__MMAP2_BUILDID_ALL) && !(event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)) { - struct evsel *evsel = evlist__event2evsel(inject->session->evlist, event); + struct evsel *saved_evsel = sample->evsel; - if (evsel && !dso_sought) { + sample->evsel = evlist__event2evsel(inject->session->evlist, event); + if (sample->evsel && !dso_sought) { dso = findnew_dso(pid, tid, filename, dso_id, machine); dso_sought = true; } - if (evsel && dso && - !tool__inject_mmap2_build_id(tool, sample, machine, evsel, + if (sample->evsel && dso && + !tool__inject_mmap2_build_id(tool, sample, machine, sample->cpumode | PERF_RECORD_MISC_MMAP_BUILD_ID, pid, tid, start, len, pgoff, dso, prot, flags, filename)) { /* Injected mmap2 so no need to repipe. */ + sample->evsel = saved_evsel; dso__put(dso); return 0; } + sample->evsel = saved_evsel; } dso__put(dso); if (inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) @@ -836,7 +985,6 @@ static bool perf_inject__lookup_known_build_id(struct perf_inject *inject, static int tool__inject_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, - const struct evsel *evsel, __u16 misc, const char *filename, struct dso *dso, u32 flags) @@ -860,7 +1008,7 @@ static int tool__inject_build_id(const struct perf_tool *tool, err = perf_event__synthesize_build_id(tool, sample, machine, perf_event__repipe, - evsel, misc, dso__bid(dso), + misc, dso__bid(dso), filename); if (err) { pr_err("Can't synthesize build_id event for %s\n", filename); @@ -873,7 +1021,6 @@ static int tool__inject_build_id(const struct perf_tool *tool, static int tool__inject_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -896,7 +1043,6 @@ static int tool__inject_mmap2_build_id(const struct perf_tool *tool, err = perf_event__synthesize_mmap2_build_id(tool, sample, machine, perf_event__repipe, - evsel, misc, pid, tid, start, len, pgoff, dso__bid(dso), @@ -913,7 +1059,7 @@ static int mark_dso_hit(const struct perf_inject *inject, const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, - const struct evsel *mmap_evsel, + struct evsel *mmap_evsel, struct map *map, bool sample_in_dso) { struct dso *dso; @@ -941,9 +1087,13 @@ static int mark_dso_hit(const struct perf_inject *inject, dso = map__dso(map); if (inject->build_id_style == BID_RWS__INJECT_HEADER_LAZY) { if (dso && !dso__hit(dso)) { + /* + * The sample is just read for identifiers which we want + * to match the for the event of the sample. + */ dso__set_hit(dso); tool__inject_build_id(tool, sample, machine, - mmap_evsel, misc, dso__long_name(dso), dso, + misc, dso__long_name(dso), dso, map__flags(map)); } } else if (inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) { @@ -951,11 +1101,13 @@ static int mark_dso_hit(const struct perf_inject *inject, const struct build_id null_bid = { .size = 0 }; const struct build_id *bid = dso ? dso__bid(dso) : &null_bid; const char *filename = dso ? dso__long_name(dso) : ""; + struct evsel *saved_evsel = sample->evsel; map__set_hit(map); + /* Creating a new mmap2 event which has an evsel for the mmap event. */ + sample->evsel = mmap_evsel; perf_event__synthesize_mmap2_build_id(tool, sample, machine, perf_event__repipe, - mmap_evsel, misc, sample->pid, sample->tid, map__start(map), @@ -965,6 +1117,7 @@ static int mark_dso_hit(const struct perf_inject *inject, map__prot(map), map__flags(map), filename); + sample->evsel = saved_evsel; } } return 0; @@ -975,7 +1128,7 @@ struct mark_dso_hit_args { const struct perf_tool *tool; struct perf_sample *sample; struct machine *machine; - const struct evsel *mmap_evsel; + struct evsel *mmap_evsel; }; static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data) @@ -987,10 +1140,8 @@ static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data) args->mmap_evsel, map, /*sample_in_dso=*/false); } -int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *event, - struct perf_sample *sample, - struct evsel *evsel __maybe_unused, - struct machine *machine) +static int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *event, + struct perf_sample *sample, struct machine *machine) { struct addr_location al; struct thread *thread; @@ -1020,7 +1171,7 @@ int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *e /*sample_in_dso=*/true); } - sample__for_each_callchain_node(thread, evsel, sample, PERF_MAX_STACK_DEPTH, + sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, /*symbols=*/false, mark_dso_hit_callback, &args); thread__put(thread); repipe: @@ -1032,7 +1183,6 @@ int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *e static int perf_inject__sched_process_exit(const struct perf_tool *tool, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel __maybe_unused, struct machine *machine __maybe_unused) { struct perf_inject *inject = container_of(tool, struct perf_inject, tool); @@ -1052,13 +1202,12 @@ static int perf_inject__sched_process_exit(const struct perf_tool *tool, static int perf_inject__sched_switch(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_inject *inject = container_of(tool, struct perf_inject, tool); struct event_entry *ent; - perf_inject__sched_process_exit(tool, event, sample, evsel, machine); + perf_inject__sched_process_exit(tool, event, sample, machine); ent = malloc(event->header.size + sizeof(struct event_entry)); if (ent == NULL) { @@ -1077,14 +1226,14 @@ static int perf_inject__sched_switch(const struct perf_tool *tool, static int perf_inject__sched_stat(const struct perf_tool *tool, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct event_entry *ent; union perf_event *event_sw; struct perf_sample sample_sw; struct perf_inject *inject = container_of(tool, struct perf_inject, tool); - u32 pid = evsel__intval(evsel, sample, "pid"); + struct evsel *evsel = sample->evsel; + u32 pid = perf_sample__intval(sample, "pid"); int ret; list_for_each_entry(ent, &inject->samples, node) { @@ -1100,8 +1249,9 @@ static int perf_inject__sched_stat(const struct perf_tool *tool, sample_sw.period = sample->period; sample_sw.time = sample->time; perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type, - evsel->core.attr.read_format, &sample_sw); - build_id__mark_dso_hit(tool, event_sw, &sample_sw, evsel, machine); + evsel->core.attr.read_format, + evsel->core.attr.branch_sample_type, &sample_sw); + build_id__mark_dso_hit(tool, event_sw, &sample_sw, machine); ret = perf_event__repipe(tool, event_sw, &sample_sw, machine); perf_sample__exit(&sample_sw); return ret; @@ -1373,7 +1523,7 @@ static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt) struct perf_session *session = inject->session; struct evlist *evlist = session->evlist; struct machine *machine = &session->machines.host; - size_t from = evlist->core.nr_entries - new_cnt; + size_t from = evlist__nr_entries(evlist) - new_cnt; return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe, evlist, machine, from); @@ -1448,7 +1598,7 @@ static int synthesize_build_id(struct perf_inject *inject, struct dso *dso, pid_ dso__set_hit(dso); return perf_event__synthesize_build_id(&inject->tool, &synth_sample, machine, - process_build_id, inject__mmap_evsel(inject), + process_build_id, /*misc=*/synth_sample.cpumode, dso__bid(dso), dso__long_name(dso)); } @@ -1908,7 +2058,7 @@ static int host__finished_init(const struct perf_tool *tool, struct perf_session if (ret) return ret; - ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries); + ret = synthesize_id_index(inject, evlist__nr_entries(gs->session->evlist)); if (ret) { pr_err("Failed to synthesize id_index\n"); return ret; @@ -2010,7 +2160,6 @@ static int evsel__check_stype(struct evsel *evsel, u64 sample_type, const char * static int drop_sample(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct machine *machine __maybe_unused) { return 0; @@ -2434,12 +2583,27 @@ static int __cmd_inject(struct perf_inject *inject) * synthesized hardware events, so clear the feature flag. */ if (inject->itrace_synth_opts.set) { + struct evsel *evsel; + perf_header__clear_feat(&session->header, HEADER_AUXTRACE); - if (inject->itrace_synth_opts.last_branch || - inject->itrace_synth_opts.add_last_branch) + + evlist__for_each_entry(session->evlist, evsel) { + evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; + } + + if (inject->itrace_synth_opts.add_last_branch) { perf_header__set_feat(&session->header, HEADER_BRANCH_STACK); + + evlist__for_each_entry(session->evlist, evsel) { + evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + if (evsel->core.attr.size < PERF_ATTR_SIZE_VER2) + evsel->core.attr.size = PERF_ATTR_SIZE_VER2; + evsel->core.attr.branch_sample_type |= + PERF_SAMPLE_BRANCH_HW_INDEX; + } + } } /* @@ -2458,6 +2622,9 @@ static int __cmd_inject(struct perf_inject *inject) } } + if (inject->aslr) + aslr_tool__strip_evlist(inject->session->tool, session->evlist); + session->header.data_offset = output_data_offset; session->header.data_size = inject->bytes_written; perf_session__inject_header(session, session->evlist, fd, &inj_fc.fc, @@ -2562,8 +2729,13 @@ int cmd_inject(int argc, const char **argv) OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory", "guest mount directory under which every guest os" " instance has a subdir"), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_BOOLEAN(0, "convert-callchain", &inject.convert_callchain, "Generate callchains using DWARF and drop register/stack data"), + OPT_BOOLEAN(0, "aslr", &inject.aslr, + "Remap virtual memory addresses similar to ASLR"), OPT_END() }; const char * const inject_usage[] = { @@ -2571,6 +2743,7 @@ int cmd_inject(int argc, const char **argv) NULL }; bool ordered_events; + struct perf_tool *tool = &inject.tool; if (!inject.itrace_synth_opts.set) { /* Disable eager loading of kernel symbols that adds overhead to perf inject. */ @@ -2591,6 +2764,11 @@ int cmd_inject(int argc, const char **argv) if (argc) usage_with_options(inject_usage, options); + if (inject.aslr && inject.convert_callchain) { + pr_err("Error: --aslr and --convert-callchain are mutually exclusive features.\n"); + return -EINVAL; + } + if (inject.strip && !inject.itrace_synth_opts.set) { pr_err("--strip option requires --itrace option\n"); return -1; @@ -2684,18 +2862,39 @@ int cmd_inject(int argc, const char **argv) inject.tool.schedstat_domain = perf_event__repipe_op2_synth; inject.tool.dont_split_sample_group = true; inject.tool.merge_deferred_callchains = false; - inject.session = __perf_session__new(&data, &inject.tool, + if (inject.aslr) { + tool = aslr_tool__new(&inject.tool); + if (!tool) { + ret = -ENOMEM; + goto out_close_output; + } + } + inject.session = __perf_session__new(&data, tool, /*trace_event_repipe=*/inject.output.is_pipe, /*host_env=*/NULL); if (IS_ERR(inject.session)) { ret = PTR_ERR(inject.session); + if (inject.aslr) + aslr_tool__delete(tool); goto out_close_output; } if (zstd_init(&(inject.session->zstd_data), 0) < 0) pr_warning("Decompression initialization failed.\n"); + if (inject.aslr) { + struct evsel *evsel; + + evlist__for_each_entry(inject.session->evlist, evsel) { + ret = aslr_tool__cache_orig_attrs(tool, evsel); + if (ret) { + pr_err("Failed to cache original attributes: %d\n", ret); + goto out_delete; + } + } + } + /* Save original section info before feature bits change */ ret = save_section_info(&inject); if (ret) @@ -2714,10 +2913,17 @@ int cmd_inject(int argc, const char **argv) * the input. */ if (!data.is_pipe) { + if (inject.aslr) + aslr_tool__strip_evlist(tool, inject.session->evlist); + ret = perf_event__synthesize_for_pipe(&inject.tool, inject.session, &inject.output, perf_event__repipe); + + if (inject.aslr) + aslr_tool__restore_evlist(tool, inject.session->evlist); + if (ret < 0) goto out_delete; } @@ -2789,6 +2995,8 @@ int cmd_inject(int argc, const char **argv) strlist__delete(inject.known_build_ids); zstd_fini(&(inject.session->zstd_data)); perf_session__delete(inject.session); + if (inject.aslr) + aslr_tool__delete(tool); out_close_output: if (!inject.in_place_update) perf_data__close(&inject.output); diff --git a/tools/perf/builtin-kmem.c b/tools/perf/builtin-kmem.c index 9c64a0d74823..e1b2f5bc1ba8 100644 --- a/tools/perf/builtin-kmem.c +++ b/tools/perf/builtin-kmem.c @@ -22,6 +22,7 @@ #include "util/cpumap.h" #include "util/debug.h" +#include "util/event.h" #include "util/string2.h" #include "util/util.h" @@ -171,12 +172,12 @@ static int insert_caller_stat(unsigned long call_site, return 0; } -static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_alloc_event(struct perf_sample *sample) { - unsigned long ptr = evsel__intval(evsel, sample, "ptr"), - call_site = evsel__intval(evsel, sample, "call_site"); - int bytes_req = evsel__intval(evsel, sample, "bytes_req"), - bytes_alloc = evsel__intval(evsel, sample, "bytes_alloc"); + unsigned long ptr = perf_sample__intval(sample, "ptr"), + call_site = perf_sample__intval(sample, "call_site"); + int bytes_req = perf_sample__intval(sample, "bytes_req"), + bytes_alloc = perf_sample__intval(sample, "bytes_alloc"); if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) || insert_caller_stat(call_site, bytes_req, bytes_alloc)) @@ -198,11 +199,11 @@ static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *s * If the tracepoint contains the field "node" the tool stats the * cross allocation. */ - if (evsel__field(evsel, "node")) { + if (evsel__field(sample->evsel, "node")) { int node1, node2; node1 = cpu__get_node((struct perf_cpu){.cpu = sample->cpu}); - node2 = evsel__intval(evsel, sample, "node"); + node2 = perf_sample__intval(sample, "node"); /* * If the field "node" is NUMA_NO_NODE (-1), we don't take it @@ -243,9 +244,9 @@ static struct alloc_stat *search_alloc_stat(unsigned long ptr, return NULL; } -static int evsel__process_free_event(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_free_event(struct perf_sample *sample) { - unsigned long ptr = evsel__intval(evsel, sample, "ptr"); + unsigned long ptr = perf_sample__intval(sample, "ptr"); struct alloc_stat *s_alloc, *s_caller; s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp); @@ -394,7 +395,7 @@ static int build_alloc_func_list(void) * Find first non-memory allocation function from callchain. * The allocation functions are in the 'alloc_func_list'. */ -static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample) +static u64 find_callsite(struct perf_sample *sample) { struct addr_location al; struct machine *machine = &kmem_session->machines.host; @@ -414,7 +415,7 @@ static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample) if (cursor == NULL) goto out; - sample__resolve_callchain(sample, cursor, NULL, evsel, &al, 16); + sample__resolve_callchain(sample, cursor, /*parent=*/NULL, &al, 16); callchain_cursor_commit(cursor); while (true) { @@ -751,8 +752,7 @@ static char *compact_gfp_string(unsigned long gfp_flags) return NULL; } -static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample, - unsigned int gfp_flags) +static int parse_gfp_flags(struct perf_sample *sample, unsigned int gfp_flags) { struct tep_record record = { .cpu = sample->cpu, @@ -773,7 +773,7 @@ static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample, } trace_seq_init(&seq); - tp_format = evsel__tp_format(evsel); + tp_format = evsel__tp_format(sample->evsel); if (tp_format) tep_print_event(tp_format->tep, &seq, &record, "%s", TEP_PRINT_INFO); @@ -784,17 +784,21 @@ static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample, new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps)); if (new == NULL) - return -ENOMEM; + goto err_out; gfps = new; - new += nr_gfps++; + new += nr_gfps; new->flags = gfp_flags; new->human_readable = strdup(str + 10); + if (!new->human_readable) + goto err_out; new->compact_str = compact_gfp_flags(str + 10); - if (!new->human_readable || !new->compact_str) - return -ENOMEM; - + if (!new->compact_str) { + free(new->human_readable); + goto err_out; + } + nr_gfps++; qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp); } @@ -803,15 +807,17 @@ static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample, trace_seq_destroy(&seq); return 0; +err_out: + trace_seq_destroy(&seq); + return -ENOMEM; } -static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_page_alloc_event(struct perf_sample *sample) { u64 page; - unsigned int order = evsel__intval(evsel, sample, "order"); - unsigned int gfp_flags = evsel__intval(evsel, sample, "gfp_flags"); - unsigned int migrate_type = evsel__intval(evsel, sample, - "migratetype"); + unsigned int order = perf_sample__intval(sample, "order"); + unsigned int gfp_flags = perf_sample__intval(sample, "gfp_flags"); + unsigned int migrate_type = perf_sample__intval(sample, "migratetype"); u64 bytes = kmem_page_size << order; u64 callsite; struct page_stat *pstat; @@ -821,10 +827,20 @@ static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_samp .migrate_type = migrate_type, }; + if (order >= MAX_PAGE_ORDER) { + pr_debug("Out-of-bounds order %u\n", order); + return -1; + } + + if (migrate_type >= MAX_MIGRATE_TYPES) { + pr_debug("Out-of-bounds migratetype %u\n", migrate_type); + return -1; + } + if (use_pfn) - page = evsel__intval(evsel, sample, "pfn"); + page = perf_sample__intval(sample, "pfn"); else - page = evsel__intval(evsel, sample, "page"); + page = perf_sample__intval(sample, "page"); nr_page_allocs++; total_page_alloc_bytes += bytes; @@ -836,10 +852,10 @@ static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_samp return 0; } - if (parse_gfp_flags(evsel, sample, gfp_flags) < 0) + if (parse_gfp_flags(sample, gfp_flags) < 0) return -1; - callsite = find_callsite(evsel, sample); + callsite = find_callsite(sample); /* * This is to find the current page (with correct gfp flags and @@ -877,20 +893,25 @@ static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_samp return 0; } -static int evsel__process_page_free_event(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_page_free_event(struct perf_sample *sample) { u64 page; - unsigned int order = evsel__intval(evsel, sample, "order"); + unsigned int order = perf_sample__intval(sample, "order"); u64 bytes = kmem_page_size << order; struct page_stat *pstat; struct page_stat this = { .order = order, }; + if (order >= MAX_PAGE_ORDER) { + pr_debug("Out-of-bounds order %u\n", order); + return -1; + } + if (use_pfn) - page = evsel__intval(evsel, sample, "pfn"); + page = perf_sample__intval(sample, "pfn"); else - page = evsel__intval(evsel, sample, "page"); + page = perf_sample__intval(sample, "page"); nr_page_frees++; total_page_free_bytes += bytes; @@ -954,33 +975,35 @@ static bool perf_kmem__skip_sample(struct perf_sample *sample) return false; } -typedef int (*tracepoint_handler)(struct evsel *evsel, - struct perf_sample *sample); +typedef int (*tracepoint_handler)(struct perf_sample *sample); static int process_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { + struct evsel *evsel = sample->evsel; int err = 0; struct thread *thread = machine__findnew_thread(machine, sample->pid, sample->tid); if (thread == NULL) { - pr_debug("problem processing %d event, skipping it.\n", - event->header.type); + pr_debug("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); return -1; } - if (perf_kmem__skip_sample(sample)) + if (perf_kmem__skip_sample(sample)) { + thread__put(thread); return 0; + } dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread__tid(thread)); if (evsel->handler != NULL) { tracepoint_handler f = evsel->handler; - err = f(evsel, sample); + err = f(sample); } thread__put(thread); diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c index 0c5e6b3aac74..00baafb05603 100644 --- a/tools/perf/builtin-kvm.c +++ b/tools/perf/builtin-kvm.c @@ -22,6 +22,7 @@ #include "util/synthetic-events.h" #include "util/top.h" #include "util/data.h" +#include "util/event.h" #include "util/ordered-events.h" #include "util/kvm-stat.h" #include "util/util.h" @@ -50,6 +51,7 @@ #include #include #include +#include #include #include @@ -605,11 +607,11 @@ static const char *get_filename_for_perf_kvm(void) const char *filename; if (perf_host && !perf_guest) - filename = strdup("perf.data.host"); + filename = "perf.data.host"; else if (!perf_host && perf_guest) - filename = strdup("perf.data.guest"); + filename = "perf.data.guest"; else - filename = strdup("perf.data.kvm"); + filename = "perf.data.kvm"; return filename; } @@ -806,7 +808,6 @@ static bool update_kvm_event(struct perf_kvm_stat *kvm, } static bool is_child_event(struct perf_kvm_stat *kvm, - struct evsel *evsel, struct perf_sample *sample, struct event_key *key) { @@ -818,8 +819,8 @@ static bool is_child_event(struct perf_kvm_stat *kvm, return false; for (; child_ops->name; child_ops++) { - if (evsel__name_is(evsel, child_ops->name)) { - child_ops->get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, child_ops->name)) { + child_ops->get_key(sample, key); return true; } } @@ -917,11 +918,10 @@ static bool handle_end_event(struct perf_kvm_stat *kvm, static struct vcpu_event_record *per_vcpu_record(struct thread *thread, - struct evsel *evsel, struct perf_sample *sample) { /* Only kvm_entry records vcpu id. */ - if (!thread__priv(thread) && kvm_entry_event(evsel)) { + if (!thread__priv(thread) && kvm_entry_event(sample->evsel)) { struct vcpu_event_record *vcpu_record; struct machine *machine = maps__machine(thread__maps(thread)); uint16_t e_machine = thread__e_machine(thread, machine, /*e_flags=*/NULL); @@ -932,7 +932,7 @@ struct vcpu_event_record *per_vcpu_record(struct thread *thread, return NULL; } - vcpu_record->vcpu_id = evsel__intval(evsel, sample, vcpu_id_str(e_machine)); + vcpu_record->vcpu_id = perf_sample__intval(sample, vcpu_id_str(e_machine)); thread__set_priv(thread, vcpu_record); } @@ -941,14 +941,13 @@ struct vcpu_event_record *per_vcpu_record(struct thread *thread, static bool handle_kvm_event(struct perf_kvm_stat *kvm, struct thread *thread, - struct evsel *evsel, struct perf_sample *sample) { struct vcpu_event_record *vcpu_record; struct event_key key = { .key = INVALID_KEY, .exit_reasons = kvm->exit_reasons }; - vcpu_record = per_vcpu_record(thread, evsel, sample); + vcpu_record = per_vcpu_record(thread, sample); if (!vcpu_record) return true; @@ -957,13 +956,13 @@ static bool handle_kvm_event(struct perf_kvm_stat *kvm, (kvm->trace_vcpu != vcpu_record->vcpu_id)) return true; - if (kvm->events_ops->is_begin_event(evsel, sample, &key)) + if (kvm->events_ops->is_begin_event(sample, &key)) return handle_begin_event(kvm, vcpu_record, &key, sample); - if (is_child_event(kvm, evsel, sample, &key)) + if (is_child_event(kvm, sample, &key)) return handle_child_event(kvm, vcpu_record, &key, sample); - if (kvm->events_ops->is_end_event(evsel, sample, &key)) + if (kvm->events_ops->is_end_event(sample, &key)) return handle_end_event(kvm, vcpu_record, &key, sample); return true; @@ -1133,7 +1132,6 @@ static bool skip_sample(struct perf_kvm_stat *kvm, static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { int err = 0; @@ -1145,18 +1143,20 @@ static int process_sample_event(const struct perf_tool *tool, return 0; if (machine__resolve(machine, &kvm->al, sample) < 0) { - pr_warning("Fail to resolve address location, skip sample.\n"); + pr_warning("WARNING: at offset %#" PRIx64 ": fail to resolve address location, skipping sample\n", + sample->file_offset); return 0; } thread = machine__findnew_thread(machine, sample->pid, sample->tid); if (thread == NULL) { - pr_debug("problem processing %d event, skipping it.\n", - event->header.type); + pr_debug("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); return -1; } - if (!handle_kvm_event(kvm, thread, evsel, sample)) + if (!handle_kvm_event(kvm, thread, sample)) err = -1; thread__put(thread); @@ -1222,7 +1222,7 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx, int err; *mmap_time = ULLONG_MAX; - md = &evlist->mmap[idx]; + md = &evlist__mmap(evlist)[idx]; err = perf_mmap__read_init(&md->core); if (err < 0) return (err == -EAGAIN) ? 0 : -1; @@ -1267,7 +1267,7 @@ static int perf_kvm__mmap_read(struct perf_kvm_stat *kvm) s64 n, ntotal = 0; u64 flush_time = ULLONG_MAX, mmap_time; - for (i = 0; i < kvm->evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(kvm->evlist)->nr_mmaps; i++) { n = perf_kvm__mmap_read_idx(kvm, i, &mmap_time); if (n < 0) return -1; @@ -1450,7 +1450,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm) evlist__enable(kvm->evlist); while (!done) { - struct fdarray *fda = &kvm->evlist->core.pollfd; + struct fdarray *fda = &evlist__core(kvm->evlist)->pollfd; int rc; rc = perf_kvm__mmap_read(kvm); @@ -1532,7 +1532,7 @@ static int kvm_live_open_events(struct perf_kvm_stat *kvm) goto out; } - if (evlist__mmap(evlist, kvm->opts.mmap_pages) < 0) { + if (evlist__do_mmap(evlist, kvm->opts.mmap_pages) < 0) { ui__error("Failed to mmap the events: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__close(evlist); @@ -1682,18 +1682,18 @@ kvm_events_record(struct perf_kvm_stat *kvm, int argc, const char **argv) return -ENOMEM; for (i = 0; i < ARRAY_SIZE(record_args); i++) - rec_argv[i] = STRDUP_FAIL_EXIT(record_args[i]); + rec_argv[i] = record_args[i]; for (j = 0; j < events_tp_size; j++) { - rec_argv[i++] = STRDUP_FAIL_EXIT("-e"); - rec_argv[i++] = STRDUP_FAIL_EXIT(kvm_events_tp(e_machine)[j]); + rec_argv[i++] = "-e"; + rec_argv[i++] = kvm_events_tp(e_machine)[j]; } - rec_argv[i++] = STRDUP_FAIL_EXIT("-o"); - rec_argv[i++] = STRDUP_FAIL_EXIT(kvm->file_name); + rec_argv[i++] = "-o"; + rec_argv[i++] = kvm->file_name; for (j = 1; j < (unsigned int)argc; j++, i++) - rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]); + rec_argv[i] = argv[j]; set_option_flag(record_options, 'e', "event", PARSE_OPT_HIDDEN); set_option_flag(record_options, 0, "filter", PARSE_OPT_HIDDEN); @@ -1717,10 +1717,6 @@ kvm_events_record(struct perf_kvm_stat *kvm, int argc, const char **argv) record_usage = kvm_stat_record_usage; ret = cmd_record(i, rec_argv); - -EXIT: - for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); free(rec_argv); return ret; } @@ -1811,7 +1807,7 @@ static struct evlist *kvm_live_event_list(void) out: if (err) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } @@ -1932,7 +1928,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, perf_session__set_id_hdr_size(kvm->session); ordered_events__set_copy_on_queue(&kvm->session->ordered_events, true); machine__synthesize_threads(&kvm->session->machines.host, &kvm->opts.target, - kvm->evlist->core.threads, true, false, 1); + evlist__core(kvm->evlist)->threads, true, false, 1); err = kvm_live_open_events(kvm); if (err) goto out; @@ -1942,7 +1938,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm, out: perf_session__delete(kvm->session); kvm->session = NULL; - evlist__delete(kvm->evlist); + evlist__put(kvm->evlist); return err; } @@ -1996,34 +1992,35 @@ static int __cmd_record(const char *file_name, int argc, const char **argv) { int rec_argc, i = 0, j, ret; const char **rec_argv; + int need_arch_event = !!kvm_need_default_arch_event(EM_HOST, argc, argv); /* * Besides the 2 more options "-o" and "filename", * kvm_add_default_arch_event() may add 2 extra options, - * so allocate 4 more items. + * so allocate more items conditionally. */ - rec_argc = argc + 2 + 2; + rec_argc = argc + 2 + (2 * need_arch_event); rec_argv = calloc(rec_argc + 1, sizeof(char *)); if (!rec_argv) return -ENOMEM; - rec_argv[i++] = STRDUP_FAIL_EXIT("record"); - rec_argv[i++] = STRDUP_FAIL_EXIT("-o"); - rec_argv[i++] = STRDUP_FAIL_EXIT(file_name); - for (j = 1; j < argc; j++, i++) - rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]); + rec_argv[i++] = "record"; + rec_argv[i++] = "-o"; + rec_argv[i++] = file_name; + if (need_arch_event) { + ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv); + if (ret) + goto EXIT; + } - BUG_ON(i + 2 != rec_argc); + for (j = 1; j < argc; j++, i++) + rec_argv[i] = argv[j]; - ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv); - if (ret) - goto EXIT; + BUG_ON(i != rec_argc); ret = cmd_record(i, rec_argv); EXIT: - for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); free(rec_argv); return ret; } @@ -2038,19 +2035,15 @@ static int __cmd_report(const char *file_name, int argc, const char **argv) if (!rec_argv) return -ENOMEM; - rec_argv[i++] = STRDUP_FAIL_EXIT("report"); - rec_argv[i++] = STRDUP_FAIL_EXIT("-i"); - rec_argv[i++] = STRDUP_FAIL_EXIT(file_name); + rec_argv[i++] = "report"; + rec_argv[i++] = "-i"; + rec_argv[i++] = file_name; for (j = 1; j < argc; j++, i++) - rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]); + rec_argv[i] = argv[j]; BUG_ON(i != rec_argc); ret = cmd_report(i, rec_argv); - -EXIT: - for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); free(rec_argv); return ret; } @@ -2066,19 +2059,15 @@ __cmd_buildid_list(const char *file_name, int argc, const char **argv) if (!rec_argv) return -ENOMEM; - rec_argv[i++] = STRDUP_FAIL_EXIT("buildid-list"); - rec_argv[i++] = STRDUP_FAIL_EXIT("-i"); - rec_argv[i++] = STRDUP_FAIL_EXIT(file_name); + rec_argv[i++] = "buildid-list"; + rec_argv[i++] = "-i"; + rec_argv[i++] = file_name; for (j = 1; j < argc; j++, i++) - rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]); + rec_argv[i] = argv[j]; BUG_ON(i != rec_argc); ret = cmd_buildid_list(i, rec_argv); - -EXIT: - for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); free(rec_argv); return ret; } @@ -2098,19 +2087,19 @@ static int __cmd_top(int argc, const char **argv) return -ENOMEM; for (i = 0; i < argc; i++) - rec_argv[i] = STRDUP_FAIL_EXIT(argv[i]); + rec_argv[i] = argv[i]; BUG_ON(i != argc); - ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv); - if (ret) - goto EXIT; + if (kvm_need_default_arch_event(EM_HOST, argc, argv)) { + ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv); + if (ret) + goto EXIT; + } ret = cmd_top(i, rec_argv); EXIT: - for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); free(rec_argv); return ret; } @@ -2146,6 +2135,7 @@ int cmd_kvm(int argc, const char **argv) const char *const kvm_subcommands[] = { "top", "record", "report", "diff", "buildid-list", "stat", NULL }; const char *kvm_usage[] = { NULL, NULL }; + int ret; exclude_GH_default = true; perf_host = 0; @@ -2159,28 +2149,22 @@ int cmd_kvm(int argc, const char **argv) if (!perf_host) perf_guest = 1; - if (!file_name) { + if (!file_name) file_name = get_filename_for_perf_kvm(); - if (!file_name) { - pr_err("Failed to allocate memory for filename\n"); - return -ENOMEM; - } - } - if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) - return __cmd_record(file_name, argc, argv); + ret = __cmd_record(file_name, argc, argv); else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) - return __cmd_report(file_name, argc, argv); + ret = __cmd_report(file_name, argc, argv); else if (strlen(argv[0]) > 2 && strstarts("diff", argv[0])) - return cmd_diff(argc, argv); + ret = cmd_diff(argc, argv); else if (!strcmp(argv[0], "top")) - return __cmd_top(argc, argv); + ret = __cmd_top(argc, argv); else if (strlen(argv[0]) > 2 && strstarts("buildid-list", argv[0])) - return __cmd_buildid_list(file_name, argc, argv); + ret = __cmd_buildid_list(file_name, argc, argv); #if defined(HAVE_LIBTRACEEVENT) else if (strlen(argv[0]) > 2 && strstarts("stat", argv[0])) - return kvm_cmd_stat(file_name, argc, argv); + ret = kvm_cmd_stat(file_name, argc, argv); #endif else usage_with_options(kvm_usage, kvm_options); @@ -2188,5 +2172,5 @@ int cmd_kvm(int argc, const char **argv) /* free usage string allocated by parse_options_subcommand */ free((void *)kvm_usage[0]); - return 0; + return ret; } diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 9d3a4c779a41..fce588441e49 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -9,6 +9,7 @@ #include "perf.h" #include "util/data.h" +#include "util/event.h" #include "util/evlist.h" #include "util/evsel.h" #include "util/header.h" @@ -323,8 +324,8 @@ static struct kwork_work *work_search(struct rb_root_cached *root, else if (cmp < 0) node = node->rb_right; else { - if (work->name == NULL) - work->name = key->name; + if (work->name == NULL && key->name != NULL) + work->name = strdup(key->name); return work; } } @@ -371,11 +372,54 @@ static struct kwork_work *work_new(struct kwork_work *key) work->id = key->id; work->cpu = key->cpu; - work->name = key->name; + work->name = key->name ? strdup(key->name) : NULL; work->class = key->class; return work; } + +static void work_delete(struct kwork_work *work) +{ + if (work) { + work_exit(work); + free(work); + } +} + +static void kwork_work__free_root(struct rb_root_cached *root) +{ + struct rb_node *next; + struct kwork_work *work; + + while ((next = rb_first_cached(root))) { + work = rb_entry(next, struct kwork_work, node); + rb_erase_cached(next, root); + work_delete(work); + } +} + +static void perf_kwork__exit(struct perf_kwork *kwork) +{ + struct kwork_class *class; + struct kwork_atom_page *page, *tmp_page; + + list_for_each_entry(class, &kwork->class_list, list) { + kwork_work__free_root(&class->work_root); + } + + kwork_work__free_root(&kwork->sorted_work_root); + + list_for_each_entry_safe(page, tmp_page, &kwork->atom_page_list, list) { + list_del_init(&page->list); + free(page); + } + + INIT_LIST_HEAD(&kwork->class_list); + INIT_LIST_HEAD(&kwork->atom_page_list); + INIT_LIST_HEAD(&kwork->sort_list); + INIT_LIST_HEAD(&kwork->cmp_id); +} + static struct kwork_work *work_findnew(struct rb_root_cached *root, struct kwork_work *key, struct list_head *sort_list) @@ -424,7 +468,9 @@ static bool profile_event_match(struct perf_kwork *kwork, u64 time = sample->time; struct perf_time_interval *ptime = &kwork->ptime; - if ((kwork->cpu_list != NULL) && !test_bit(cpu, kwork->cpu_bitmap)) + /* Guard test_bit: cpu == -1 (absent PERF_SAMPLE_CPU) would index past the bitmap */ + if ((kwork->cpu_list != NULL) && + ((unsigned int)cpu >= MAX_NR_CPUS || !test_bit(cpu, kwork->cpu_bitmap))) return false; if (((ptime->start != 0) && (ptime->start > time)) || @@ -448,31 +494,34 @@ static int work_push_atom(struct perf_kwork *kwork, struct kwork_class *class, enum kwork_trace_type src_type, enum kwork_trace_type dst_type, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine, struct kwork_work **ret_work, bool overwrite) { - struct kwork_atom *atom, *dst_atom, *last_atom; + struct kwork_atom *atom = NULL, *dst_atom, *last_atom; struct kwork_work *work, key; + int ret = 0; BUG_ON(class->work_init == NULL); - class->work_init(kwork, class, &key, src_type, evsel, sample, machine); + class->work_init(kwork, class, &key, src_type, sample, machine); atom = atom_new(kwork, sample); - if (atom == NULL) - return -1; + if (atom == NULL) { + ret = -1; + goto out; + } work = work_findnew(&class->work_root, &key, &kwork->cmp_id); if (work == NULL) { atom_free(atom); - return -1; + ret = -1; + goto out; } if (!profile_event_match(kwork, work, sample)) { atom_free(atom); - return 0; + goto out; } if (dst_type < KWORK_TRACE_MAX) { @@ -499,39 +548,39 @@ static int work_push_atom(struct perf_kwork *kwork, } list_add_tail(&atom->list, &work->atom_list[src_type]); - - return 0; +out: + work_exit(&key); + return ret; } static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork, struct kwork_class *class, enum kwork_trace_type src_type, enum kwork_trace_type dst_type, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine, struct kwork_work **ret_work) { - struct kwork_atom *atom, *src_atom; + struct kwork_atom *atom = NULL, *src_atom; struct kwork_work *work, key; BUG_ON(class->work_init == NULL); - class->work_init(kwork, class, &key, src_type, evsel, sample, machine); + class->work_init(kwork, class, &key, src_type, sample, machine); work = work_findnew(&class->work_root, &key, &kwork->cmp_id); if (ret_work != NULL) *ret_work = work; if (work == NULL) - return NULL; + goto out; if (!profile_event_match(kwork, work, sample)) - return NULL; + goto out; atom = list_last_entry_or_null(&work->atom_list[dst_type], struct kwork_atom, list); if (atom != NULL) - return atom; + goto out; src_atom = atom_new(kwork, sample); if (src_atom != NULL) @@ -540,8 +589,9 @@ static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork, if (ret_work != NULL) *ret_work = NULL; } - - return NULL; +out: + work_exit(&key); + return atom; } static struct kwork_work *find_work_by_id(struct rb_root_cached *root, @@ -599,18 +649,16 @@ static void report_update_exit_event(struct kwork_work *work, static int report_entry_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { return work_push_atom(kwork, class, KWORK_TRACE_ENTRY, - KWORK_TRACE_MAX, evsel, sample, + KWORK_TRACE_MAX, sample, machine, NULL, true); } static int report_exit_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -618,7 +666,7 @@ static int report_exit_event(struct perf_kwork *kwork, struct kwork_work *work = NULL; atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, - KWORK_TRACE_ENTRY, evsel, sample, + KWORK_TRACE_ENTRY, sample, machine, &work); if (work == NULL) return -1; @@ -654,18 +702,16 @@ static void latency_update_entry_event(struct kwork_work *work, static int latency_raise_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { return work_push_atom(kwork, class, KWORK_TRACE_RAISE, - KWORK_TRACE_MAX, evsel, sample, + KWORK_TRACE_MAX, sample, machine, NULL, true); } static int latency_entry_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -673,7 +719,7 @@ static int latency_entry_event(struct perf_kwork *kwork, struct kwork_work *work = NULL; atom = work_pop_atom(kwork, class, KWORK_TRACE_ENTRY, - KWORK_TRACE_RAISE, evsel, sample, + KWORK_TRACE_RAISE, sample, machine, &work); if (work == NULL) return -1; @@ -688,7 +734,6 @@ static int latency_entry_event(struct perf_kwork *kwork, static void timehist_save_callchain(struct perf_kwork *kwork, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct symbol *sym; @@ -708,7 +753,7 @@ static void timehist_save_callchain(struct perf_kwork *kwork, cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(thread, cursor, evsel, sample, + if (thread__resolve_callchain(thread, cursor, sample, NULL, NULL, kwork->max_stack + 2) != 0) { pr_debug("Failed to resolve callchain, skipping\n"); goto out_put; @@ -725,7 +770,7 @@ static void timehist_save_callchain(struct perf_kwork *kwork, if (sym) { if (!strcmp(sym->name, "__softirqentry_text_start") || !strcmp(sym->name, "__do_softirq")) - sym->ignore = 1; + symbol__set_ignore(sym, true); } callchain_cursor_advance(cursor); @@ -813,18 +858,16 @@ static void timehist_print_event(struct perf_kwork *kwork, static int timehist_raise_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { return work_push_atom(kwork, class, KWORK_TRACE_RAISE, - KWORK_TRACE_MAX, evsel, sample, + KWORK_TRACE_MAX, sample, machine, NULL, true); } static int timehist_entry_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -832,20 +875,19 @@ static int timehist_entry_event(struct perf_kwork *kwork, struct kwork_work *work = NULL; ret = work_push_atom(kwork, class, KWORK_TRACE_ENTRY, - KWORK_TRACE_RAISE, evsel, sample, + KWORK_TRACE_RAISE, sample, machine, &work, true); if (ret) return ret; if (work != NULL) - timehist_save_callchain(kwork, sample, evsel, machine); + timehist_save_callchain(kwork, sample, machine); return 0; } static int timehist_exit_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -856,13 +898,14 @@ static int timehist_exit_event(struct perf_kwork *kwork, addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_debug("Problem processing event, skipping it\n"); + pr_debug("problem processing event at offset %#" PRIx64 ", skipping it\n", + sample->file_offset); ret = -1; goto out; } atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, - KWORK_TRACE_ENTRY, evsel, sample, + KWORK_TRACE_ENTRY, sample, machine, &work); if (work == NULL) { ret = -1; @@ -896,18 +939,16 @@ static void top_update_runtime(struct kwork_work *work, static int top_entry_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { return work_push_atom(kwork, class, KWORK_TRACE_ENTRY, - KWORK_TRACE_MAX, evsel, sample, + KWORK_TRACE_MAX, sample, machine, NULL, true); } static int top_exit_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -916,7 +957,7 @@ static int top_exit_event(struct perf_kwork *kwork, struct kwork_atom *atom; atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, - KWORK_TRACE_ENTRY, evsel, sample, + KWORK_TRACE_ENTRY, sample, machine, &work); if (!work) return -1; @@ -937,7 +978,6 @@ static int top_exit_event(struct perf_kwork *kwork, static int top_sched_switch_event(struct perf_kwork *kwork, struct kwork_class *class, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -945,7 +985,7 @@ static int top_sched_switch_event(struct perf_kwork *kwork, struct kwork_work *work; atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, - KWORK_TRACE_ENTRY, evsel, sample, + KWORK_TRACE_ENTRY, sample, machine, &work); if (!work) return -1; @@ -955,12 +995,11 @@ static int top_sched_switch_event(struct perf_kwork *kwork, atom_del(atom); } - return top_entry_event(kwork, class, evsel, sample, machine); + return top_entry_event(kwork, class, sample, machine); } static struct kwork_class kwork_irq; static int process_irq_handler_entry_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -968,12 +1007,11 @@ static int process_irq_handler_entry_event(const struct perf_tool *tool, if (kwork->tp_handler->entry_event) return kwork->tp_handler->entry_event(kwork, &kwork_irq, - evsel, sample, machine); + sample, machine); return 0; } static int process_irq_handler_exit_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -981,7 +1019,7 @@ static int process_irq_handler_exit_event(const struct perf_tool *tool, if (kwork->tp_handler->exit_event) return kwork->tp_handler->exit_event(kwork, &kwork_irq, - evsel, sample, machine); + sample, machine); return 0; } @@ -1006,7 +1044,6 @@ static void irq_work_init(struct perf_kwork *kwork, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type __maybe_unused, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) { @@ -1014,17 +1051,20 @@ static void irq_work_init(struct perf_kwork *kwork, work->cpu = sample->cpu; if (kwork->report == KWORK_REPORT_TOP) { - work->id = evsel__intval_common(evsel, sample, "common_pid"); + work->id = perf_sample__intval_common(sample, "common_pid"); work->name = NULL; } else { - work->id = evsel__intval(evsel, sample, "irq"); - work->name = evsel__strval(evsel, sample, "name"); + work->id = perf_sample__intval(sample, "irq"); + work->name = strdup(perf_sample__strval(sample, "name") ?: ""); } } static void irq_work_name(struct kwork_work *work, char *buf, int len) { - snprintf(buf, len, "%s:%" PRIu64 "", work->name, work->id); + if (work->name != NULL) + snprintf(buf, len, "%s:%" PRIu64 "", work->name, work->id); + else + snprintf(buf, len, "%" PRIu64 "", work->id); } static struct kwork_class kwork_irq = { @@ -1039,7 +1079,6 @@ static struct kwork_class kwork_irq = { static struct kwork_class kwork_softirq; static int process_softirq_raise_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1047,13 +1086,12 @@ static int process_softirq_raise_event(const struct perf_tool *tool, if (kwork->tp_handler->raise_event) return kwork->tp_handler->raise_event(kwork, &kwork_softirq, - evsel, sample, machine); + sample, machine); return 0; } static int process_softirq_entry_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1061,13 +1099,12 @@ static int process_softirq_entry_event(const struct perf_tool *tool, if (kwork->tp_handler->entry_event) return kwork->tp_handler->entry_event(kwork, &kwork_softirq, - evsel, sample, machine); + sample, machine); return 0; } static int process_softirq_exit_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1075,7 +1112,7 @@ static int process_softirq_exit_event(const struct perf_tool *tool, if (kwork->tp_handler->exit_event) return kwork->tp_handler->exit_event(kwork, &kwork_softirq, - evsel, sample, machine); + sample, machine); return 0; } @@ -1134,7 +1171,6 @@ static void softirq_work_init(struct perf_kwork *kwork, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type __maybe_unused, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) { @@ -1144,18 +1180,21 @@ static void softirq_work_init(struct perf_kwork *kwork, work->cpu = sample->cpu; if (kwork->report == KWORK_REPORT_TOP) { - work->id = evsel__intval_common(evsel, sample, "common_pid"); + work->id = perf_sample__intval_common(sample, "common_pid"); work->name = NULL; } else { - num = evsel__intval(evsel, sample, "vec"); + num = perf_sample__intval(sample, "vec"); work->id = num; - work->name = evsel__softirq_name(evsel, num); + work->name = evsel__softirq_name(sample->evsel, num); } } static void softirq_work_name(struct kwork_work *work, char *buf, int len) { - snprintf(buf, len, "(s)%s:%" PRIu64 "", work->name, work->id); + if (work->name != NULL) + snprintf(buf, len, "(s)%s:%" PRIu64 "", work->name, work->id); + else + snprintf(buf, len, "(s)%" PRIu64 "", work->id); } static struct kwork_class kwork_softirq = { @@ -1170,7 +1209,6 @@ static struct kwork_class kwork_softirq = { static struct kwork_class kwork_workqueue; static int process_workqueue_activate_work_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1178,13 +1216,12 @@ static int process_workqueue_activate_work_event(const struct perf_tool *tool, if (kwork->tp_handler->raise_event) return kwork->tp_handler->raise_event(kwork, &kwork_workqueue, - evsel, sample, machine); + sample, machine); return 0; } static int process_workqueue_execute_start_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1192,13 +1229,12 @@ static int process_workqueue_execute_start_event(const struct perf_tool *tool, if (kwork->tp_handler->entry_event) return kwork->tp_handler->entry_event(kwork, &kwork_workqueue, - evsel, sample, machine); + sample, machine); return 0; } static int process_workqueue_execute_end_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1206,7 +1242,7 @@ static int process_workqueue_execute_end_event(const struct perf_tool *tool, if (kwork->tp_handler->exit_event) return kwork->tp_handler->exit_event(kwork, &kwork_workqueue, - evsel, sample, machine); + sample, machine); return 0; } @@ -1234,19 +1270,23 @@ static void workqueue_work_init(struct perf_kwork *kwork __maybe_unused, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type __maybe_unused, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { char *modp = NULL; - unsigned long long function_addr = evsel__intval(evsel, - sample, "function"); + unsigned long long function_addr = perf_sample__intval(sample, "function"); work->class = class; work->cpu = sample->cpu; - work->id = evsel__intval(evsel, sample, "work"); - work->name = function_addr == 0 ? NULL : - machine__resolve_kernel_addr(machine, &function_addr, &modp); + work->id = perf_sample__intval(sample, "work"); + work->name = NULL; + + if (function_addr != 0) { + const char *name = machine__resolve_kernel_addr(machine, &function_addr, &modp); + + if (name) + work->name = strdup(name); + } } static void workqueue_work_name(struct kwork_work *work, char *buf, int len) @@ -1269,7 +1309,6 @@ static struct kwork_class kwork_workqueue = { static struct kwork_class kwork_sched; static int process_sched_switch_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -1277,7 +1316,7 @@ static int process_sched_switch_event(const struct perf_tool *tool, if (kwork->tp_handler->sched_switch_event) return kwork->tp_handler->sched_switch_event(kwork, &kwork_sched, - evsel, sample, machine); + sample, machine); return 0; } @@ -1302,7 +1341,6 @@ static void sched_work_init(struct perf_kwork *kwork __maybe_unused, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) { @@ -1310,17 +1348,17 @@ static void sched_work_init(struct perf_kwork *kwork __maybe_unused, work->cpu = sample->cpu; if (src_type == KWORK_TRACE_EXIT) { - work->id = evsel__intval(evsel, sample, "prev_pid"); - work->name = strdup(evsel__strval(evsel, sample, "prev_comm")); + work->id = perf_sample__intval(sample, "prev_pid"); + work->name = strdup(perf_sample__strval(sample, "prev_comm") ?: ""); } else if (src_type == KWORK_TRACE_ENTRY) { - work->id = evsel__intval(evsel, sample, "next_pid"); - work->name = strdup(evsel__strval(evsel, sample, "next_comm")); + work->id = perf_sample__intval(sample, "next_pid"); + work->name = strdup(perf_sample__strval(sample, "next_comm") ?: ""); } } static void sched_work_name(struct kwork_work *work, char *buf, int len) { - snprintf(buf, len, "%s", work->name); + snprintf(buf, len, "%s", work->name ?: ""); } static struct kwork_class kwork_sched = { @@ -1776,7 +1814,7 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, } } - list_for_each_entry(evsel, &session->evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(session->evlist)->entries, core.node) { if (kwork->show_callchain && !evsel__has_callchain(evsel)) { pr_debug("Samples do not have callchains\n"); kwork->show_callchain = 0; @@ -1826,9 +1864,9 @@ static int perf_kwork__read_events(struct perf_kwork *kwork) goto out_delete; } - kwork->nr_events = session->evlist->stats.nr_events[0]; - kwork->nr_lost_events = session->evlist->stats.total_lost; - kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + kwork->nr_events = evlist__stats(session->evlist)->nr_events[0]; + kwork->nr_lost_events = evlist__stats(session->evlist)->total_lost; + kwork->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; out_delete: perf_session__delete(session); @@ -1948,22 +1986,21 @@ static int perf_kwork__report(struct perf_kwork *kwork) } typedef int (*tracepoint_handler)(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); static int perf_kwork__process_tracepoint_sample(const struct perf_tool *tool, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { + struct evsel *evsel = sample->evsel; int err = 0; if (evsel->handler != NULL) { tracepoint_handler f = evsel->handler; - err = f(tool, evsel, sample, machine); + err = f(tool, sample, machine); } return err; @@ -2008,7 +2045,18 @@ static void top_calc_total_runtime(struct perf_kwork *kwork) next = rb_first_cached(&class->work_root); while (next) { work = rb_entry(next, struct kwork_work, node); - BUG_ON(work->cpu >= MAX_NR_CPUS); + /* + * work->cpu comes from sample->cpu which is -1 when + * PERF_SAMPLE_CPU is absent. As int that's -1, but as + * unsigned it exceeds MAX_NR_CPUS — skip to avoid OOB + * on cpus_runtime[]. + */ + /* Counted and reported in perf_kwork__top_report() */ + if ((unsigned int)work->cpu >= MAX_NR_CPUS) { + stat->nr_skipped_cpu++; + next = rb_next(next); + continue; + } stat->cpus_runtime[work->cpu].total += work->total_runtime; stat->cpus_runtime[MAX_NR_CPUS].total += work->total_runtime; next = rb_next(next); @@ -2020,7 +2068,8 @@ static void top_calc_idle_time(struct perf_kwork *kwork, { struct kwork_top_stat *stat = &kwork->top_stat; - if (work->id == 0) { + /* See comment in top_calc_total_runtime() */ + if (work->id == 0 && (unsigned int)work->cpu < MAX_NR_CPUS) { stat->cpus_runtime[work->cpu].idle += work->total_runtime; stat->cpus_runtime[MAX_NR_CPUS].idle += work->total_runtime; } @@ -2032,6 +2081,10 @@ static void top_calc_irq_runtime(struct perf_kwork *kwork, { struct kwork_top_stat *stat = &kwork->top_stat; + /* See comment in top_calc_total_runtime() */ + if ((unsigned int)work->cpu >= MAX_NR_CPUS) + return; + if (type == KWORK_CLASS_IRQ) { stat->cpus_runtime[work->cpu].irq += work->total_runtime; stat->cpus_runtime[MAX_NR_CPUS].irq += work->total_runtime; @@ -2084,12 +2137,19 @@ static void top_calc_cpu_usage(struct perf_kwork *kwork) if (work->total_runtime == 0) goto next; + /* See comment in top_calc_total_runtime() */ + if ((unsigned int)work->cpu >= MAX_NR_CPUS) + goto next; + __set_bit(work->cpu, stat->all_cpus_bitmap); top_subtract_irq_runtime(kwork, work); - work->cpu_usage = work->total_runtime * 10000 / - stat->cpus_runtime[work->cpu].total; + /* Guard against division by zero if no runtime was accumulated */ + if (stat->cpus_runtime[work->cpu].total) { + work->cpu_usage = work->total_runtime * 10000 / + stat->cpus_runtime[work->cpu].total; + } top_calc_idle_time(kwork, work); next: @@ -2102,7 +2162,8 @@ static void top_calc_load_runtime(struct perf_kwork *kwork, { struct kwork_top_stat *stat = &kwork->top_stat; - if (work->id != 0) { + /* See comment in top_calc_total_runtime() */ + if (work->id != 0 && (unsigned int)work->cpu < MAX_NR_CPUS) { stat->cpus_runtime[work->cpu].load += work->total_runtime; stat->cpus_runtime[MAX_NR_CPUS].load += work->total_runtime; } @@ -2128,8 +2189,10 @@ static void top_merge_tasks(struct perf_kwork *kwork) rb_erase_cached(node, &class->work_root); data = rb_entry(node, struct kwork_work, node); - if (!profile_name_match(kwork, data)) + if (!profile_name_match(kwork, data)) { + work_delete(data); continue; + } cpu = data->cpu; merged_work = find_work_by_id(&merged_root, data->id, @@ -2137,11 +2200,17 @@ static void top_merge_tasks(struct perf_kwork *kwork) if (!merged_work) { work_insert(&merged_root, data, &kwork->cmp_id); } else { + if (merged_work->name == NULL && data->name != NULL) + merged_work->name = strdup(data->name); + merged_work->total_runtime += data->total_runtime; merged_work->cpu_usage += data->cpu_usage; } top_calc_load_runtime(kwork, data); + + if (merged_work) + work_delete(data); } work_sort(kwork, class, &merged_root); @@ -2170,6 +2239,13 @@ static void perf_kwork__top_report(struct perf_kwork *kwork) next = rb_next(next); } + if (kwork->top_stat.nr_skipped_cpu) { + printf(" Warning: %u work entries with invalid CPU were excluded from totals.\n" + " Task runtimes may appear inflated (IRQ time not subtracted).\n" + " Consider re-recording with PERF_SAMPLE_CPU enabled.\n", + kwork->top_stat.nr_skipped_cpu); + } + printf("\n"); } @@ -2286,7 +2362,7 @@ static void setup_event_list(struct perf_kwork *kwork, static int perf_kwork__record(struct perf_kwork *kwork, int argc, const char **argv) { - const char **rec_argv; + const char **rec_argv, **to_free = NULL; unsigned int rec_argc, i, j; struct kwork_class *class; int ret; @@ -2323,16 +2399,27 @@ static int perf_kwork__record(struct perf_kwork *kwork, BUG_ON(i != rec_argc); + /* Save the pointers as the array will be mutated by cmd_record. */ + to_free = calloc(rec_argc + 1, sizeof(char *)); + if (to_free == NULL) { + ret = -ENOMEM; + goto EXIT; + } + pr_debug("record comm: "); - for (j = 0; j < rec_argc; j++) + for (j = 0; j < rec_argc; j++) { pr_debug("%s ", rec_argv[j]); + to_free[j] = rec_argv[j]; + } pr_debug("\n"); ret = cmd_record(i, rec_argv); EXIT: for (i = 0; i < rec_argc; i++) - free((void *)rec_argv[i]); + free((void *)(to_free ? to_free[i] : rec_argv[i])); + + free(to_free); free(rec_argv); return ret; } @@ -2475,6 +2562,7 @@ int cmd_kwork(int argc, const char **argv) const char *const kwork_subcommands[] = { "record", "report", "latency", "timehist", "top", NULL }; + int ret = 0; perf_tool__init(&kwork.tool, /*ordered_events=*/true); kwork.tool.mmap = perf_event__process_mmap; @@ -2491,7 +2579,7 @@ int cmd_kwork(int argc, const char **argv) if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) { setup_event_list(&kwork, kwork_options, kwork_usage); - return perf_kwork__record(&kwork, argc, argv); + ret = perf_kwork__record(&kwork, argc, argv); } else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) { kwork.sort_order = default_report_sort_order; if (argc > 1) { @@ -2502,7 +2590,7 @@ int cmd_kwork(int argc, const char **argv) kwork.report = KWORK_REPORT_RUNTIME; setup_sorting(&kwork, report_options, report_usage); setup_event_list(&kwork, kwork_options, kwork_usage); - return perf_kwork__report(&kwork); + ret = perf_kwork__report(&kwork); } else if (strlen(argv[0]) > 2 && strstarts("latency", argv[0])) { kwork.sort_order = default_latency_sort_order; if (argc > 1) { @@ -2513,7 +2601,7 @@ int cmd_kwork(int argc, const char **argv) kwork.report = KWORK_REPORT_LATENCY; setup_sorting(&kwork, latency_options, latency_usage); setup_event_list(&kwork, kwork_options, kwork_usage); - return perf_kwork__report(&kwork); + ret = perf_kwork__report(&kwork); } else if (strlen(argv[0]) > 2 && strstarts("timehist", argv[0])) { if (argc > 1) { argc = parse_options(argc, argv, timehist_options, timehist_usage, 0); @@ -2522,7 +2610,7 @@ int cmd_kwork(int argc, const char **argv) } kwork.report = KWORK_REPORT_TIMEHIST; setup_event_list(&kwork, kwork_options, kwork_usage); - return perf_kwork__timehist(&kwork); + ret = perf_kwork__timehist(&kwork); } else if (strlen(argv[0]) > 2 && strstarts("top", argv[0])) { kwork.sort_order = default_top_sort_order; if (argc > 1) { @@ -2535,12 +2623,14 @@ int cmd_kwork(int argc, const char **argv) kwork.event_list_str = "sched, irq, softirq"; setup_event_list(&kwork, kwork_options, kwork_usage); setup_sorting(&kwork, top_options, top_usage); - return perf_kwork__top(&kwork); + ret = perf_kwork__top(&kwork); } else usage_with_options(kwork_usage, kwork_options); + perf_kwork__exit(&kwork); + /* free usage string allocated by parse_options_subcommand */ free((void *)kwork_usage[0]); - return 0; + return ret; } diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 5585aeb97684..d5c0d55cb82d 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -21,6 +21,7 @@ #include "util/tracepoint.h" #include "util/debug.h" +#include "util/event.h" #include "util/session.h" #include "util/tool.h" #include "util/data.h" @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -473,28 +475,22 @@ static struct lock_stat *pop_from_result(void) struct trace_lock_handler { /* it's used on CONFIG_LOCKDEP */ - int (*acquire_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*acquire_event)(struct perf_sample *sample); /* it's used on CONFIG_LOCKDEP && CONFIG_LOCK_STAT */ - int (*acquired_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*acquired_event)(struct perf_sample *sample); /* it's used on CONFIG_LOCKDEP && CONFIG_LOCK_STAT */ - int (*contended_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*contended_event)(struct perf_sample *sample); /* it's used on CONFIG_LOCKDEP */ - int (*release_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*release_event)(struct perf_sample *sample); /* it's used when CONFIG_LOCKDEP is off */ - int (*contention_begin_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*contention_begin_event)(struct perf_sample *sample); /* it's used when CONFIG_LOCKDEP is off */ - int (*contention_end_event)(struct evsel *evsel, - struct perf_sample *sample); + int (*contention_end_event)(struct perf_sample *sample); }; static struct lock_seq_stat *get_seq(struct thread_stat *ts, u64 addr) @@ -551,27 +547,26 @@ static int get_key_by_aggr_mode_simple(u64 *key, u64 addr, u32 tid) return 0; } -static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample); +static u64 callchain_id(struct perf_sample *sample); -static int get_key_by_aggr_mode(u64 *key, u64 addr, struct evsel *evsel, +static int get_key_by_aggr_mode(u64 *key, u64 addr, struct perf_sample *sample) { if (aggr_mode == LOCK_AGGR_CALLER) { - *key = callchain_id(evsel, sample); + *key = callchain_id(sample); return 0; } return get_key_by_aggr_mode_simple(key, addr, sample->tid); } -static int report_lock_acquire_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_acquire_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; - const char *name = evsel__strval(evsel, sample, "name"); - u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); - int flag = evsel__intval(evsel, sample, "flags"); + const char *name = perf_sample__strval(sample, "name"); + u64 addr = perf_sample__intval(sample, "lockdep_addr"); + int flag = perf_sample__intval(sample, "flags"); u64 key; int ret; @@ -638,15 +633,14 @@ static int report_lock_acquire_event(struct evsel *evsel, return 0; } -static int report_lock_acquired_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_acquired_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; u64 contended_term; - const char *name = evsel__strval(evsel, sample, "name"); - u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); + const char *name = perf_sample__strval(sample, "name"); + u64 addr = perf_sample__intval(sample, "lockdep_addr"); u64 key; int ret; @@ -704,14 +698,13 @@ static int report_lock_acquired_event(struct evsel *evsel, return 0; } -static int report_lock_contended_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_contended_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; - const char *name = evsel__strval(evsel, sample, "name"); - u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); + const char *name = perf_sample__strval(sample, "name"); + u64 addr = perf_sample__intval(sample, "lockdep_addr"); u64 key; int ret; @@ -762,14 +755,13 @@ static int report_lock_contended_event(struct evsel *evsel, return 0; } -static int report_lock_release_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_release_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; - const char *name = evsel__strval(evsel, sample, "name"); - u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); + const char *name = perf_sample__strval(sample, "name"); + u64 addr = perf_sample__intval(sample, "lockdep_addr"); u64 key; int ret; @@ -841,7 +833,7 @@ static int get_symbol_name_offset(struct map *map, struct symbol *sym, u64 ip, else return strlcpy(buf, sym->name, size); } -static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sample, +static int lock_contention_caller(struct perf_sample *sample, char *buf, int size) { struct thread *thread; @@ -862,7 +854,7 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl cursor = get_tls_callchain_cursor(); /* use caller function name from the callchain */ - ret = thread__resolve_callchain(thread, cursor, evsel, sample, + ret = thread__resolve_callchain(thread, cursor, sample, NULL, NULL, max_stack_depth); if (ret != 0) { thread__put(thread); @@ -896,7 +888,7 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl return -1; } -static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample) +static u64 callchain_id(struct perf_sample *sample) { struct callchain_cursor *cursor; struct machine *machine = &session->machines.host; @@ -911,7 +903,7 @@ static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample) cursor = get_tls_callchain_cursor(); /* use caller function name from the callchain */ - ret = thread__resolve_callchain(thread, cursor, evsel, sample, + ret = thread__resolve_callchain(thread, cursor, sample, NULL, NULL, max_stack_depth); thread__put(thread); @@ -948,9 +940,16 @@ static u64 *get_callstack(struct perf_sample *sample, int max_stack) u64 i; int c; + if (!sample->callchain) { + pr_debug("Sample unexpectedly missing callchain\n"); + return NULL; + } + callstack = calloc(max_stack, sizeof(*callstack)); - if (callstack == NULL) + if (callstack == NULL) { + pr_debug("Failed to allocate callstack\n"); return NULL; + } for (i = 0, c = 0; i < sample->callchain->nr && c < max_stack; i++) { u64 ip = sample->callchain->ips[i]; @@ -963,14 +962,13 @@ static u64 *get_callstack(struct perf_sample *sample, int max_stack) return callstack; } -static int report_lock_contention_begin_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_contention_begin_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; - u64 addr = evsel__intval(evsel, sample, "lock_addr"); - unsigned int flags = evsel__intval(evsel, sample, "flags"); + u64 addr = perf_sample__intval(sample, "lock_addr"); + unsigned int flags = perf_sample__intval(sample, "flags"); u64 key; int i, ret; static bool kmap_loaded; @@ -978,7 +976,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, struct map *kmap; struct symbol *sym; - ret = get_key_by_aggr_mode(&key, addr, evsel, sample); + ret = get_key_by_aggr_mode(&key, addr, sample); if (ret < 0) return ret; @@ -1025,7 +1023,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, break; case LOCK_AGGR_CALLER: name = buf; - if (lock_contention_caller(evsel, sample, buf, sizeof(buf)) < 0) + if (lock_contention_caller(sample, buf, sizeof(buf)) < 0) name = "Unknown"; break; case LOCK_AGGR_CGROUP: @@ -1070,7 +1068,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, if (needs_callstack()) { u64 *callstack = get_callstack(sample, max_stack_depth); if (callstack == NULL) - return -ENOMEM; + return 0; if (!match_callstack_filter(machine, callstack, max_stack_depth)) { free(callstack); @@ -1127,18 +1125,17 @@ static int report_lock_contention_begin_event(struct evsel *evsel, return 0; } -static int report_lock_contention_end_event(struct evsel *evsel, - struct perf_sample *sample) +static int report_lock_contention_end_event(struct perf_sample *sample) { struct lock_stat *ls; struct thread_stat *ts; struct lock_seq_stat *seq; u64 contended_term; - u64 addr = evsel__intval(evsel, sample, "lock_addr"); + u64 addr = perf_sample__intval(sample, "lock_addr"); u64 key; int ret; - ret = get_key_by_aggr_mode(&key, addr, evsel, sample); + ret = get_key_by_aggr_mode(&key, addr, sample); if (ret < 0) return ret; @@ -1191,7 +1188,7 @@ static int report_lock_contention_end_event(struct evsel *evsel, /* lock oriented handlers */ /* TODO: handlers for CPU oriented, thread oriented */ -static struct trace_lock_handler report_lock_ops = { +static const struct trace_lock_handler report_lock_ops = { .acquire_event = report_lock_acquire_event, .acquired_event = report_lock_acquired_event, .contended_event = report_lock_contended_event, @@ -1200,53 +1197,53 @@ static struct trace_lock_handler report_lock_ops = { .contention_end_event = report_lock_contention_end_event, }; -static struct trace_lock_handler contention_lock_ops = { +static const struct trace_lock_handler contention_lock_ops = { .contention_begin_event = report_lock_contention_begin_event, .contention_end_event = report_lock_contention_end_event, }; -static struct trace_lock_handler *trace_handler; +static const struct trace_lock_handler *trace_handler; -static int evsel__process_lock_acquire(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_lock_acquire(struct perf_sample *sample) { if (trace_handler->acquire_event) - return trace_handler->acquire_event(evsel, sample); + return trace_handler->acquire_event(sample); return 0; } -static int evsel__process_lock_acquired(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_lock_acquired(struct perf_sample *sample) { if (trace_handler->acquired_event) - return trace_handler->acquired_event(evsel, sample); + return trace_handler->acquired_event(sample); return 0; } -static int evsel__process_lock_contended(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_lock_contended(struct perf_sample *sample) { if (trace_handler->contended_event) - return trace_handler->contended_event(evsel, sample); + return trace_handler->contended_event(sample); return 0; } -static int evsel__process_lock_release(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_lock_release(struct perf_sample *sample) { if (trace_handler->release_event) - return trace_handler->release_event(evsel, sample); + return trace_handler->release_event(sample); return 0; } -static int evsel__process_contention_begin(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_contention_begin(struct perf_sample *sample) { if (trace_handler->contention_begin_event) - return trace_handler->contention_begin_event(evsel, sample); + return trace_handler->contention_begin_event(sample); return 0; } -static int evsel__process_contention_end(struct evsel *evsel, struct perf_sample *sample) +static int evsel__process_contention_end(struct perf_sample *sample) { if (trace_handler->contention_end_event) - return trace_handler->contention_end_event(evsel, sample); + return trace_handler->contention_end_event(sample); return 0; } @@ -1424,28 +1421,28 @@ static int process_event_update(const struct perf_tool *tool, return 0; } -typedef int (*tracepoint_handler)(struct evsel *evsel, - struct perf_sample *sample); +typedef int (*tracepoint_handler)(struct perf_sample *sample); static int process_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { + struct evsel *evsel = sample->evsel; int err = 0; struct thread *thread = machine__findnew_thread(machine, sample->pid, sample->tid); if (thread == NULL) { - pr_debug("problem processing %d event, skipping it.\n", - event->header.type); + pr_debug("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); return -1; } if (evsel->handler != NULL) { tracepoint_handler f = evsel->handler; - err = f(evsel, sample); + err = f(sample); } thread__put(thread); @@ -1921,8 +1918,11 @@ static int __cmd_report(bool display_info) return err; } +static volatile sig_atomic_t done; + static void sighandler(int sig __maybe_unused) { + done = 1; } static int check_lock_contention_options(const struct option *options, @@ -2060,6 +2060,7 @@ static int __cmd_contention(int argc, const char **argv) goto out_delete; } + done = 0; signal(SIGINT, sighandler); signal(SIGCHLD, sighandler); signal(SIGTERM, sighandler); @@ -2127,8 +2128,11 @@ static int __cmd_contention(int argc, const char **argv) if (argc) evlist__start_workload(con.evlist); - /* wait for signal */ - pause(); + while (!done) { + if (argc && waitpid(evlist__workload_pid(con.evlist), NULL, WNOHANG) > 0) + break; + sleep(1); + } lock_contention_stop(); lock_contention_read(&con); @@ -2145,7 +2149,7 @@ static int __cmd_contention(int argc, const char **argv) out_delete: lock_filter_finish(); - evlist__delete(con.evlist); + evlist__put(con.evlist); lock_contention_finish(&con); perf_session__delete(session); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-mem.c b/tools/perf/builtin-mem.c index d43500b92a7b..6101a26b3a78 100644 --- a/tools/perf/builtin-mem.c +++ b/tools/perf/builtin-mem.c @@ -255,7 +255,6 @@ dump_raw_samples(const struct perf_tool *tool, static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel __maybe_unused, struct machine *machine) { return dump_raw_samples(tool, event, sample, machine); diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 4a5eba498c02..ebd3ed0c9b3e 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -14,6 +14,7 @@ #include "util/parse-events.h" #include "util/config.h" +#include "util/arm64-frame-pointer-unwind-support.h" #include "util/callchain.h" #include "util/cgroup.h" #include "util/header.h" @@ -501,12 +502,12 @@ static void record__aio_mmap_read_sync(struct record *rec) { int i; struct evlist *evlist = rec->evlist; - struct mmap *maps = evlist->mmap; + struct mmap *maps = evlist__mmap(evlist); if (!record__aio_enabled(rec)) return; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct mmap *map = &maps[i]; if (map->core.base) @@ -810,8 +811,8 @@ static int record__auxtrace_read_snapshot_all(struct record *rec) int i; int rc = 0; - for (i = 0; i < rec->evlist->core.nr_mmaps; i++) { - struct mmap *map = &rec->evlist->mmap[i]; + for (i = 0; i < evlist__core(rec->evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(rec->evlist)[i]; if (!map->auxtrace_mmap.base) continue; @@ -865,6 +866,7 @@ static int record__auxtrace_init(struct record *rec) } if (!rec->itr) { + err = -EINVAL; rec->itr = auxtrace_record__init(rec->evlist, &err); if (err) return err; @@ -1053,15 +1055,15 @@ static void record__thread_data_close_pipes(struct record_thread *thread_data) static bool evlist__per_thread(struct evlist *evlist) { - return cpu_map__is_dummy(evlist->core.user_requested_cpus); + return cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus); } static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist) { - int m, tm, nr_mmaps = evlist->core.nr_mmaps; - struct mmap *mmap = evlist->mmap; - struct mmap *overwrite_mmap = evlist->overwrite_mmap; - struct perf_cpu_map *cpus = evlist->core.all_cpus; + int m, tm, nr_mmaps = evlist__core(evlist)->nr_mmaps; + struct mmap *mmap = evlist__mmap(evlist); + struct mmap *overwrite_mmap = evlist__overwrite_mmap(evlist); + struct perf_cpu_map *cpus = evlist__core(evlist)->all_cpus; bool per_thread = evlist__per_thread(evlist); if (per_thread) @@ -1116,16 +1118,17 @@ static int record__thread_data_init_pollfd(struct record_thread *thread_data, st overwrite_map = thread_data->overwrite_maps ? thread_data->overwrite_maps[tm] : NULL; - for (f = 0; f < evlist->core.pollfd.nr; f++) { - void *ptr = evlist->core.pollfd.priv[f].ptr; + for (f = 0; f < evlist__core(evlist)->pollfd.nr; f++) { + void *ptr = evlist__core(evlist)->pollfd.priv[f].ptr; if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) { pos = fdarray__dup_entry_from(&thread_data->pollfd, f, - &evlist->core.pollfd); + &evlist__core(evlist)->pollfd); if (pos < 0) return pos; pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n", - thread_data, pos, evlist->core.pollfd.entries[f].fd); + thread_data, pos, + evlist__core(evlist)->pollfd.entries[f].fd); } } } @@ -1169,7 +1172,7 @@ static int record__update_evlist_pollfd_from_thread(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct pollfd *e_entries = evlist->core.pollfd.entries; + struct pollfd *e_entries = evlist__core(evlist)->pollfd.entries; struct pollfd *t_entries = thread_data->pollfd.entries; int err = 0; size_t i; @@ -1193,7 +1196,7 @@ static int record__dup_non_perf_events(struct record *rec, struct evlist *evlist, struct record_thread *thread_data) { - struct fdarray *fda = &evlist->core.pollfd; + struct fdarray *fda = &evlist__core(evlist)->pollfd; int i, ret; for (i = 0; i < fda->nr; i++) { @@ -1320,17 +1323,17 @@ static int record__mmap_evlist(struct record *rec, return ret; if (record__threads_enabled(rec)) { - ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps); + ret = perf_data__create_dir(&rec->data, evlist__core(evlist)->nr_mmaps); if (ret) { errno = -ret; pr_err("Failed to create data directory: %m\n"); return ret; } - for (i = 0; i < evlist->core.nr_mmaps; i++) { - if (evlist->mmap) - evlist->mmap[i].file = &rec->data.dir.files[i]; - if (evlist->overwrite_mmap) - evlist->overwrite_mmap[i].file = &rec->data.dir.files[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + if (evlist__mmap(evlist)) + evlist__mmap(evlist)[i].file = &rec->data.dir.files[i]; + if (evlist__overwrite_mmap(evlist)) + evlist__overwrite_mmap(evlist)[i].file = &rec->data.dir.files[i]; } } @@ -1479,17 +1482,16 @@ static int record__open(struct record *rec) static void set_timestamp_boundary(struct record *rec, u64 sample_time) { - if (rec->evlist->first_sample_time == 0) - rec->evlist->first_sample_time = sample_time; + if (evlist__first_sample_time(rec->evlist) == 0) + evlist__set_first_sample_time(rec->evlist, sample_time); if (sample_time) - rec->evlist->last_sample_time = sample_time; + evlist__set_last_sample_time(rec->evlist, sample_time); } static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct record *rec = container_of(tool, struct record, tool); @@ -1500,7 +1502,7 @@ static int process_sample_event(const struct perf_tool *tool, return 0; rec->samples++; - return build_id__mark_dso_hit(tool, event, sample, evsel, machine); + return build_id__mark_dso_hit(tool, event, sample, machine); } static int process_buildids(struct record *rec) @@ -1652,7 +1654,7 @@ static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist, if (!maps) return 0; - if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING) + if (overwrite && evlist__bkw_mmap_state(evlist) != BKW_MMAP_DATA_PENDING) return 0; if (record__aio_enabled(rec)) @@ -1807,7 +1809,7 @@ static void record__init_features(struct record *rec) if (rec->no_buildid) perf_header__clear_feat(&session->header, HEADER_BUILD_ID); - if (!have_tracepoints(&rec->evlist->core.entries)) + if (!have_tracepoints(&evlist__core(rec->evlist)->entries)) perf_header__clear_feat(&session->header, HEADER_TRACING_DATA); if (!rec->opts.branch_stack) @@ -1873,7 +1875,7 @@ static int record__synthesize_workload(struct record *rec, bool tail) if (rec->opts.tail_synthesize != tail) return 0; - thread_map = thread_map__new_by_tid(rec->evlist->workload.pid); + thread_map = thread_map__new_by_tid(evlist__workload_pid(rec->evlist)); if (thread_map == NULL) return -1; @@ -2066,10 +2068,10 @@ static void alarm_sig_handler(int sig); static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist) { if (evlist) { - if (evlist->mmap && evlist->mmap[0].core.base) - return evlist->mmap[0].core.base; - if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base) - return evlist->overwrite_mmap[0].core.base; + if (evlist__mmap(evlist) && evlist__mmap(evlist)[0].core.base) + return evlist__mmap(evlist)[0].core.base; + if (evlist__overwrite_mmap(evlist) && evlist__overwrite_mmap(evlist)[0].core.base) + return evlist__overwrite_mmap(evlist)[0].core.base; } return NULL; } @@ -2149,7 +2151,7 @@ static int record__synthesize(struct record *rec, bool tail) if (err) goto out; - err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads, + err = perf_event__synthesize_thread_map2(&rec->tool, evlist__core(rec->evlist)->threads, process_synthesized_event, NULL); if (err < 0) { @@ -2157,7 +2159,7 @@ static int record__synthesize(struct record *rec, bool tail) return err; } - err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus, + err = perf_event__synthesize_cpu_map(&rec->tool, evlist__core(rec->evlist)->all_cpus, process_synthesized_event, NULL); if (err < 0) { pr_err("Couldn't synthesize cpu map.\n"); @@ -2190,7 +2192,7 @@ static int record__synthesize(struct record *rec, bool tail) bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP; err = __machine__synthesize_threads(machine, tool, &opts->target, - rec->evlist->core.threads, + evlist__core(rec->evlist)->threads, f, needs_mmap, opts->record_data_mmap, rec->opts.nr_threads_synthesize); } @@ -2543,7 +2545,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * because we synthesize event name through the pipe * and need the id for that. */ - if (data->is_pipe && rec->evlist->core.nr_entries == 1) + if (data->is_pipe && evlist__nr_entries(rec->evlist) == 1) rec->opts.sample_id = true; if (rec->timestamp_filename && perf_data__is_pipe(data)) { @@ -2567,7 +2569,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } /* Debug message used by test scripts */ pr_debug3("perf record done opening and mmapping events\n"); - env->comp_mmap_len = session->evlist->core.mmap_len; + env->comp_mmap_len = evlist__core(session->evlist)->mmap_len; if (rec->opts.kcore) { err = record__kcore_copy(&session->machines.host, data); @@ -2668,7 +2670,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize COMM event to prevent it. */ tgid = perf_event__synthesize_comm(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), process_synthesized_event, machine); free(event); @@ -2688,7 +2690,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * Synthesize NAMESPACES event for the command specified. */ perf_event__synthesize_namespaces(tool, event, - rec->evlist->workload.pid, + evlist__workload_pid(rec->evlist), tgid, process_synthesized_event, machine); free(event); @@ -2705,7 +2707,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__start(rec->evlist->eet); + err = event_enable_timer__start(evlist__event_enable_timer(rec->evlist)); if (err) goto out_child; @@ -2743,7 +2745,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) trigger_error(&auxtrace_snapshot_trigger); trigger_error(&switch_output_trigger); err = -1; - goto out_child; + goto out_child_no_flush; } if (auxtrace_record__snapshot_started) { @@ -2767,7 +2769,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) * record__mmap_read_all() didn't collect data from * overwritable ring buffer. Read again. */ - if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING) + if (evlist__bkw_mmap_state(rec->evlist) == BKW_MMAP_RUNNING) continue; trigger_ready(&switch_output_trigger); @@ -2836,7 +2838,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } - err = event_enable_timer__process(rec->evlist->eet); + err = event_enable_timer__process(evlist__event_enable_timer(rec->evlist)); if (err < 0) goto out_child; if (err) { @@ -2890,6 +2892,10 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) out_child: record__stop_threads(rec); record__mmap_read_all(rec, true); + goto out_free_threads; +out_child_no_flush: + /* mmap read already failed — retrying would just fail again */ + record__stop_threads(rec); out_free_threads: record__free_thread_data(rec); evlist__finalize_ctlfd(rec->evlist); @@ -2904,7 +2910,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) int exit_status; if (!child_finished) - kill(rec->evlist->workload.pid, SIGTERM); + kill(evlist__workload_pid(rec->evlist), SIGTERM); wait(&exit_status); @@ -3079,6 +3085,7 @@ static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits) static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask) { bitmap_free(mask->bits); + mask->bits = NULL; mask->nbits = 0; } @@ -3230,10 +3237,6 @@ static int record__parse_off_cpu_thresh(const struct option *opt, return 0; } -void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused) -{ -} - static int parse_control_option(const struct option *opt, const char *str, int unset __maybe_unused) @@ -4030,7 +4033,7 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu static int record__init_thread_masks(struct record *rec) { int ret = 0; - struct perf_cpu_map *cpus = rec->evlist->core.all_cpus; + struct perf_cpu_map *cpus = evlist__core(rec->evlist)->all_cpus; if (!record__threads_enabled(rec)) return record__init_thread_default_masks(rec, cpus); @@ -4281,15 +4284,15 @@ int cmd_record(int argc, const char **argv) if (record.opts.overwrite) record.opts.tail_synthesize = true; - if (rec->evlist->core.nr_entries == 0) { + if (evlist__nr_entries(rec->evlist) == 0) { struct evlist *def_evlist = evlist__new_default(&rec->opts.target, callchain_param.enabled); if (!def_evlist) goto out; - evlist__splice_list_tail(rec->evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(rec->evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } if (rec->opts.target.tid && !rec->opts.no_inherit_set) @@ -4319,8 +4322,10 @@ int cmd_record(int argc, const char **argv) evlist__warn_user_requested_cpus(rec->evlist, rec->opts.target.cpu_list); - if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP) - arch__add_leaf_frame_record_opts(&rec->opts); + if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP) { + if (EM_HOST == EM_AARCH64) + add_leaf_frame_caller_opts_aarch64(&rec->opts); + } err = -ENOMEM; if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0) { @@ -4397,7 +4402,7 @@ int cmd_record(int argc, const char **argv) auxtrace_record__free(rec->itr); out_opts: evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close); - evlist__delete(rec->evlist); + evlist__put(rec->evlist); return err; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 95c0bdba6b11..10db1e5f1e6c 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -27,6 +27,7 @@ #include "perf.h" #include "util/debug.h" +#include "util/event.h" #include "util/evlist.h" #include "util/evsel.h" #include "util/evswitch.h" @@ -48,6 +49,7 @@ #include "util/time-utils.h" #include "util/auxtrace.h" #include "util/units.h" +#include "util/unwind.h" #include "util/util.h" // perf_tip() #include "ui/ui.h" #include "ui/progress.h" @@ -170,7 +172,6 @@ static int hist_iter__report_callback(struct hist_entry_iter *iter, int err = 0; struct report *rep = arg; struct hist_entry *he = iter->he; - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; struct mem_info *mi; struct branch_info *bi; @@ -180,25 +181,25 @@ static int hist_iter__report_callback(struct hist_entry_iter *iter, if (sort__mode == SORT_MODE__BRANCH) { bi = he->branch_info; - err = addr_map_symbol__inc_samples(&bi->from, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->from, sample); if (err) goto out; - err = addr_map_symbol__inc_samples(&bi->to, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->to, sample); } else if (rep->mem_mode) { mi = he->mem_info; - err = addr_map_symbol__inc_samples(mem_info__daddr(mi), sample, evsel); + err = addr_map_symbol__inc_samples(mem_info__daddr(mi), sample); if (err) goto out; - err = hist_entry__inc_addr_samples(he, sample, evsel, al->addr); + err = hist_entry__inc_addr_samples(he, sample, al->addr); } else if (symbol_conf.cumulate_callchain) { if (single) - err = hist_entry__inc_addr_samples(he, sample, evsel, al->addr); + err = hist_entry__inc_addr_samples(he, sample, al->addr); } else { - err = hist_entry__inc_addr_samples(he, sample, evsel, al->addr); + err = hist_entry__inc_addr_samples(he, sample, al->addr); } out: @@ -214,7 +215,6 @@ static int hist_iter__branch_callback(struct hist_entry_iter *iter, struct report *rep = arg; struct branch_info *bi = he->branch_info; struct perf_sample *sample = iter->sample; - struct evsel *evsel = iter->evsel; int err; branch_type_count(&rep->brtype_stat, &bi->flags, @@ -223,11 +223,11 @@ static int hist_iter__branch_callback(struct hist_entry_iter *iter, if (!ui__has_annotation() && !rep->symbol_ipc) return 0; - err = addr_map_symbol__inc_samples(&bi->from, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->from, sample); if (err) goto out; - err = addr_map_symbol__inc_samples(&bi->to, sample, evsel); + err = addr_map_symbol__inc_samples(&bi->to, sample); out: return err; @@ -264,13 +264,11 @@ static int process_feature_event(const struct perf_tool *tool, static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct report *rep = container_of(tool, struct report, tool); struct addr_location al; struct hist_entry_iter iter = { - .evsel = evsel, .sample = sample, .hide_unresolved = symbol_conf.hide_unresolved, .add_entry_cb = hist_iter__report_callback, @@ -282,13 +280,14 @@ static int process_sample_event(const struct perf_tool *tool, return 0; } - if (evswitch__discard(&rep->evswitch, evsel)) + if (evswitch__discard(&rep->evswitch, sample->evsel)) return 0; addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_debug("problem processing %d event, skipping it.\n", - event->header.type); + pr_debug("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out_put; } @@ -299,7 +298,8 @@ static int process_sample_event(const struct perf_tool *tool, if (symbol_conf.hide_unresolved && al.sym == NULL) goto out_put; - if (rep->cpu_list && !test_bit(sample->cpu, rep->cpu_bitmap)) + if (rep->cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, rep->cpu_bitmap))) goto out_put; if (sort__mode == SORT_MODE__BRANCH) { @@ -326,7 +326,7 @@ static int process_sample_event(const struct perf_tool *tool, if (ui__has_annotation() || rep->symbol_ipc || rep->total_cycles_mode) { hist__account_cycles(sample->branch_stack, &al, sample, rep->nonany_branch_mode, - &rep->total_cycles, evsel); + &rep->total_cycles); } rep->total_samples++; @@ -335,7 +335,8 @@ static int process_sample_event(const struct perf_tool *tool, ret = hist_entry_iter__add(&iter, &al, rep->max_stack, rep); if (ret < 0) - pr_debug("problem adding hist entry, skipping event\n"); + pr_debug("problem adding hist entry at offset %#" PRIx64 ", skipping event\n", + sample->file_offset); out_put: addr_location__exit(&al); return ret; @@ -344,7 +345,6 @@ static int process_sample_event(const struct perf_tool *tool, static int process_read_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample __maybe_unused, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct report *rep = container_of(tool, struct report, tool); @@ -352,7 +352,7 @@ static int process_read_event(const struct perf_tool *tool, if (rep->show_threads) { int err = perf_read_values_add_value(&rep->show_threads_values, event->read.pid, event->read.tid, - evsel, + sample->evsel, event->read.value); if (err) @@ -561,7 +561,7 @@ static int evlist__tty_browse_hists(struct evlist *evlist, struct report *rep, c if (!quiet) { fprintf(stdout, "#\n# Total Lost Samples: %" PRIu64 "\n#\n", - evlist->stats.total_lost_samples); + evlist__stats(evlist)->total_lost_samples); } evlist__for_each_entry(evlist, pos) { @@ -753,7 +753,7 @@ static int hists__resort_cb(struct hist_entry *he, void *arg) struct report *rep = arg; struct symbol *sym = he->ms.sym; - if (rep->symbol_ipc && sym && !sym->annotate2) { + if (rep->symbol_ipc && sym && !symbol__is_annotate2(sym)) { struct evsel *evsel = hists_to_evsel(he->hists); symbol__annotate2(&he->ms, evsel, NULL); @@ -778,11 +778,10 @@ static void report__output_resort(struct report *rep) static int count_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, - struct perf_sample *sample __maybe_unused, - struct evsel *evsel, + struct perf_sample *sample, struct machine *machine __maybe_unused) { - struct hists *hists = evsel__hists(evsel); + struct hists *hists = evsel__hists(sample->evsel); hists__inc_nr_events(hists); return 0; @@ -794,9 +793,11 @@ static int count_lost_samples_event(const struct perf_tool *tool, struct machine *machine __maybe_unused) { struct report *rep = container_of(tool, struct report, tool); - struct evsel *evsel; + struct evsel *evsel = sample->evsel; + + if (!evsel) + evsel = evlist__id2evsel(rep->session->evlist, sample->id); - evsel = evlist__id2evsel(rep->session->evlist, sample->id); if (evsel) { struct hists *hists = evsel__hists(evsel); u32 count = event->lost_samples.lost; @@ -1155,7 +1156,7 @@ static int __cmd_report(struct report *rep) PERF_HPP_REPORT__BLOCK_AVG_CYCLES, }; - if (session->evlist->nr_br_cntr > 0) + if (evlist__nr_br_cntr(session->evlist) > 0) block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER; block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_RANGE; @@ -1290,7 +1291,7 @@ static int process_attr(const struct perf_tool *tool __maybe_unused, * on events sample_type. */ sample_type = evlist__combined_sample_type(*pevlist); - session = (*pevlist)->session; + session = evlist__session(*pevlist); callchain_param_setup(sample_type, perf_session__e_machine(session, /*e_flags=*/NULL)); return 0; } @@ -1449,6 +1450,9 @@ int cmd_report(int argc, const char **argv) OPT_CALLBACK(0, "addr2line-style", NULL, "addr2line style", "addr2line styles (libdw,llvm,libbfd,addr2line)", report_parse_addr2line_config), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_BOOLEAN(0, "demangle", &symbol_conf.demangle, "Symbol demangling. Enabled by default, use --no-demangle to disable."), OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 555247568e7a..6e0d8f4d270e 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,7 @@ #define COMM_LEN 20 #define SYM_LEN 129 #define MAX_PID 1024000 +#define PID_MAX_LIMIT 4194304 /* kernel limit on 64-bit */ #define MAX_PRIO 140 #define SEP_LEN 100 @@ -129,21 +131,20 @@ typedef int (*sort_fn_t)(struct work_atoms *, struct work_atoms *); struct perf_sched; struct trace_sched_handler { - int (*switch_event)(struct perf_sched *sched, struct evsel *evsel, - struct perf_sample *sample, struct machine *machine); + int (*switch_event)(struct perf_sched *sched, struct perf_sample *sample, + struct machine *machine); - int (*runtime_event)(struct perf_sched *sched, struct evsel *evsel, - struct perf_sample *sample, struct machine *machine); + int (*runtime_event)(struct perf_sched *sched, struct perf_sample *sample, + struct machine *machine); - int (*wakeup_event)(struct perf_sched *sched, struct evsel *evsel, - struct perf_sample *sample, struct machine *machine); + int (*wakeup_event)(struct perf_sched *sched, struct perf_sample *sample, + struct machine *machine); /* PERF_RECORD_FORK event, not sched_process_fork tracepoint */ int (*fork_event)(struct perf_sched *sched, union perf_event *event, struct machine *machine); int (*migrate_task_event)(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); }; @@ -273,6 +274,7 @@ struct thread_runtime { u64 migrations; int prio; + bool color; }; /* per event run time data */ @@ -364,14 +366,25 @@ get_new_event(struct task_desc *task, u64 timestamp) struct sched_atom *event = zalloc(sizeof(*event)); unsigned long idx = task->nr_events; size_t size; + struct sched_atom **atoms_p; + + if (event == NULL) { + pr_err("ERROR: sched: failed to allocate event\n"); + return NULL; + } event->timestamp = timestamp; event->nr = idx; + size = sizeof(struct sched_atom *) * (task->nr_events + 1); + atoms_p = realloc(task->atoms, size); + if (!atoms_p) { + pr_err("ERROR: sched: failed to grow atoms array\n"); + free(event); + return NULL; + } + task->atoms = atoms_p; task->nr_events++; - size = sizeof(struct sched_atom *) * task->nr_events; - task->atoms = realloc(task->atoms, size); - BUG_ON(!task->atoms); task->atoms[idx] = event; @@ -402,6 +415,8 @@ static void add_sched_event_run(struct perf_sched *sched, struct task_desc *task } event = get_new_event(task, timestamp); + if (event == NULL) + return; event->type = SCHED_EVENT_RUN; event->duration = duration; @@ -415,6 +430,8 @@ static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *t struct sched_atom *event, *wakee_event; event = get_new_event(task, timestamp); + if (event == NULL) + return; event->type = SCHED_EVENT_WAKEUP; event->wakee = wakee; @@ -429,6 +446,10 @@ static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *t } wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem)); + if (!wakee_event->wait_sem) { + pr_err("ERROR: sched: failed to allocate semaphore\n"); + return; + } sem_init(wakee_event->wait_sem, 0, 0); event->wait_sem = wakee_event->wait_sem; @@ -440,6 +461,9 @@ static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *ta { struct sched_atom *event = get_new_event(task, timestamp); + if (event == NULL) + return; + event->type = SCHED_EVENT_SLEEP; sched->nr_sleep_events++; @@ -448,17 +472,28 @@ static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *ta static struct task_desc *register_pid(struct perf_sched *sched, unsigned long pid, const char *comm) { - struct task_desc *task; + struct task_desc *task, **tasks_p; static int pid_max; + /* perf.data is untrusted — cap pid to prevent overflow in size calculations */ + if (pid >= PID_MAX_LIMIT) { + pr_err("pid %lu exceeds limit %d, skipping\n", pid, PID_MAX_LIMIT); + return NULL; + } + if (sched->pid_to_task == NULL) { if (sysctl__read_int("kernel/pid_max", &pid_max) < 0) pid_max = MAX_PID; - BUG_ON((sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *))) == NULL); + sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *)); + if (sched->pid_to_task == NULL) + return NULL; } if (pid >= (unsigned long)pid_max) { - BUG_ON((sched->pid_to_task = realloc(sched->pid_to_task, (pid + 1) * - sizeof(struct task_desc *))) == NULL); + void *p = realloc(sched->pid_to_task, (pid + 1) * sizeof(struct task_desc *)); + + if (p == NULL) + return NULL; + sched->pid_to_task = p; while (pid >= (unsigned long)pid_max) sched->pid_to_task[pid_max++] = NULL; } @@ -469,9 +504,11 @@ static struct task_desc *register_pid(struct perf_sched *sched, return task; task = zalloc(sizeof(*task)); + if (task == NULL) + return NULL; task->pid = pid; - task->nr = sched->nr_tasks; - strcpy(task->comm, comm); + if (comm) + strlcpy(task->comm, comm, sizeof(task->comm)); /* * every task starts in sleeping state - this gets ignored * if there's no wakeup pointing to this sleep state: @@ -479,10 +516,12 @@ static struct task_desc *register_pid(struct perf_sched *sched, add_sched_event_sleep(sched, task, 0); sched->pid_to_task[pid] = task; - sched->nr_tasks++; - sched->tasks = realloc(sched->tasks, sched->nr_tasks * sizeof(struct task_desc *)); - BUG_ON(!sched->tasks); - sched->tasks[task->nr] = task; + tasks_p = realloc(sched->tasks, (sched->nr_tasks + 1) * sizeof(struct task_desc *)); + if (!tasks_p) + return NULL; + sched->tasks = tasks_p; + sched->tasks[sched->nr_tasks] = task; + task->nr = sched->nr_tasks++; if (verbose > 0) printf("registered task #%ld, PID %ld (%s)\n", sched->nr_tasks, pid, comm); @@ -826,42 +865,43 @@ static void test_calibrations(struct perf_sched *sched) static int replay_wakeup_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, + struct perf_sample *sample, struct machine *machine __maybe_unused) { - const char *comm = evsel__strval(evsel, sample, "comm"); - const u32 pid = evsel__intval(evsel, sample, "pid"); + const char *comm = perf_sample__strval(sample, "comm"); + const u32 pid = perf_sample__intval(sample, "pid"); struct task_desc *waker, *wakee; if (verbose > 0) { - printf("sched_wakeup event %p\n", evsel); + printf("sched_wakeup event %p\n", sample->evsel); printf(" ... pid %d woke up %s/%d\n", sample->tid, comm, pid); } waker = register_pid(sched, sample->tid, ""); wakee = register_pid(sched, pid, comm); + if (waker == NULL || wakee == NULL) + return -1; add_sched_event_wakeup(sched, waker, sample->time, wakee); return 0; } static int replay_switch_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) { - const char *prev_comm = evsel__strval(evsel, sample, "prev_comm"), - *next_comm = evsel__strval(evsel, sample, "next_comm"); - const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"), - next_pid = evsel__intval(evsel, sample, "next_pid"); + const char *prev_comm = perf_sample__strval(sample, "prev_comm"), + *next_comm = perf_sample__strval(sample, "next_comm"); + const u32 prev_pid = perf_sample__intval(sample, "prev_pid"), + next_pid = perf_sample__intval(sample, "next_pid"); struct task_desc *prev, __maybe_unused *next; u64 timestamp0, timestamp = sample->time; int cpu = sample->cpu; s64 delta; if (verbose > 0) - printf("sched_switch event %p\n", evsel); + printf("sched_switch event %p\n", sample->evsel); if (cpu >= MAX_CPUS || cpu < 0) return 0; @@ -882,6 +922,8 @@ static int replay_switch_event(struct perf_sched *sched, prev = register_pid(sched, prev_pid, prev_comm); next = register_pid(sched, next_pid, next_comm); + if (prev == NULL || next == NULL) + return -1; sched->cpu_last_switched[cpu] = timestamp; @@ -1134,20 +1176,24 @@ static void free_work_atoms(struct work_atoms *atoms) } static int latency_switch_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { - const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"), - next_pid = evsel__intval(evsel, sample, "next_pid"); - const char prev_state = evsel__taskstate(evsel, sample, "prev_state"); + const u32 prev_pid = perf_sample__intval(sample, "prev_pid"), + next_pid = perf_sample__intval(sample, "next_pid"); + const char prev_state = perf_sample__taskstate(sample, "prev_state"); struct work_atoms *out_events, *in_events; struct thread *sched_out, *sched_in; u64 timestamp0, timestamp = sample->time; int cpu = sample->cpu, err = -1; s64 delta; - BUG_ON(cpu >= MAX_CPUS || cpu < 0); + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (cpu >= MAX_CPUS || cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, cpu); + return 0; + } timestamp0 = sched->cpu_last_switched[cpu]; sched->cpu_last_switched[cpu] = timestamp; @@ -1177,7 +1223,7 @@ static int latency_switch_event(struct perf_sched *sched, } } if (add_sched_out_event(out_events, prev_state, timestamp)) - return -1; + goto out_put; in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid); if (!in_events) { @@ -1204,21 +1250,28 @@ static int latency_switch_event(struct perf_sched *sched, } static int latency_runtime_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { - const u32 pid = evsel__intval(evsel, sample, "pid"); - const u64 runtime = evsel__intval(evsel, sample, "runtime"); + const u32 pid = perf_sample__intval(sample, "pid"); + const u64 runtime = perf_sample__intval(sample, "runtime"); struct thread *thread = machine__findnew_thread(machine, -1, pid); - struct work_atoms *atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid); + struct work_atoms *atoms; u64 timestamp = sample->time; int cpu = sample->cpu, err = -1; if (thread == NULL) return -1; - BUG_ON(cpu >= MAX_CPUS || cpu < 0); + atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid); + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (cpu >= MAX_CPUS || cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, cpu); + err = 0; + goto out_put; + } if (!atoms) { if (thread_atoms_insert(sched, thread)) goto out_put; @@ -1239,11 +1292,10 @@ static int latency_runtime_event(struct perf_sched *sched, } static int latency_wakeup_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { - const u32 pid = evsel__intval(evsel, sample, "pid"); + const u32 pid = perf_sample__intval(sample, "pid"); struct work_atoms *atoms; struct work_atom *atom; struct thread *wakee; @@ -1300,11 +1352,10 @@ static int latency_wakeup_event(struct perf_sched *sched, } static int latency_migrate_task_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { - const u32 pid = evsel__intval(evsel, sample, "pid"); + const u32 pid = perf_sample__intval(sample, "pid"); u64 timestamp = sample->time; struct work_atoms *atoms; struct work_atom *atom; @@ -1519,20 +1570,18 @@ static void perf_sched__sort_lat(struct perf_sched *sched) } static int process_sched_wakeup_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { struct perf_sched *sched = container_of(tool, struct perf_sched, tool); if (sched->tp_handler->wakeup_event) - return sched->tp_handler->wakeup_event(sched, evsel, sample, machine); + return sched->tp_handler->wakeup_event(sched, sample, machine); return 0; } static int process_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unused, - struct evsel *evsel __maybe_unused, struct perf_sample *sample __maybe_unused, struct machine *machine __maybe_unused) { @@ -1541,22 +1590,32 @@ static int process_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unus static bool thread__has_color(struct thread *thread) { - return thread__priv(thread) != NULL; + struct thread_runtime *tr = thread__priv(thread); + + return tr != NULL && tr->color; } static struct thread* map__findnew_thread(struct perf_sched *sched, struct machine *machine, pid_t pid, pid_t tid) { struct thread *thread = machine__findnew_thread(machine, pid, tid); - bool color = false; - if (!sched->map.color_pids || !thread || thread__priv(thread)) + if (!sched->map.color_pids || !thread) return thread; - if (thread_map__has(sched->map.color_pids, tid)) - color = true; + /* + * Always check the color-pids map, even if thread__priv() is + * already set. COMM events processed before the first sched_switch + * allocate a thread_runtime via thread__get_runtime(), so priv is + * non-NULL before we ever get here. Skipping the check on non-NULL + * priv would prevent those threads from being colored. + */ + if (thread_map__has(sched->map.color_pids, tid)) { + struct thread_runtime *tr = thread__get_runtime(thread); - thread__set_priv(thread, color ? ((void*)1) : NULL); + if (tr) + tr->color = true; + } return thread; } @@ -1626,11 +1685,11 @@ static void print_sched_map(struct perf_sched *sched, struct perf_cpu this_cpu, } } -static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, - struct perf_sample *sample, struct machine *machine) +static int map_switch_event(struct perf_sched *sched, struct perf_sample *sample, + struct machine *machine) { - const u32 next_pid = evsel__intval(evsel, sample, "next_pid"); - const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"); + const u32 next_pid = perf_sample__intval(sample, "next_pid"); + const u32 prev_pid = perf_sample__intval(sample, "prev_pid"); struct thread *sched_in, *sched_out; struct thread_runtime *tr; int new_shortname; @@ -1647,7 +1706,12 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, const char *str; int ret = -1; - BUG_ON(this_cpu.cpu >= MAX_CPUS || this_cpu.cpu < 0); + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (this_cpu.cpu >= MAX_CPUS || this_cpu.cpu < 0) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, this_cpu.cpu); + return 0; + } if (this_cpu.cpu > sched->max_cpu.cpu) sched->max_cpu = this_cpu; @@ -1659,7 +1723,7 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, new_cpu = true; } } else - cpus_nr = sched->max_cpu.cpu; + cpus_nr = sched->max_cpu.cpu + 1; timestamp0 = sched->cpu_last_switched[this_cpu.cpu]; sched->cpu_last_switched[this_cpu.cpu] = timestamp; @@ -1791,14 +1855,20 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, } static int process_sched_switch_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { struct perf_sched *sched = container_of(tool, struct perf_sched, tool); int this_cpu = sample->cpu, err = 0; - u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"), - next_pid = evsel__intval(evsel, sample, "next_pid"); + u32 prev_pid = perf_sample__intval(sample, "prev_pid"), + next_pid = perf_sample__intval(sample, "next_pid"); + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (this_cpu < 0 || this_cpu >= MAX_CPUS) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, this_cpu); + return 0; + } if (sched->curr_pid[this_cpu] != (u32)-1) { /* @@ -1810,21 +1880,27 @@ static int process_sched_switch_event(const struct perf_tool *tool, } if (sched->tp_handler->switch_event) - err = sched->tp_handler->switch_event(sched, evsel, sample, machine); + err = sched->tp_handler->switch_event(sched, sample, machine); sched->curr_pid[this_cpu] = next_pid; return err; } static int process_sched_runtime_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { struct perf_sched *sched = container_of(tool, struct perf_sched, tool); + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (sample->cpu >= MAX_CPUS) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %u, skipping sample\n", + sample->file_offset, sample->cpu); + return 0; + } + if (sched->tp_handler->runtime_event) - return sched->tp_handler->runtime_event(sched, evsel, sample, machine); + return sched->tp_handler->runtime_event(sched, sample, machine); return 0; } @@ -1847,34 +1923,32 @@ static int perf_sched__process_fork_event(const struct perf_tool *tool, } static int process_sched_migrate_task_event(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { struct perf_sched *sched = container_of(tool, struct perf_sched, tool); if (sched->tp_handler->migrate_task_event) - return sched->tp_handler->migrate_task_event(sched, evsel, sample, machine); + return sched->tp_handler->migrate_task_event(sched, sample, machine); return 0; } typedef int (*tracepoint_handler)(const struct perf_tool *tool, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); static int perf_sched__process_tracepoint_sample(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { + struct evsel *evsel = sample->evsel; int err = 0; if (evsel->handler != NULL) { tracepoint_handler f = evsel->handler; - err = f(tool, evsel, sample, machine); + err = f(tool, sample, machine); } return err; @@ -1951,9 +2025,9 @@ static int perf_sched__read_events(struct perf_sched *sched) goto out_delete; } - sched->nr_events = session->evlist->stats.nr_events[0]; - sched->nr_lost_events = session->evlist->stats.total_lost; - sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST]; + sched->nr_events = evlist__stats(session->evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(session->evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST]; } rc = 0; @@ -2068,12 +2142,11 @@ static char *timehist_get_commstr(struct thread *thread) /* prio field format: xxx or xxx->yyy */ #define MAX_PRIO_STR_LEN 8 -static char *timehist_get_priostr(struct evsel *evsel, - struct thread *thread, +static char *timehist_get_priostr(struct thread *thread, struct perf_sample *sample) { static char prio_str[16]; - int prev_prio = (int)evsel__intval(evsel, sample, "prev_prio"); + int prev_prio = (int)perf_sample__intval(sample, "prev_prio"); struct thread_runtime *tr = thread__priv(thread); if (tr->prio != prev_prio && tr->prio != -1) @@ -2161,21 +2234,21 @@ static void timehist_header(struct perf_sched *sched) } static void timehist_print_sample(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct addr_location *al, struct thread *thread, u64 t, const char state) { struct thread_runtime *tr = thread__priv(thread); - const char *next_comm = evsel__strval(evsel, sample, "next_comm"); - const u32 next_pid = evsel__intval(evsel, sample, "next_pid"); + const char *next_comm = perf_sample__strval(sample, "next_comm"); + const u32 next_pid = perf_sample__intval(sample, "next_pid"); u32 max_cpus = sched->max_cpu.cpu + 1; char tstr[64]; char nstr[30]; u64 wait_time; - if (cpu_list && !test_bit(sample->cpu, cpu_bitmap)) + if (cpu_list && (sample->cpu >= MAX_NR_CPUS || + !test_bit(sample->cpu, cpu_bitmap))) return; timestamp__scnprintf_usec(t, tstr, sizeof(tstr)); @@ -2197,15 +2270,10 @@ static void timehist_print_sample(struct perf_sched *sched, printf(" "); } - if (!thread__comm_set(thread)) { - const char *prev_comm = evsel__strval(evsel, sample, "prev_comm"); - thread__set_comm(thread, prev_comm, sample->time); - } - printf(" %-*s ", comm_width, timehist_get_commstr(thread)); if (sched->show_prio) - printf(" %-*s ", MAX_PRIO_STR_LEN, timehist_get_priostr(evsel, thread, sample)); + printf(" %-*s ", MAX_PRIO_STR_LEN, timehist_get_priostr(thread, sample)); wait_time = tr->dt_sleep + tr->dt_iowait + tr->dt_preempt; print_sched_time(wait_time, 6); @@ -2314,19 +2382,17 @@ static void timehist_update_runtime_stats(struct thread_runtime *r, r->total_pre_mig_time += r->dt_pre_mig; } -static bool is_idle_sample(struct perf_sample *sample, - struct evsel *evsel) +static bool is_idle_sample(struct perf_sample *sample) { /* pid 0 == swapper == idle task */ - if (evsel__name_is(evsel, "sched:sched_switch")) - return evsel__intval(evsel, sample, "prev_pid") == 0; + if (evsel__name_is(sample->evsel, "sched:sched_switch")) + return perf_sample__intval(sample, "prev_pid") == 0; return sample->pid == 0; } static void save_task_callchain(struct perf_sched *sched, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct callchain_cursor *cursor; @@ -2346,7 +2412,7 @@ static void save_task_callchain(struct perf_sched *sched, cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(thread, cursor, evsel, sample, + if (thread__resolve_callchain(thread, cursor, sample, NULL, NULL, sched->max_stack + 2) != 0) { if (verbose > 0) pr_err("Failed to resolve callchain. Skipping\n"); @@ -2371,7 +2437,7 @@ static void save_task_callchain(struct perf_sched *sched, if (!strcmp(sym->name, "schedule") || !strcmp(sym->name, "__schedule") || !strcmp(sym->name, "preempt_schedule")) - sym->ignore = 1; + symbol__set_ignore(sym, true); } callchain_cursor_advance(cursor); @@ -2439,10 +2505,13 @@ static void free_idle_threads(void) struct idle_thread_runtime *itr; itr = thread__priv(idle); - if (itr) + if (itr) { thread__put(itr->last_thread); + free_callchain(&itr->callchain); + callchain_cursor_cleanup(&itr->cursor); + } - thread__delete(idle); + thread__put(idle); } } @@ -2475,8 +2544,11 @@ static struct thread *get_idle_thread(int cpu) idle_threads[cpu] = thread__new(0, 0); if (idle_threads[cpu]) { - if (init_idle_thread(idle_threads[cpu]) < 0) + if (init_idle_thread(idle_threads[cpu]) < 0) { + /* clean up so next call doesn't find a half-initialized thread */ + thread__zput(idle_threads[cpu]); return NULL; + } } } @@ -2501,12 +2573,11 @@ static void save_idle_callchain(struct perf_sched *sched, static struct thread *timehist_get_thread(struct perf_sched *sched, struct perf_sample *sample, - struct machine *machine, - struct evsel *evsel) + struct machine *machine) { struct thread *thread; - if (is_idle_sample(sample, evsel)) { + if (is_idle_sample(sample)) { thread = get_idle_thread(sample->cpu); if (thread == NULL) pr_err("Failed to get idle thread for cpu %d.\n", sample->cpu); @@ -2520,7 +2591,7 @@ static struct thread *timehist_get_thread(struct perf_sched *sched, sample->tid); } - save_task_callchain(sched, sample, evsel, machine); + save_task_callchain(sched, sample, machine); if (sched->idle_hist) { struct thread *idle; struct idle_thread_runtime *itr; @@ -2528,19 +2599,25 @@ static struct thread *timehist_get_thread(struct perf_sched *sched, idle = get_idle_thread(sample->cpu); if (idle == NULL) { pr_err("Failed to get idle thread for cpu %d.\n", sample->cpu); + thread__put(thread); return NULL; } itr = thread__priv(idle); - if (itr == NULL) + if (itr == NULL) { + thread__put(idle); + thread__put(thread); return NULL; + } thread__put(itr->last_thread); itr->last_thread = thread__get(thread); /* copy task callchain when entering to idle */ - if (evsel__intval(evsel, sample, "next_pid") == 0) + if (perf_sample__intval(sample, "next_pid") == 0) save_idle_callchain(sched, itr, sample); + + thread__put(idle); } } @@ -2549,7 +2626,6 @@ static struct thread *timehist_get_thread(struct perf_sched *sched, static bool timehist_skip_sample(struct perf_sched *sched, struct thread *thread, - struct evsel *evsel, struct perf_sample *sample) { bool rc = false; @@ -2571,20 +2647,22 @@ static bool timehist_skip_sample(struct perf_sched *sched, tr = thread__get_runtime(thread); if (tr && tr->prio != -1) prio = tr->prio; - else if (evsel__name_is(evsel, "sched:sched_switch")) - prio = evsel__intval(evsel, sample, "prev_prio"); + else if (evsel__name_is(sample->evsel, "sched:sched_switch")) + prio = perf_sample__intval(sample, "prev_prio"); - if (prio != -1 && !test_bit(prio, sched->prio_bitmap)) { + /* negative prio means no info; out-of-range prio can't match the filter */ + if (prio >= 0 && + (prio >= MAX_PRIO || !test_bit(prio, sched->prio_bitmap))) { rc = true; sched->skipped_samples++; } } if (sched->idle_hist) { - if (!evsel__name_is(evsel, "sched:sched_switch")) + if (!evsel__name_is(sample->evsel, "sched:sched_switch")) rc = true; - else if (evsel__intval(evsel, sample, "prev_pid") != 0 && - evsel__intval(evsel, sample, "next_pid") != 0) + else if (perf_sample__intval(sample, "prev_pid") != 0 && + perf_sample__intval(sample, "next_pid") != 0) rc = true; } @@ -2592,7 +2670,6 @@ static bool timehist_skip_sample(struct perf_sched *sched, } static void timehist_print_wakeup_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine, struct thread *awakened) @@ -2605,8 +2682,8 @@ static void timehist_print_wakeup_event(struct perf_sched *sched, return; /* show wakeup unless both awakee and awaker are filtered */ - if (timehist_skip_sample(sched, thread, evsel, sample) && - timehist_skip_sample(sched, awakened, evsel, sample)) { + if (timehist_skip_sample(sched, thread, sample) && + timehist_skip_sample(sched, awakened, sample)) { thread__put(thread); return; } @@ -2630,7 +2707,6 @@ static void timehist_print_wakeup_event(struct perf_sched *sched, static int timehist_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, - struct evsel *evsel __maybe_unused, struct perf_sample *sample __maybe_unused, struct machine *machine __maybe_unused) { @@ -2639,7 +2715,6 @@ static int timehist_sched_wakeup_ignore(const struct perf_tool *tool __maybe_unu static int timehist_sched_wakeup_event(const struct perf_tool *tool, union perf_event *event __maybe_unused, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -2647,7 +2722,7 @@ static int timehist_sched_wakeup_event(const struct perf_tool *tool, struct thread *thread; struct thread_runtime *tr = NULL; /* want pid of awakened task not pid in sample */ - const u32 pid = evsel__intval(evsel, sample, "pid"); + const u32 pid = perf_sample__intval(sample, "pid"); thread = machine__findnew_thread(machine, 0, pid); if (thread == NULL) @@ -2665,14 +2740,13 @@ static int timehist_sched_wakeup_event(const struct perf_tool *tool, /* show wakeups if requested */ if (sched->show_wakeups && !perf_time__skip_sample(&sched->ptime, sample->time)) - timehist_print_wakeup_event(sched, evsel, sample, machine, thread); + timehist_print_wakeup_event(sched, sample, machine, thread); thread__put(thread); return 0; } static void timehist_print_migration_event(struct perf_sched *sched, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine, struct thread *migrated) @@ -2686,15 +2760,15 @@ static void timehist_print_migration_event(struct perf_sched *sched, return; max_cpus = sched->max_cpu.cpu + 1; - ocpu = evsel__intval(evsel, sample, "orig_cpu"); - dcpu = evsel__intval(evsel, sample, "dest_cpu"); + ocpu = perf_sample__intval(sample, "orig_cpu"); + dcpu = perf_sample__intval(sample, "dest_cpu"); thread = machine__findnew_thread(machine, sample->pid, sample->tid); if (thread == NULL) return; - if (timehist_skip_sample(sched, thread, evsel, sample) && - timehist_skip_sample(sched, migrated, evsel, sample)) { + if (timehist_skip_sample(sched, thread, sample) && + timehist_skip_sample(sched, migrated, sample)) { thread__put(thread); return; } @@ -2728,7 +2802,6 @@ static void timehist_print_migration_event(struct perf_sched *sched, static int timehist_migrate_task_event(const struct perf_tool *tool, union perf_event *event __maybe_unused, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -2736,7 +2809,7 @@ static int timehist_migrate_task_event(const struct perf_tool *tool, struct thread *thread; struct thread_runtime *tr = NULL; /* want pid of migrated task not pid in sample */ - const u32 pid = evsel__intval(evsel, sample, "pid"); + const u32 pid = perf_sample__intval(sample, "pid"); thread = machine__findnew_thread(machine, 0, pid); if (thread == NULL) @@ -2753,22 +2826,20 @@ static int timehist_migrate_task_event(const struct perf_tool *tool, /* show migrations if requested */ if (sched->show_migrations) { - timehist_print_migration_event(sched, evsel, sample, - machine, thread); + timehist_print_migration_event(sched, sample, machine, thread); } thread__put(thread); return 0; } -static void timehist_update_task_prio(struct evsel *evsel, - struct perf_sample *sample, +static void timehist_update_task_prio(struct perf_sample *sample, struct machine *machine) { struct thread *thread; struct thread_runtime *tr = NULL; - const u32 next_pid = evsel__intval(evsel, sample, "next_pid"); - const u32 next_prio = evsel__intval(evsel, sample, "next_prio"); + const u32 next_pid = perf_sample__intval(sample, "next_pid"); + const u32 next_prio = perf_sample__intval(sample, "next_prio"); if (next_pid == 0) thread = get_idle_thread(sample->cpu); @@ -2787,7 +2858,6 @@ static void timehist_update_task_prio(struct evsel *evsel, static int timehist_sched_change_event(const struct perf_tool *tool, union perf_event *event, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -2798,26 +2868,34 @@ static int timehist_sched_change_event(const struct perf_tool *tool, struct thread_runtime *tr = NULL; u64 tprev, t = sample->time; int rc = 0; - const char state = evsel__taskstate(evsel, sample, "prev_state"); + const char state = perf_sample__taskstate(sample, "prev_state"); + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (sample->cpu >= MAX_CPUS) { + pr_warning("WARNING: at offset %#" PRIx64 ": out-of-bound sample CPU %d, skipping sample\n", + sample->file_offset, sample->cpu); + return 0; + } addr_location__init(&al); if (machine__resolve(machine, &al, sample) < 0) { - pr_err("problem processing %d event. skipping it\n", - event->header.type); + pr_err("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); rc = -1; goto out; } if (sched->show_prio || sched->prio_str) - timehist_update_task_prio(evsel, sample, machine); + timehist_update_task_prio(sample, machine); - thread = timehist_get_thread(sched, sample, machine, evsel); + thread = timehist_get_thread(sched, sample, machine); if (thread == NULL) { rc = -1; goto out; } - if (timehist_skip_sample(sched, thread, evsel, sample)) + if (timehist_skip_sample(sched, thread, sample)) goto out; tr = thread__get_runtime(thread); @@ -2826,7 +2904,7 @@ static int timehist_sched_change_event(const struct perf_tool *tool, goto out; } - tprev = evsel__get_time(evsel, sample->cpu); + tprev = evsel__get_time(sample->evsel, sample->cpu); /* * If start time given: @@ -2856,8 +2934,15 @@ static int timehist_sched_change_event(const struct perf_tool *tool, t = ptime->end; } - if (!sched->idle_hist || thread__tid(thread) == 0) { - if (!cpu_list || test_bit(sample->cpu, cpu_bitmap)) + /* + * Use is_idle_sample() not thread__tid() == 0: a crafted perf.data + * can set common_pid=0 with prev_pid!=0, giving us a machine thread + * whose priv is thread_runtime, not idle_thread_runtime — the cast + * below would read past the allocation. + */ + if (!sched->idle_hist || is_idle_sample(sample)) { + if (!cpu_list || (sample->cpu < MAX_NR_CPUS && + test_bit(sample->cpu, cpu_bitmap))) timehist_update_runtime_stats(tr, t, tprev); if (sched->idle_hist) { @@ -2887,11 +2972,21 @@ static int timehist_sched_change_event(const struct perf_tool *tool, if (itr->cursor.nr) callchain_append(&itr->callchain, &itr->cursor, t - tprev); - itr->last_thread = NULL; + thread__zput(itr->last_thread); + } + + /* + * If the process name is not set for the thread, use "prev_comm" + * to set it. Otherwise the sched summary will have just pid information + */ + if (!thread__comm_set(thread)) { + const char *prev_comm = perf_sample__strval(sample, "prev_comm"); + + thread__set_comm(thread, prev_comm, sample->time); } if (!sched->summary_only) - timehist_print_sample(sched, evsel, sample, &al, thread, t, state); + timehist_print_sample(sched, sample, &al, thread, t, state); } out: @@ -2916,7 +3011,7 @@ static int timehist_sched_change_event(const struct perf_tool *tool, tr->migrated = 0; } - evsel__save_time(evsel, sample->time, sample->cpu); + evsel__save_time(sample->evsel, sample->time, sample->cpu); thread__put(thread); addr_location__exit(&al); @@ -2925,11 +3020,10 @@ static int timehist_sched_change_event(const struct perf_tool *tool, static int timehist_sched_switch_event(const struct perf_tool *tool, union perf_event *event, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) { - return timehist_sched_change_event(tool, event, evsel, sample, machine); + return timehist_sched_change_event(tool, event, sample, machine); } static int process_lost(const struct perf_tool *tool __maybe_unused, @@ -3035,7 +3129,7 @@ static size_t callchain__fprintf_folded(FILE *fp, struct callchain_node *node) list_for_each_entry(chain, &node->val, list) { if (chain->ip >= PERF_CONTEXT_MAX) continue; - if (chain->ms.sym && chain->ms.sym->ignore) + if (chain->ms.sym && symbol__ignore(chain->ms.sym)) continue; ret += fprintf(fp, "%s%s", first ? "" : sep, callchain_list__sym_name(chain, bf, sizeof(bf), @@ -3051,7 +3145,8 @@ static size_t timehist_print_idlehist_callchain(struct rb_root_cached *root) size_t ret = 0; FILE *fp = stdout; struct callchain_node *chain; - struct rb_node *rb_node = rb_first_cached(root); + /* sort() uses rb_insert_color() on rb_root, not rb_root_cached */ + struct rb_node *rb_node = rb_first(&root->rb_root); printf(" %16s %8s %s\n", "Idle time (msec)", "Count", "Callchains"); printf(" %.16s %.8s %.50s\n", graph_dotted_line, graph_dotted_line, @@ -3177,29 +3272,30 @@ static void timehist_print_summary(struct perf_sched *sched, typedef int (*sched_handler)(const struct perf_tool *tool, union perf_event *event, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); static int perf_timehist__process_sample(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_sched *sched = container_of(tool, struct perf_sched, tool); + struct evsel *evsel = sample->evsel; int err = 0; struct perf_cpu this_cpu = { .cpu = sample->cpu, }; - if (this_cpu.cpu > sched->max_cpu.cpu) + /* max_cpu indexes arrays allocated with MAX_CPUS entries */ + if (this_cpu.cpu >= 0 && this_cpu.cpu < MAX_CPUS && + this_cpu.cpu > sched->max_cpu.cpu) sched->max_cpu = this_cpu; if (evsel->handler != NULL) { sched_handler f = evsel->handler; - err = f(tool, event, evsel, sample, machine); + err = f(tool, event, sample, machine); } return err; @@ -3211,7 +3307,7 @@ static int timehist_check_attr(struct perf_sched *sched, struct evsel *evsel; struct evsel_runtime *er; - list_for_each_entry(evsel, &evlist->core.entries, core.node) { + list_for_each_entry(evsel, &evlist__core(evlist)->entries, core.node) { er = evsel__get_runtime(evsel); if (er == NULL) { pr_err("Failed to allocate memory for evsel runtime data\n"); @@ -3299,6 +3395,7 @@ static int perf_sched__timehist(struct perf_sched *sched) */ sched->tool.sample = perf_timehist__process_sample; sched->tool.mmap = perf_event__process_mmap; + sched->tool.mmap2 = perf_event__process_mmap2; sched->tool.comm = perf_event__process_comm; sched->tool.exit = perf_event__process_exit; sched->tool.fork = perf_event__process_fork; @@ -3362,8 +3459,8 @@ static int perf_sched__timehist(struct perf_sched *sched) perf_session__set_tracepoints_handlers(session, migrate_handlers)) goto out; - /* pre-allocate struct for per-CPU idle stats */ - sched->max_cpu.cpu = env->nr_cpus_online; + /* pre-allocate struct for per-CPU idle stats; cap to array bounds */ + sched->max_cpu.cpu = min(env->nr_cpus_online, MAX_CPUS); if (sched->max_cpu.cpu == 0) sched->max_cpu.cpu = 4; if (init_idle_threads(sched->max_cpu.cpu)) @@ -3382,9 +3479,9 @@ static int perf_sched__timehist(struct perf_sched *sched) goto out; } - sched->nr_events = evlist->stats.nr_events[0]; - sched->nr_lost_events = evlist->stats.total_lost; - sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST]; + sched->nr_events = evlist__stats(evlist)->nr_events[0]; + sched->nr_lost_events = evlist__stats(evlist)->total_lost; + sched->nr_lost_chunks = evlist__stats(evlist)->nr_events[PERF_RECORD_LOST]; if (sched->summary) timehist_print_summary(sched, session); @@ -3556,10 +3653,8 @@ static int perf_sched__lat(struct perf_sched *sched) static int setup_map_cpus(struct perf_sched *sched) { - sched->max_cpu.cpu = sysconf(_SC_NPROCESSORS_CONF); - if (sched->map.comp) { - sched->map.comp_cpus = calloc(sched->max_cpu.cpu, sizeof(int)); + sched->map.comp_cpus = calloc(MAX_CPUS, sizeof(*sched->map.comp_cpus)); if (!sched->map.comp_cpus) return -1; } @@ -3757,8 +3852,11 @@ static int process_synthesized_schedstat_event(const struct perf_tool *tool, return 0; } +static volatile sig_atomic_t done; + static void sighandler(int sig __maybe_unused) { + done = 1; } static int enable_sched_schedstats(int *reset) @@ -3818,6 +3916,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, .mode = PERF_DATA_MODE_WRITE, }; + done = 0; signal(SIGINT, sighandler); signal(SIGCHLD, sighandler); signal(SIGTERM, sighandler); @@ -3829,7 +3928,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, session = perf_session__new(&data, &sched->tool); if (IS_ERR(session)) { pr_err("Perf session creation failed.\n"); - evlist__delete(evlist); + evlist__put(evlist); return PTR_ERR(session); } @@ -3887,7 +3986,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_schedstat_event, @@ -3902,8 +4001,11 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, if (argc) evlist__start_workload(evlist); - /* wait for signal */ - pause(); + while (!done) { + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) + break; + sleep(1); + } if (reset) { err = disable_sched_schedstat(); @@ -3925,7 +4027,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched, else fprintf(stderr, "[ perf sched stats: Failed !! ]\n"); - evlist__delete(evlist); + evlist__put(evlist); close(fd); return err; } @@ -3947,6 +4049,8 @@ static struct schedstat_domain *domain_second_pass; static bool after_workload_flag; static bool verbose_field; +static void free_schedstat(struct list_head *head); + static void store_schedstat_cpu_diff(struct schedstat_cpu *after_workload) { struct perf_record_schedstat_cpu *before = cpu_second_pass->cpu_data; @@ -4170,37 +4274,50 @@ static void summarize_schedstat_domain(struct schedstat_domain *summary_domain, */ static int get_all_cpu_stats(struct list_head *head) { - struct schedstat_cpu *cptr = list_first_entry(head, struct schedstat_cpu, cpu_list); - struct schedstat_cpu *summary_head = NULL; - struct perf_record_schedstat_domain *ds; - struct perf_record_schedstat_cpu *cs; + struct schedstat_cpu *cptr, *summary_head = NULL; struct schedstat_domain *dptr, *tdptr; bool is_last = false; int cnt = 1; int ret = 0; + struct list_head tmp_cleanup_list; - if (cptr) { - summary_head = zalloc(sizeof(*summary_head)); - if (!summary_head) - return -ENOMEM; + assert(!list_empty(head)); + cptr = list_first_entry(head, struct schedstat_cpu, cpu_list); - summary_head->cpu_data = zalloc(sizeof(*cs)); - memcpy(summary_head->cpu_data, cptr->cpu_data, sizeof(*cs)); + INIT_LIST_HEAD(&tmp_cleanup_list); - INIT_LIST_HEAD(&summary_head->domain_head); + summary_head = zalloc(sizeof(*summary_head)); + if (!summary_head) + return -ENOMEM; - list_for_each_entry(dptr, &cptr->domain_head, domain_list) { - tdptr = zalloc(sizeof(*tdptr)); - if (!tdptr) - return -ENOMEM; + INIT_LIST_HEAD(&summary_head->domain_head); + INIT_LIST_HEAD(&summary_head->cpu_list); + list_add(&summary_head->cpu_list, &tmp_cleanup_list); - tdptr->domain_data = zalloc(sizeof(*ds)); - if (!tdptr->domain_data) - return -ENOMEM; + summary_head->cpu_data = zalloc(sizeof(*summary_head->cpu_data)); + if (!summary_head->cpu_data) { + ret = -ENOMEM; + goto out_cleanup; + } + memcpy(summary_head->cpu_data, cptr->cpu_data, sizeof(*summary_head->cpu_data)); - memcpy(tdptr->domain_data, dptr->domain_data, sizeof(*ds)); - list_add_tail(&tdptr->domain_list, &summary_head->domain_head); + list_for_each_entry(dptr, &cptr->domain_head, domain_list) { + tdptr = zalloc(sizeof(*tdptr)); + if (!tdptr) { + ret = -ENOMEM; + goto out_cleanup; } + INIT_LIST_HEAD(&tdptr->domain_list); + + tdptr->domain_data = zalloc(sizeof(*tdptr->domain_data)); + if (!tdptr->domain_data) { + free(tdptr); + ret = -ENOMEM; + goto out_cleanup; + } + + memcpy(tdptr->domain_data, dptr->domain_data, sizeof(*tdptr->domain_data)); + list_add_tail(&tdptr->domain_list, &summary_head->domain_head); } list_for_each_entry(cptr, head, cpu_list) { @@ -4212,32 +4329,52 @@ static int get_all_cpu_stats(struct list_head *head) cnt++; summarize_schedstat_cpu(summary_head, cptr, cnt, is_last); + if (list_empty(&summary_head->domain_head)) + continue; + tdptr = list_first_entry(&summary_head->domain_head, struct schedstat_domain, domain_list); list_for_each_entry(dptr, &cptr->domain_head, domain_list) { summarize_schedstat_domain(tdptr, dptr, cnt, is_last); + if (list_is_last(&tdptr->domain_list, &summary_head->domain_head)) { + tdptr = NULL; + break; + } tdptr = list_next_entry(tdptr, domain_list); } } + list_del_init(&summary_head->cpu_list); list_add(&summary_head->cpu_list, head); + return 0; + +out_cleanup: + free_schedstat(&tmp_cleanup_list); return ret; } -static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map **cd_map1, - struct list_head *head2, struct cpu_domain_map **cd_map2, +static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map **cd_map1, int nr1, + struct list_head *head2, struct cpu_domain_map **cd_map2, int nr2, bool summary_only) { struct schedstat_cpu *cptr1 = list_first_entry(head1, struct schedstat_cpu, cpu_list); struct perf_record_schedstat_domain *ds1 = NULL, *ds2 = NULL; - struct perf_record_schedstat_cpu *cs1 = NULL, *cs2 = NULL; struct schedstat_domain *dptr1 = NULL, *dptr2 = NULL; struct schedstat_cpu *cptr2 = NULL; __u64 jiffies1 = 0, jiffies2 = 0; bool is_summary = true; int ret = 0; + if (!cd_map1) { + pr_err("Error: CPU domain map 1 is missing.\n"); + return -1; + } + if (head2 && !cd_map2) { + pr_err("Error: CPU domain map 2 is missing.\n"); + return -1; + } + printf("Description\n"); print_separator2(SEP_LEN, "", 0); printf("%-30s-> %s\n", "DESC", "Description of the field"); @@ -4260,21 +4397,47 @@ static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map ** printf("\n"); ret = get_all_cpu_stats(head1); + if (ret) + return ret; if (cptr2) { ret = get_all_cpu_stats(head2); + if (ret) + return ret; cptr2 = list_first_entry(head2, struct schedstat_cpu, cpu_list); } list_for_each_entry(cptr1, head1, cpu_list) { struct cpu_domain_map *cd_info1 = NULL, *cd_info2 = NULL; + struct perf_record_schedstat_cpu *cs1 = cptr1->cpu_data; + struct perf_record_schedstat_cpu *cs2 = NULL; - cs1 = cptr1->cpu_data; + dptr2 = NULL; + if (cs1->cpu >= (u32)nr1) { + pr_err("Error: CPU %d exceeds domain map size %d\n", cs1->cpu, nr1); + return -1; + } cd_info1 = cd_map1[cs1->cpu]; + if (!cd_info1) { + pr_err("Error: CPU %d domain info is missing in map 1.\n", + cs1->cpu); + return -1; + } if (cptr2) { cs2 = cptr2->cpu_data; + if (cs2->cpu >= (u32)nr2) { + pr_err("Error: CPU %d exceeds domain map size %d\n", cs2->cpu, nr2); + return -1; + } cd_info2 = cd_map2[cs2->cpu]; - dptr2 = list_first_entry(&cptr2->domain_head, struct schedstat_domain, - domain_list); + if (!cd_info2) { + pr_err("Error: CPU %d domain info is missing in map 2.\n", + cs2->cpu); + return -1; + } + if (!list_empty(&cptr2->domain_head)) + dptr2 = list_first_entry(&cptr2->domain_head, + struct schedstat_domain, + domain_list); } if (cs2 && cs1->cpu != cs2->cpu) { @@ -4302,10 +4465,31 @@ static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map ** struct domain_info *dinfo1 = NULL, *dinfo2 = NULL; ds1 = dptr1->domain_data; + ds2 = NULL; + if (ds1->domain >= cd_info1->nr_domains) { + pr_err("Error: Domain %d exceeds max domains %d for CPU %d in map 1.\n", + ds1->domain, cd_info1->nr_domains, cs1->cpu); + return -1; + } dinfo1 = cd_info1->domains[ds1->domain]; + if (!dinfo1) { + pr_err("Error: Domain %d info is missing for CPU %d in map 1.\n", + ds1->domain, cs1->cpu); + return -1; + } if (dptr2) { ds2 = dptr2->domain_data; + if (ds2->domain >= cd_info2->nr_domains) { + pr_err("Error: Domain %d exceeds max domains %d for CPU %d in map 2.\n", + ds2->domain, cd_info2->nr_domains, cs2->cpu); + return -1; + } dinfo2 = cd_info2->domains[ds2->domain]; + if (!dinfo2) { + pr_err("Error: Domain %d info is missing for CPU %d in map 2.\n", + ds2->domain, cs2->cpu); + return -1; + } } if (dinfo2 && dinfo1->domain != dinfo2->domain) { @@ -4334,14 +4518,22 @@ static int show_schedstat_data(struct list_head *head1, struct cpu_domain_map ** print_domain_stats(ds1, ds2, jiffies1, jiffies2); print_separator2(SEP_LEN, "", 0); - if (dptr2) - dptr2 = list_next_entry(dptr2, domain_list); + if (dptr2) { + if (list_is_last(&dptr2->domain_list, &cptr2->domain_head)) + dptr2 = NULL; + else + dptr2 = list_next_entry(dptr2, domain_list); + } } if (summary_only) break; - if (cptr2) - cptr2 = list_next_entry(cptr2, cpu_list); + if (cptr2) { + if (list_is_last(&cptr2->cpu_list, head2)) + cptr2 = NULL; + else + cptr2 = list_next_entry(cptr2, cpu_list); + } is_summary = false; } @@ -4473,9 +4665,11 @@ static void free_schedstat(struct list_head *head) list_for_each_entry_safe(cptr, n2, head, cpu_list) { list_for_each_entry_safe(dptr, n1, &cptr->domain_head, domain_list) { list_del_init(&dptr->domain_list); + free(dptr->domain_data); free(dptr); } list_del_init(&cptr->cpu_list); + free(cptr->cpu_data); free(cptr); } } @@ -4509,7 +4703,7 @@ static int perf_sched__schedstat_report(struct perf_sched *sched) if (err < 0) goto out; - user_requested_cpus = session->evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(session->evlist)->user_requested_cpus; err = perf_session__process_events(session); @@ -4523,7 +4717,9 @@ static int perf_sched__schedstat_report(struct perf_sched *sched) } cd_map = session->header.env.cpu_domain; - err = show_schedstat_data(&cpu_head, cd_map, NULL, NULL, false); + err = show_schedstat_data(&cpu_head, cd_map, + session->header.env.nr_cpus_avail, + NULL, NULL, 0, false); } out: @@ -4538,7 +4734,7 @@ static int perf_sched__schedstat_diff(struct perf_sched *sched, struct cpu_domain_map **cd_map0 = NULL, **cd_map1 = NULL; struct list_head cpu_head_ses0, cpu_head_ses1; struct perf_session *session[2]; - struct perf_data data[2]; + struct perf_data data[2] = {0}; int ret = 0, err = 0; static const char *defaults[] = { "perf.data.old", @@ -4573,8 +4769,10 @@ static int perf_sched__schedstat_diff(struct perf_sched *sched, } err = perf_session__process_events(session[0]); - if (err) + if (err) { + free_schedstat(&cpu_head); goto out_delete_ses0; + } cd_map0 = session[0]->header.env.cpu_domain; list_replace_init(&cpu_head, &cpu_head_ses0); @@ -4590,8 +4788,10 @@ static int perf_sched__schedstat_diff(struct perf_sched *sched, } err = perf_session__process_events(session[1]); - if (err) + if (err) { + free_schedstat(&cpu_head); goto out_delete_ses1; + } cd_map1 = session[1]->header.env.cpu_domain; list_replace_init(&cpu_head, &cpu_head_ses1); @@ -4607,10 +4807,13 @@ static int perf_sched__schedstat_diff(struct perf_sched *sched, if (list_empty(&cpu_head_ses0)) { pr_err("Data is not available\n"); ret = -1; - goto out_delete_ses0; + goto out_delete_ses1; } - show_schedstat_data(&cpu_head_ses0, cd_map0, &cpu_head_ses1, cd_map1, true); + ret = show_schedstat_data(&cpu_head_ses0, cd_map0, session[0]->header.env.nr_cpus_avail, + &cpu_head_ses1, cd_map1, session[1]->header.env.nr_cpus_avail, true); + if (ret) + goto out_delete_ses1; out_delete_ses1: free_schedstat(&cpu_head_ses1); @@ -4645,6 +4848,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, int reset = 0; int err = 0; + done = 0; signal(SIGINT, sighandler); signal(SIGCHLD, sighandler); signal(SIGTERM, sighandler); @@ -4675,7 +4879,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, if (err < 0) goto out; - user_requested_cpus = evlist->core.user_requested_cpus; + user_requested_cpus = evlist__core(evlist)->user_requested_cpus; err = perf_event__synthesize_schedstat(&(sched->tool), process_synthesized_event_live, @@ -4690,8 +4894,11 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, if (argc) evlist__start_workload(evlist); - /* wait for signal */ - pause(); + while (!done) { + if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0) + break; + sleep(1); + } if (reset) { err = disable_sched_schedstat(); @@ -4720,11 +4927,11 @@ static int perf_sched__schedstat_live(struct perf_sched *sched, goto out; } - show_schedstat_data(&cpu_head, cd_map, NULL, NULL, false); + err = show_schedstat_data(&cpu_head, cd_map, nr, NULL, NULL, 0, false); free_cpu_domain_info(cd_map, sv, nr); out: free_schedstat(&cpu_head); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index c8ac9f01a36b..f91d8b1fbd01 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -1,74 +1,78 @@ // SPDX-License-Identifier: GPL-2.0 -#include "builtin.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "asm/bug.h" +#include "builtin.h" +#include "perf.h" +#include "print_binary.h" +#include "print_insn.h" +#include "ui/ui.h" +#include "util/annotate.h" +#include "util/auxtrace.h" +#include "util/cgroup.h" +#include "util/color.h" #include "util/counts.h" +#include "util/cpumap.h" +#include "util/data.h" #include "util/debug.h" +#include "util/dlfilter.h" #include "util/dso.h" -#include -#include "util/header.h" -#include -#include "util/perf_regs.h" -#include "util/session.h" -#include "util/tool.h" -#include "util/map.h" -#include "util/srcline.h" -#include "util/symbol.h" -#include "util/thread.h" -#include "util/trace-event.h" +#include "util/dump-insn.h" #include "util/env.h" +#include "util/event.h" #include "util/evlist.h" #include "util/evsel.h" #include "util/evsel_fprintf.h" #include "util/evswitch.h" +#include "util/header.h" +#include "util/map.h" +#include "util/mem-events.h" +#include "util/mem-info.h" +#include "util/metricgroup.h" +#include "util/path.h" +#include "util/perf_regs.h" +#include "util/pmu.h" +#include "util/record.h" +#include "util/session.h" #include "util/sort.h" -#include "util/data.h" -#include "util/auxtrace.h" -#include "util/cpumap.h" -#include "util/thread_map.h" +#include "util/srcline.h" #include "util/stat.h" -#include "util/color.h" #include "util/string2.h" +#include "util/symbol.h" #include "util/thread-stack.h" +#include "util/thread.h" +#include "util/thread_map.h" #include "util/time-utils.h" -#include "util/path.h" -#include "util/event.h" -#include "util/mem-info.h" -#include "util/metricgroup.h" -#include "ui/ui.h" -#include "print_binary.h" -#include "print_insn.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "asm/bug.h" -#include "util/mem-events.h" -#include "util/dump-insn.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "util/dlfilter.h" -#include "util/record.h" +#include "util/tool.h" +#include "util/trace-event.h" +#include "util/unwind.h" #include "util/util.h" -#include "util/cgroup.h" -#include "util/annotate.h" -#include "perf.h" -#include #ifdef HAVE_LIBTRACEEVENT #include #endif @@ -1287,8 +1291,12 @@ static int ip__fprintf_jump(uint64_t ip, struct branch_entry *en, if (!verbose) { for (j = 0; j < num; j++) printed += fprintf(fp, "%s", pos->abbr_name); - } else - printed += fprintf(fp, "%s %d ", pos->name, num); + if (mask && (num == mask)) + printed += fprintf(fp, "+"); + } else { + printed += fprintf(fp, "%s %d%s", pos->name, + num, mask && (num == mask) ? "+ " : " "); + } } if (numprinted == 0 && !verbose) printed += fprintf(fp, "-"); @@ -1683,7 +1691,7 @@ static int perf_sample__fprintf_bts(struct perf_sample *sample, if (symbol_conf.use_callchain && sample->callchain) { cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack)) cursor = NULL; @@ -2166,6 +2174,7 @@ static int script_find_metrics(const struct pmu_metric *pm, struct evsel *metric_evsel; int ret = metricgroup__parse_groups(metric_evlist, /*pmu=*/"all", + /*cputype_filter=*/false, pm->metric_name, /*metric_no_group=*/false, /*metric_no_merge=*/false, @@ -2221,9 +2230,10 @@ static int script_find_metrics(const struct pmu_metric *pm, evlist__for_each_entry(metric_evlist, metric_evsel) { struct evsel *script_evsel = map_metric_evsel_to_script_evsel(script_evlist, metric_evsel); - struct metric_event *metric_me = metricgroup__lookup(&metric_evlist->metric_events, - metric_evsel, - /*create=*/false); + struct metric_event *metric_me = + metricgroup__lookup(evlist__metric_events(metric_evlist), + metric_evsel, + /*create=*/false); if (script_evsel->metric_id == NULL) { script_evsel->metric_id = metric_evsel->metric_id; @@ -2243,7 +2253,7 @@ static int script_find_metrics(const struct pmu_metric *pm, if (metric_me) { struct metric_expr *expr; struct metric_event *script_me = - metricgroup__lookup(&script_evlist->metric_events, + metricgroup__lookup(evlist__metric_events(script_evlist), script_evsel, /*create=*/true); @@ -2266,7 +2276,7 @@ static int script_find_metrics(const struct pmu_metric *pm, } pr_debug("Found metric '%s' whose evsels match those of in the perf data\n", pm->metric_name); - evlist__delete(metric_evlist); + evlist__put(metric_evlist); out: return 0; } @@ -2313,7 +2323,7 @@ static void perf_sample__fprint_metric(struct thread *thread, assert(stat_config.aggr_mode == AGGR_GLOBAL); stat_config.aggr_get_id = script_aggr_cpu_id_get; stat_config.aggr_map = - cpu_aggr_map__new(evsel->evlist->core.user_requested_cpus, + cpu_aggr_map__new(evlist__core(evsel->evlist)->user_requested_cpus, aggr_cpu_id__global, /*data=*/NULL, /*needs_sort=*/false); } @@ -2418,12 +2428,13 @@ static bool show_event(struct perf_sample *sample, } static void process_event(struct perf_script *script, - struct perf_sample *sample, struct evsel *evsel, + struct perf_sample *sample, struct addr_location *al, struct addr_location *addr_al, struct machine *machine) { struct thread *thread = al->thread; + struct evsel *evsel = sample->evsel; struct perf_event_attr *attr = &evsel->core.attr; unsigned int type = evsel__output_type(evsel); struct evsel_script *es = evsel->priv; @@ -2506,7 +2517,7 @@ static void process_event(struct perf_script *script, if (symbol_conf.use_callchain && sample->callchain) { cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack)) cursor = NULL; @@ -2637,7 +2648,7 @@ static int cleanup_scripting(void) static bool filter_cpu(struct perf_sample *sample) { - if (cpu_list && sample->cpu != (u32)-1) + if (cpu_list && sample->cpu != (u32)-1 && sample->cpu < MAX_NR_CPUS) return !test_bit(sample->cpu, cpu_bitmap); return false; } @@ -2645,10 +2656,10 @@ static bool filter_cpu(struct perf_sample *sample) static int process_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_script *scr = container_of(tool, struct perf_script, tool); + struct evsel *evsel = sample->evsel; struct addr_location al; struct addr_location addr_al; int ret = 0; @@ -2684,8 +2695,9 @@ static int process_sample_event(const struct perf_tool *tool, goto out_put; if (!al.thread && machine__resolve(machine, &al, sample) < 0) { - pr_err("problem processing %d event, skipping it.\n", - event->header.type); + pr_err("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out_put; } @@ -2715,9 +2727,9 @@ static int process_sample_event(const struct perf_tool *tool, thread__resolve(al.thread, &addr_al, sample); addr_al_ptr = &addr_al; } - scripting_ops->process_event(event, sample, evsel, &al, addr_al_ptr); + scripting_ops->process_event(event, sample, &al, addr_al_ptr); } else { - process_event(scr, sample, evsel, &al, &addr_al, machine); + process_event(scr, sample, &al, &addr_al, machine); } out_put: @@ -2729,10 +2741,10 @@ static int process_sample_event(const struct perf_tool *tool, static int process_deferred_sample_event(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct perf_script *scr = container_of(tool, struct perf_script, tool); + struct evsel *evsel = sample->evsel; struct perf_event_attr *attr = &evsel->core.attr; struct evsel_script *es = evsel->priv; unsigned int type = output_type(attr->type); @@ -2766,8 +2778,9 @@ static int process_deferred_sample_event(const struct perf_tool *tool, goto out_put; if (machine__resolve(machine, &al, sample) < 0) { - pr_err("problem processing %d event, skipping it.\n", - event->header.type); + pr_err("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), event->header.type, + sample->file_offset); ret = -1; goto out_put; } @@ -2791,7 +2804,7 @@ static int process_deferred_sample_event(const struct perf_tool *tool, if (symbol_conf.use_callchain && sample->callchain) { cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al.thread, cursor, evsel, + if (thread__resolve_callchain(al.thread, cursor, sample, NULL, NULL, scripting_max_stack)) { pr_info("cannot resolve deferred callchains\n"); @@ -2893,9 +2906,12 @@ static int print_event_with_time(const struct perf_tool *tool, { struct perf_script *script = container_of(tool, struct perf_script, tool); struct perf_session *session = script->session; - struct evsel *evsel = evlist__id2evsel(session->evlist, sample->id); + struct evsel *evsel = sample->evsel; struct thread *thread = NULL; + if (!evsel) + evsel = evlist__id2evsel(session->evlist, sample->id); + if (evsel && !evsel->core.attr.sample_id_all) { sample->cpu = 0; sample->time = timestamp; @@ -3895,7 +3911,7 @@ static int set_maps(struct perf_script *script) if (WARN_ONCE(script->allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evlist->core, script->cpus, script->threads); + perf_evlist__set_maps(evlist__core(evlist), script->cpus, script->threads); if (evlist__alloc_stats(&stat_config, evlist, /*alloc_raw=*/true)) return -ENOMEM; @@ -4159,6 +4175,9 @@ int cmd_script(int argc, const char **argv) "Enable symbol demangling"), OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, "Enable kernel symbol demangling"), + OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", + "unwind styles (libdw,libunwind)", + unwind__option), OPT_STRING(0, "addr2line", &symbol_conf.addr2line_path, "path", "addr2line binary to use for line numbers"), OPT_STRING(0, "time", &script.time_str, "str", diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 99d7db372b48..92cdb2df7285 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -321,7 +321,7 @@ static int read_single_counter(struct evsel *counter, int cpu_map_idx, int threa */ static int read_counter_cpu(struct evsel *counter, int cpu_map_idx) { - int nthreads = perf_thread_map__nr(evsel_list->core.threads); + int nthreads = perf_thread_map__nr(evlist__core(evsel_list)->threads); int thread; if (!counter->supported) @@ -628,11 +628,12 @@ static int dispatch_events(bool forks, int timeout, int interval, int *times) time_to_sleep = sleep_time; while (!done) { - if (forks) + if (forks) { child_exited = waitpid(child_pid, &status, WNOHANG); - else - child_exited = !is_target_alive(&target, evsel_list->core.threads) ? 1 : 0; - + } else { + child_exited = !is_target_alive(&target, + evlist__core(evsel_list)->threads) ? 1 : 0; + } if (child_exited) break; @@ -681,14 +682,15 @@ static enum counter_recovery stat_handle_error(struct evsel *counter, int err) return COUNTER_RETRY; } if (target__has_per_thread(&target) && err != EOPNOTSUPP && - evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) { + evlist__core(evsel_list)->threads && + evlist__core(evsel_list)->threads->err_thread != -1) { /* * For global --per-thread case, skip current * error thread. */ - if (!thread_map__remove(evsel_list->core.threads, - evsel_list->core.threads->err_thread)) { - evsel_list->core.threads->err_thread = -1; + if (!thread_map__remove(evlist__core(evsel_list)->threads, + evlist__core(evsel_list)->threads->err_thread)) { + evlist__core(evsel_list)->threads->err_thread = -1; counter->supported = true; return COUNTER_RETRY; } @@ -787,11 +789,12 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) bool second_pass = false, has_supported_counters; if (forks) { - if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, workload_exec_failed_signal) < 0) { + if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, + workload_exec_failed_signal) < 0) { perror("failed to prepare workload"); return -1; } - child_pid = evsel_list->workload.pid; + child_pid = evlist__workload_pid(evsel_list); } evlist__for_each_entry(evsel_list, counter) { @@ -1199,7 +1202,7 @@ static int parse_cputype(const struct option *opt, const struct perf_pmu *pmu; struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define cputype before events/metrics\n"); return -1; } @@ -1210,6 +1213,7 @@ static int parse_cputype(const struct option *opt, return -1; } parse_events_option_args.pmu_filter = pmu->name; + parse_events_option_args.cputype_filter = true; return 0; } @@ -1220,12 +1224,13 @@ static int parse_pmu_filter(const struct option *opt, { struct evlist *evlist = *(struct evlist **)opt->value; - if (!list_empty(&evlist->core.entries)) { + if (!list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "Must define pmu-filter before events/metrics\n"); return -1; } parse_events_option_args.pmu_filter = str; + parse_events_option_args.cputype_filter = false; return 0; } @@ -1586,8 +1591,9 @@ static int perf_stat_init_aggr_mode(void) if (get_id) { bool needs_sort = stat_config.aggr_mode != AGGR_NONE; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, - get_id, /*data=*/NULL, needs_sort); + stat_config.aggr_map = cpu_aggr_map__new( + evlist__core(evsel_list)->user_requested_cpus, + get_id, /*data=*/NULL, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); return -1; @@ -1596,7 +1602,7 @@ static int perf_stat_init_aggr_mode(void) } if (stat_config.aggr_mode == AGGR_THREAD) { - nr = perf_thread_map__nr(evsel_list->core.threads); + nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) return -ENOMEM; @@ -1615,7 +1621,7 @@ static int perf_stat_init_aggr_mode(void) * taking the highest cpu number to be the size of * the aggregation translate cpumap. */ - nr = perf_cpu_map__max(evsel_list->core.all_cpus).cpu + 1; + nr = perf_cpu_map__max(evlist__core(evsel_list)->all_cpus).cpu + 1; stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr); return stat_config.cpus_aggr_map ? 0 : -ENOMEM; } @@ -1637,9 +1643,10 @@ static struct aggr_cpu_id perf_env__get_socket_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) - id.socket = env->cpu[cpu.cpu].socket_id; + if (topo) + id.socket = topo->socket_id; return id; } @@ -1648,15 +1655,16 @@ static struct aggr_cpu_id perf_env__get_die_aggr_by_cpu(struct perf_cpu cpu, voi { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * die_id is relative to socket, so start * with the socket ID and then add die to * make a unique ID. */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; + id.socket = topo->socket_id; + id.die = topo->die_id; } return id; @@ -1704,12 +1712,13 @@ static struct aggr_cpu_id perf_env__get_cache_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { u32 cache_level = (perf_stat.aggr_level) ?: stat_config.aggr_level; - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; + id.socket = topo->socket_id; + id.die = topo->die_id; perf_env__get_cache_id_for_cpu(cpu, env, cache_level, &id); } @@ -1721,11 +1730,12 @@ static struct aggr_cpu_id perf_env__get_cluster_aggr_by_cpu(struct perf_cpu cpu, { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.cluster = env->cpu[cpu.cpu].cluster_id; + if (topo) { + id.socket = topo->socket_id; + id.die = topo->die_id; + id.cluster = topo->cluster_id; } return id; @@ -1735,16 +1745,17 @@ static struct aggr_cpu_id perf_env__get_core_aggr_by_cpu(struct perf_cpu cpu, vo { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * core_id is relative to socket, die and cluster, we need a * global id. So we set socket, die id, cluster id and core id. */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.cluster = env->cpu[cpu.cpu].cluster_id; - id.core = env->cpu[cpu.cpu].core_id; + id.socket = topo->socket_id; + id.die = topo->die_id; + id.cluster = topo->cluster_id; + id.core = topo->core_id; } return id; @@ -1754,18 +1765,19 @@ static struct aggr_cpu_id perf_env__get_cpu_aggr_by_cpu(struct perf_cpu cpu, voi { struct perf_env *env = data; struct aggr_cpu_id id = aggr_cpu_id__empty(); + struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu); - if (cpu.cpu != -1) { + if (topo) { /* * core_id is relative to socket and die, * we need a global id. So we set * socket, die id and core id */ - id.socket = env->cpu[cpu.cpu].socket_id; - id.die = env->cpu[cpu.cpu].die_id; - id.core = env->cpu[cpu.cpu].core_id; - id.cpu = cpu; + id.socket = topo->socket_id; + id.die = topo->die_id; + id.core = topo->core_id; } + id.cpu = cpu; return id; } @@ -1896,7 +1908,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) bool needs_sort = stat_config.aggr_mode != AGGR_NONE; if (stat_config.aggr_mode == AGGR_THREAD) { - int nr = perf_thread_map__nr(evsel_list->core.threads); + int nr = perf_thread_map__nr(evlist__core(evsel_list)->threads); stat_config.aggr_map = cpu_aggr_map__empty_new(nr); if (stat_config.aggr_map == NULL) @@ -1914,7 +1926,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st) if (!get_id) return 0; - stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus, + stat_config.aggr_map = cpu_aggr_map__new(evlist__core(evsel_list)->user_requested_cpus, get_id, env, needs_sort); if (!stat_config.aggr_map) { pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]); @@ -1993,7 +2005,9 @@ static int add_default_events(void) ret = -1; goto out; } - ret = metricgroup__parse_groups(evlist, pmu, "transaction", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "transaction", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2030,7 +2044,9 @@ static int add_default_events(void) if (!force_metric_only) stat_config.metric_only = true; - ret = metricgroup__parse_groups(evlist, pmu, "smi", + ret = metricgroup__parse_groups(evlist, pmu, + parse_events_option_args.cputype_filter, + "smi", stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2067,7 +2083,7 @@ static int add_default_events(void) } str[8] = stat_config.topdown_level + '0'; if (metricgroup__parse_groups(evlist, - pmu, str, + pmu, parse_events_option_args.cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, @@ -2082,7 +2098,7 @@ static int add_default_events(void) if (!stat_config.topdown_level) stat_config.topdown_level = 1; - if (!evlist->core.nr_entries && !evsel_list->core.nr_entries) { + if (!evlist__nr_entries(evlist) && !evlist__nr_entries(evsel_list)) { /* * Add Default metrics. To minimize multiplexing, don't request * threshold computation, but it will be computed if the events @@ -2106,14 +2122,16 @@ static int add_default_events(void) ret = -ENOMEM; break; } - if (metricgroup__parse_groups(metric_evlist, pmu, default_metricgroup_names[i], + if (metricgroup__parse_groups(metric_evlist, pmu, + parse_events_option_args.cputype_filter, + default_metricgroup_names[i], /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/true, stat_config.user_requested_cpu_list, stat_config.system_wide, stat_config.hardware_aware_grouping) < 0) { - evlist__delete(metric_evlist); + evlist__put(metric_evlist); ret = -1; break; } @@ -2121,13 +2139,13 @@ static int add_default_events(void) evlist__for_each_entry(metric_evlist, evsel) evsel->default_metricgroup = true; - evlist__splice_list_tail(evlist, &metric_evlist->core.entries); + evlist__splice_list_tail(evlist, &evlist__core(metric_evlist)->entries); metricgroup__copy_metric_events(evlist, /*cgrp=*/NULL, - &evlist->metric_events, - &metric_evlist->metric_events); - evlist__delete(metric_evlist); + evlist__metric_events(evlist), + evlist__metric_events(metric_evlist)); + evlist__put(metric_evlist); } - list_sort(/*priv=*/NULL, &evlist->core.entries, default_evlist_evsel_cmp); + list_sort(/*priv=*/NULL, &evlist__core(evlist)->entries, default_evlist_evsel_cmp); } out: @@ -2142,11 +2160,11 @@ static int add_default_events(void) } } parse_events_error__exit(&err); - evlist__splice_list_tail(evsel_list, &evlist->core.entries); + evlist__splice_list_tail(evsel_list, &evlist__core(evlist)->entries); metricgroup__copy_metric_events(evsel_list, /*cgrp=*/NULL, - &evsel_list->metric_events, - &evlist->metric_events); - evlist__delete(evlist); + evlist__metric_events(evsel_list), + evlist__metric_events(evlist)); + evlist__put(evlist); return ret; } @@ -2266,7 +2284,7 @@ static int set_maps(struct perf_stat *st) if (WARN_ONCE(st->maps_allocated, "stats double allocation\n")) return -EINVAL; - perf_evlist__set_maps(&evsel_list->core, st->cpus, st->threads); + perf_evlist__set_maps(evlist__core(evsel_list), st->cpus, st->threads); if (evlist__alloc_stats(&stat_config, evsel_list, /*alloc_raw=*/true)) return -ENOMEM; @@ -2381,7 +2399,7 @@ static int __cmd_report(int argc, const char **argv) perf_stat.session = session; stat_config.output = stderr; - evlist__delete(evsel_list); + evlist__put(evsel_list); evsel_list = session->evlist; ret = perf_session__process_events(session); @@ -2418,7 +2436,7 @@ static void setup_system_wide(int forks) } } - if (evsel_list->core.nr_entries) + if (evlist__nr_entries(evsel_list)) target.system_wide = true; } } @@ -2645,7 +2663,7 @@ int cmd_stat(int argc, const char **argv) stat_config.csv_sep = DEFAULT_SEPARATOR; if (affinity_set) - evsel_list->no_affinity = !affinity; + evlist__set_no_affinity(evsel_list, !affinity); if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) { argc = __cmd_record(stat_options, &opt_mode, argc, argv); @@ -2812,7 +2830,7 @@ int cmd_stat(int argc, const char **argv) } if (stat_config.iostat_run) { - status = iostat_prepare(evsel_list, &stat_config); + status = iostat_prepare(&evsel_list, &stat_config); if (status) goto out; if (iostat_mode == IOSTAT_LIST) { @@ -2842,7 +2860,9 @@ int cmd_stat(int argc, const char **argv) */ if (metrics) { const char *pmu = parse_events_option_args.pmu_filter ?: "all"; - int ret = metricgroup__parse_groups(evsel_list, pmu, metrics, + int ret = metricgroup__parse_groups(evsel_list, pmu, + parse_events_option_args.cputype_filter, + metrics, stat_config.metric_no_group, stat_config.metric_no_merge, stat_config.metric_no_threshold, @@ -2876,9 +2896,10 @@ int cmd_stat(int argc, const char **argv) } #ifdef HAVE_BPF_SKEL if (target.use_bpf && nr_cgroups && - (evsel_list->core.nr_entries / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { + (evlist__nr_entries(evsel_list) / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) { pr_warning("Disabling BPF counters due to more events (%d) than the max (%d)\n", - evsel_list->core.nr_entries / nr_cgroups, BPERF_CGROUP__MAX_EVENTS); + evlist__nr_entries(evsel_list) / nr_cgroups, + BPERF_CGROUP__MAX_EVENTS); target.use_bpf = false; } #endif // HAVE_BPF_SKEL @@ -2916,7 +2937,7 @@ int cmd_stat(int argc, const char **argv) * so we could print it out on output. */ if (stat_config.aggr_mode == AGGR_THREAD) { - thread_map__read_comms(evsel_list->core.threads); + thread_map__read_comms(evlist__core(evsel_list)->threads); } if (stat_config.aggr_mode == AGGR_NODE) @@ -3060,7 +3081,7 @@ int cmd_stat(int argc, const char **argv) if (smi_cost && smi_reset) sysfs__write_int(FREEZE_ON_SMI_PATH, 0); - evlist__delete(evsel_list); + evlist__put(evsel_list); evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close); diff --git a/tools/perf/builtin-timechart.c b/tools/perf/builtin-timechart.c index 28f33e39895d..3f9153d5ecfb 100644 --- a/tools/perf/builtin-timechart.c +++ b/tools/perf/builtin-timechart.c @@ -56,6 +56,7 @@ struct timechart { struct per_pid *all_data; struct power_event *power_events; struct wake_event *wake_events; + struct perf_session *session; int proc_num; unsigned int numcpus; u64 min_freq, /* Lowest CPU frequency seen */ @@ -428,8 +429,10 @@ static void sched_wakeup(struct timechart *tchart, int cpu, u64 timestamp, struct per_pid *p; struct wake_event *we = zalloc(sizeof(*we)); - if (!we) + if (!we) { + free((char *)backtrace); return; + } we->time = timestamp; we->waker = waker; @@ -460,20 +463,28 @@ static void sched_switch(struct timechart *tchart, int cpu, u64 timestamp, const char *backtrace) { struct per_pid *p = NULL, *prev_p; + bool backtrace_used = false; prev_p = find_create_pid(tchart, prev_pid); p = find_create_pid(tchart, next_pid); - if (prev_p->current && prev_p->current->state != TYPE_NONE) + if (prev_p->current && prev_p->current->state != TYPE_NONE) { pid_put_sample(tchart, prev_pid, TYPE_RUNNING, cpu, prev_p->current->state_since, timestamp, backtrace); + backtrace_used = true; + } if (p && p->current) { - if (p->current->state != TYPE_NONE) + if (p->current->state != TYPE_NONE) { + if (backtrace && backtrace_used) + backtrace = strdup(backtrace); + pid_put_sample(tchart, next_pid, p->current->state, cpu, p->current->state_since, timestamp, backtrace); + backtrace_used = true; + } p->current->state_since = timestamp; p->current->state = TYPE_RUNNING; @@ -487,11 +498,17 @@ static void sched_switch(struct timechart *tchart, int cpu, u64 timestamp, if (prev_state == 0) prev_p->current->state = TYPE_WAITING; } + + if (!backtrace_used) + free((char *)backtrace); } -static const char *cat_backtrace(union perf_event *event, - struct perf_sample *sample, - struct machine *machine) +/* + * Returns a malloc'd backtrace string built via open_memstream, or NULL + * on error. Caller must free() the returned pointer. + */ +static char *cat_backtrace(struct perf_sample *sample, + struct machine *machine) { struct addr_location al; unsigned int i; @@ -500,6 +517,7 @@ static const char *cat_backtrace(union perf_event *event, u8 cpumode = PERF_RECORD_MISC_USER; struct ip_callchain *chain = sample->callchain; FILE *f = open_memstream(&p, &p_len); + bool corrupted = false; if (!f) { perror("open_memstream error"); @@ -511,8 +529,8 @@ static const char *cat_backtrace(union perf_event *event, goto exit; if (machine__resolve(machine, &al, sample) < 0) { - fprintf(stderr, "problem processing %d event, skipping it.\n", - event->header.type); + pr_err("problem processing SAMPLE (%u) event at offset %#" PRIx64 ", skipping it.\n", + PERF_RECORD_SAMPLE, sample->file_offset); goto exit; } @@ -537,14 +555,8 @@ static const char *cat_backtrace(union perf_event *event, cpumode = PERF_RECORD_MISC_USER; break; default: - pr_debug("invalid callchain context: " - "%"PRId64"\n", (s64) ip); - - /* - * It seems the callchain is corrupted. - * Discard all. - */ - zfree(&p); + pr_debug("invalid callchain context: %" PRId64 "\n", (s64) ip); + corrupted = true; goto exit; } continue; @@ -561,23 +573,29 @@ static const char *cat_backtrace(union perf_event *event, } exit: addr_location__exit(&al); + /* + * fclose() on an open_memstream always sets p to a valid buffer, + * even if nothing was written — see open_memstream(3). So p is + * never NULL after fclose and we need the flag to discard it. + */ fclose(f); + if (corrupted) + zfree(&p); return p; } typedef int (*tracepoint_handler)(struct timechart *tchart, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace); + struct perf_sample *sample); static int process_sample_event(const struct perf_tool *tool, - union perf_event *event, + union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, - struct machine *machine) + struct machine *machine __maybe_unused) { struct timechart *tchart = container_of(tool, struct timechart, tool); + struct evsel *evsel = sample->evsel; + int ret = 0; if (evsel->core.attr.sample_type & PERF_SAMPLE_TIME) { if (!tchart->first_time || tchart->first_time > sample->time) @@ -588,22 +606,26 @@ static int process_sample_event(const struct perf_tool *tool, if (evsel->handler != NULL) { tracepoint_handler f = evsel->handler; - return f(tchart, evsel, sample, - cat_backtrace(event, sample, machine)); + + ret = f(tchart, sample); } - return 0; + return ret; } static int process_sample_cpu_idle(struct timechart *tchart __maybe_unused, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace __maybe_unused) + struct perf_sample *sample) { - u32 state = evsel__intval(evsel, sample, "state"); - u32 cpu_id = evsel__intval(evsel, sample, "cpu_id"); + u32 state = perf_sample__intval(sample, "state"); + u32 cpu_id = perf_sample__intval(sample, "cpu_id"); + /* perf.data is untrusted input — cpu_id may be corrupted */ + if (cpu_id >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu_id %u\n", + sample->file_offset, cpu_id); + return -1; + } if (state == (u32)PWR_EVENT_EXIT) c_state_end(tchart, cpu_id, sample->time); else @@ -613,41 +635,59 @@ process_sample_cpu_idle(struct timechart *tchart __maybe_unused, static int process_sample_cpu_frequency(struct timechart *tchart, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace __maybe_unused) + struct perf_sample *sample) { - u32 state = evsel__intval(evsel, sample, "state"); - u32 cpu_id = evsel__intval(evsel, sample, "cpu_id"); + u32 state = perf_sample__intval(sample, "state"); + u32 cpu_id = perf_sample__intval(sample, "cpu_id"); + /* perf.data is untrusted input — cpu_id may be corrupted */ + if (cpu_id >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu_id %u\n", + sample->file_offset, cpu_id); + return -1; + } p_state_change(tchart, cpu_id, sample->time, state); return 0; } static int process_sample_sched_wakeup(struct timechart *tchart, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace) + struct perf_sample *sample) { - u8 flags = evsel__intval(evsel, sample, "common_flags"); - int waker = evsel__intval(evsel, sample, "common_pid"); - int wakee = evsel__intval(evsel, sample, "pid"); + u8 flags = perf_sample__intval(sample, "common_flags"); + int waker = perf_sample__intval(sample, "common_pid"); + int wakee = perf_sample__intval(sample, "pid"); + char *backtrace; + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (sample->cpu >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu %u\n", + sample->file_offset, sample->cpu); + return -1; + } + backtrace = cat_backtrace(sample, &tchart->session->machines.host); sched_wakeup(tchart, sample->cpu, sample->time, waker, wakee, flags, backtrace); return 0; } static int process_sample_sched_switch(struct timechart *tchart, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace) + struct perf_sample *sample) { - int prev_pid = evsel__intval(evsel, sample, "prev_pid"); - int next_pid = evsel__intval(evsel, sample, "next_pid"); - u64 prev_state = evsel__intval(evsel, sample, "prev_state"); + int prev_pid = perf_sample__intval(sample, "prev_pid"); + int next_pid = perf_sample__intval(sample, "next_pid"); + u64 prev_state = perf_sample__intval(sample, "prev_state"); + char *backtrace; + + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (sample->cpu >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu %u\n", + sample->file_offset, sample->cpu); + return -1; + } + backtrace = cat_backtrace(sample, &tchart->session->machines.host); sched_switch(tchart, sample->cpu, sample->time, prev_pid, next_pid, prev_state, backtrace); return 0; @@ -656,36 +696,48 @@ process_sample_sched_switch(struct timechart *tchart, #ifdef SUPPORT_OLD_POWER_EVENTS static int process_sample_power_start(struct timechart *tchart __maybe_unused, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace __maybe_unused) + struct perf_sample *sample) { - u64 cpu_id = evsel__intval(evsel, sample, "cpu_id"); - u64 value = evsel__intval(evsel, sample, "value"); + u64 cpu_id = perf_sample__intval(sample, "cpu_id"); + u64 value = perf_sample__intval(sample, "value"); + /* perf.data is untrusted input — cpu_id may be corrupted */ + if (cpu_id >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu_id %llu\n", + sample->file_offset, (unsigned long long)cpu_id); + return -1; + } c_state_start(cpu_id, sample->time, value); return 0; } static int process_sample_power_end(struct timechart *tchart, - struct evsel *evsel __maybe_unused, - struct perf_sample *sample, - const char *backtrace __maybe_unused) + struct perf_sample *sample) { + /* perf.data is untrusted input — CPU may be absent or corrupted */ + if (sample->cpu >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu %u\n", + sample->file_offset, sample->cpu); + return -1; + } c_state_end(tchart, sample->cpu, sample->time); return 0; } static int process_sample_power_frequency(struct timechart *tchart, - struct evsel *evsel, - struct perf_sample *sample, - const char *backtrace __maybe_unused) + struct perf_sample *sample) { - u64 cpu_id = evsel__intval(evsel, sample, "cpu_id"); - u64 value = evsel__intval(evsel, sample, "value"); + u64 cpu_id = perf_sample__intval(sample, "cpu_id"); + u64 value = perf_sample__intval(sample, "value"); + /* perf.data is untrusted input — cpu_id may be corrupted */ + if (cpu_id >= MAX_CPUS) { + pr_debug("at offset %#" PRIx64 ": out-of-bounds cpu_id %llu\n", + sample->file_offset, (unsigned long long)cpu_id); + return -1; + } p_state_change(tchart, cpu_id, sample->time, value); return 0; } @@ -697,10 +749,9 @@ process_sample_power_frequency(struct timechart *tchart, */ static void end_sample_processing(struct timechart *tchart) { - u64 cpu; - struct power_event *pwr; + for (u64 cpu = 0; cpu < tchart->numcpus; cpu++) { + struct power_event *pwr; - for (cpu = 0; cpu <= tchart->numcpus; cpu++) { /* C state */ #if 0 pwr = zalloc(sizeof(*pwr)); @@ -849,120 +900,108 @@ static int pid_end_io_sample(struct timechart *tchart, int pid, int type, static int process_enter_read(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_READ, sample->time, fd); } static int process_exit_read(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_READ, sample->time, ret); } static int process_enter_write(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_WRITE, sample->time, fd); } static int process_exit_write(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_WRITE, sample->time, ret); } static int process_enter_sync(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_SYNC, sample->time, fd); } static int process_exit_sync(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_SYNC, sample->time, ret); } static int process_enter_tx(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_TX, sample->time, fd); } static int process_exit_tx(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_TX, sample->time, ret); } static int process_enter_rx(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_RX, sample->time, fd); } static int process_exit_rx(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_RX, sample->time, ret); } static int process_enter_poll(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long fd = evsel__intval(evsel, sample, "fd"); + long fd = perf_sample__intval(sample, "fd"); return pid_begin_io_sample(tchart, sample->tid, IOTYPE_POLL, sample->time, fd); } static int process_exit_poll(struct timechart *tchart, - struct evsel *evsel, struct perf_sample *sample) { - long ret = evsel__intval(evsel, sample, "ret"); + long ret = perf_sample__intval(sample, "ret"); return pid_end_io_sample(tchart, sample->tid, IOTYPE_POLL, sample->time, ret); } @@ -1071,12 +1110,12 @@ static void draw_wakeups(struct timechart *tchart) c = c->next; } c = p->all; - while (c) { - if (p->pid == we->waker && !from) { + while (c && (!from || !to)) { + if (c->Y && p->pid == we->waker && !from) { from = c->Y; task_from = strdup(c->comm); } - if (p->pid == we->wakee && !to) { + if (c->Y && p->pid == we->wakee && !to) { to = c->Y; task_to = strdup(c->comm); } @@ -1509,6 +1548,73 @@ static void write_svg_file(struct timechart *tchart, const char *filename) svg_close(); } +static void timechart__release(struct timechart *tchart) +{ + struct per_pid *p = tchart->all_data; + struct power_event *pwr = tchart->power_events; + struct wake_event *we = tchart->wake_events; + + while (p) { + struct per_pid *next_pid = p->next; + struct per_pidcomm *c = p->all; + + while (c) { + struct per_pidcomm *next_comm = c->next; + struct cpu_sample *cs = c->samples; + struct io_sample *ios = c->io_samples; + + while (cs) { + struct cpu_sample *next = cs->next; + + zfree(&cs->backtrace); + cs->next = NULL; + free(cs); + + cs = next; + } + + while (ios) { + struct io_sample *next = ios->next; + + ios->next = NULL; + free(ios); + + ios = next; + } + + zfree(&c->comm); + c->next = NULL; + free(c); + + c = next_comm; + } + + p->next = NULL; + free(p); + + p = next_pid; + } + + while (pwr) { + struct power_event *next = pwr->next; + + pwr->next = NULL; + free(pwr); + + pwr = next; + } + + while (we) { + struct wake_event *next = we->next; + + zfree(&we->backtrace); + we->next = NULL; + free(we); + + we = next; + } +} + static int process_header(struct perf_file_section *section __maybe_unused, struct perf_header *ph, int feat, @@ -1520,6 +1626,8 @@ static int process_header(struct perf_file_section *section __maybe_unused, switch (feat) { case HEADER_NRCPUS: tchart->numcpus = ph->env.nr_cpus_avail; + if (tchart->numcpus > MAX_CPUS) + tchart->numcpus = MAX_CPUS; break; case HEADER_CPU_TOPOLOGY: @@ -1618,6 +1726,7 @@ static int __cmd_timechart(struct timechart *tchart, const char *output_name) if (IS_ERR(session)) return PTR_ERR(session); + tchart->session = session; symbol__init(perf_session__env(session)); (void)perf_header__process_sections(&session->header, @@ -1725,6 +1834,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ "syscalls:sys_exit_select", }; unsigned int poll_events_nr = ARRAY_SIZE(poll_events); + int ret; rec_argc = common_args_nr + disk_events_nr * 4 + @@ -1743,7 +1853,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ p = rec_argv; for (i = 0; i < common_args_nr; i++) - *p++ = strdup(common_args[i]); + *p++ = common_args[i]; for (i = 0; i < disk_events_nr; i++) { if (!is_valid_tracepoint(disk_events[i])) { @@ -1752,7 +1862,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ } *p++ = "-e"; - *p++ = strdup(disk_events[i]); + *p++ = disk_events[i]; *p++ = "--filter"; *p++ = filter; } @@ -1763,7 +1873,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ } *p++ = "-e"; - *p++ = strdup(net_events[i]); + *p++ = net_events[i]; *p++ = "--filter"; *p++ = filter; } @@ -1774,7 +1884,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ } *p++ = "-e"; - *p++ = strdup(poll_events[i]); + *p++ = poll_events[i]; *p++ = "--filter"; *p++ = filter; } @@ -1782,7 +1892,11 @@ static int timechart__io_record(int argc, const char **argv, const char *output_ for (i = 0; i < (unsigned int)argc; i++) *p++ = argv[i]; - return cmd_record(rec_argc, rec_argv); + ret = cmd_record(rec_argc, rec_argv); + + free(rec_argv); + free(filter); + return ret; } @@ -1793,6 +1907,7 @@ static int timechart__record(struct timechart *tchart, int argc, const char **ar const char **rec_argv; const char **p; unsigned int record_elems; + int ret; const char * const common_args[] = { "record", "-a", "-R", "-c", "1", "-o", output_data, @@ -1857,24 +1972,27 @@ static int timechart__record(struct timechart *tchart, int argc, const char **ar p = rec_argv; for (i = 0; i < common_args_nr; i++) - *p++ = strdup(common_args[i]); + *p++ = common_args[i]; for (i = 0; i < backtrace_args_no; i++) - *p++ = strdup(backtrace_args[i]); + *p++ = backtrace_args[i]; for (i = 0; i < tasks_args_nr; i++) - *p++ = strdup(tasks_args[i]); + *p++ = tasks_args[i]; for (i = 0; i < power_args_nr; i++) - *p++ = strdup(power_args[i]); + *p++ = power_args[i]; for (i = 0; i < old_power_args_nr; i++) - *p++ = strdup(old_power_args[i]); + *p++ = old_power_args[i]; for (j = 0; j < (unsigned int)argc; j++) *p++ = argv[j]; - return cmd_record(rec_argc, rec_argv); + ret = cmd_record(rec_argc, rec_argv); + + free(rec_argv); + return ret; } static int @@ -2037,6 +2155,7 @@ int cmd_timechart(int argc, const char **argv) ret = __cmd_timechart(&tchart, output_name); out: + timechart__release(&tchart); zfree(&cpus_cstate_start_times); zfree(&cpus_cstate_state); zfree(&cpus_pstate_start_times); diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index f6eb543de537..5933c46ee137 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -141,7 +141,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) notes = symbol__annotation(sym); annotation__lock(notes); - if (!symbol__hists(sym, top->evlist->core.nr_entries)) { + if (!symbol__hists(sym, evlist__nr_entries(top->evlist))) { annotation__unlock(notes); pr_err("Not enough memory for annotating '%s' symbol!\n", sym->name); @@ -186,8 +186,8 @@ static void ui__warn_map_erange(struct map *map, struct symbol *sym, u64 ip) "Please report to linux-kernel@vger.kernel.org\n", ip, dso__long_name(dso), dso__symtab_origin(dso), map__start(map), map__end(map), sym->start, sym->end, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w', sym->name, + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', sym->name, err ? "[unknown]" : uts.machine, err ? "[unknown]" : uts.release, perf_version_string); if (use_browser <= 0) @@ -199,7 +199,7 @@ static void ui__warn_map_erange(struct map *map, struct symbol *sym, u64 ip) static void perf_top__record_precise_ip(struct perf_top *top, struct hist_entry *he, struct perf_sample *sample, - struct evsel *evsel, u64 ip) + u64 ip) EXCLUSIVE_LOCKS_REQUIRED(he->hists->lock) { struct annotation *notes; @@ -216,7 +216,7 @@ static void perf_top__record_precise_ip(struct perf_top *top, if (!annotation__trylock(notes)) return; - err = hist_entry__inc_addr_samples(he, sample, evsel, ip); + err = hist_entry__inc_addr_samples(he, sample, ip); annotation__unlock(notes); @@ -267,7 +267,7 @@ static void perf_top__show_details(struct perf_top *top) more = hist_entry__annotate_printf(he, top->sym_evsel); - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { if (top->zero) symbol__annotate_zero_histogram(symbol, top->sym_evsel); else @@ -293,7 +293,7 @@ static void perf_top__resort_hists(struct perf_top *t) */ hists__unlink(hists); - if (evlist->enabled) { + if (evlist__enabled(evlist)) { if (t->zero) { hists__delete_entries(hists); } else { @@ -334,13 +334,13 @@ static void perf_top__print_sym_table(struct perf_top *top) printf("%-*.*s\n", win_width, win_width, graph_dotted_line); if (!top->record_opts.overwrite && - (top->evlist->stats.nr_lost_warned != - top->evlist->stats.nr_events[PERF_RECORD_LOST])) { - top->evlist->stats.nr_lost_warned = - top->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(top->evlist)->nr_lost_warned != + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(top->evlist)->nr_lost_warned = + evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST]; color_fprintf(stdout, PERF_COLOR_RED, "WARNING: LOST %d chunks, Check IO/CPU overload", - top->evlist->stats.nr_lost_warned); + evlist__stats(top->evlist)->nr_lost_warned); ++printed; } @@ -447,7 +447,7 @@ static void perf_top__print_mapped_keys(struct perf_top *top) fprintf(stdout, "\t[d] display refresh delay. \t(%d)\n", top->delay_secs); fprintf(stdout, "\t[e] display entries (lines). \t(%d)\n", top->print_entries); - if (top->evlist->core.nr_entries > 1) + if (evlist__nr_entries(top->evlist) > 1) fprintf(stdout, "\t[E] active event counter. \t(%s)\n", evsel__name(top->sym_evsel)); fprintf(stdout, "\t[f] profile display filter (count). \t(%d)\n", top->count_filter); @@ -482,7 +482,7 @@ static int perf_top__key_mapped(struct perf_top *top, int c) case 'S': return 1; case 'E': - return top->evlist->core.nr_entries > 1 ? 1 : 0; + return evlist__nr_entries(top->evlist) > 1 ? 1 : 0; default: break; } @@ -528,7 +528,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) } break; case 'E': - if (top->evlist->core.nr_entries > 1) { + if (evlist__nr_entries(top->evlist) > 1) { /* Select 0 as the default event: */ int counter = 0; @@ -539,7 +539,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c) prompt_integer(&counter, "Enter details event counter"); - if (counter >= top->evlist->core.nr_entries) { + if (counter >= evlist__nr_entries(top->evlist)) { top->sym_evsel = evlist__first(top->evlist); fprintf(stderr, "Sorry, no such event, using %s.\n", evsel__name(top->sym_evsel)); sleep(1); @@ -598,8 +598,8 @@ static void perf_top__sort_new_samples(void *arg) { struct perf_top *t = arg; - if (t->evlist->selected != NULL) - t->sym_evsel = t->evlist->selected; + if (evlist__selected(t->evlist) != NULL) + t->sym_evsel = evlist__selected(t->evlist); perf_top__resort_hists(t); @@ -732,20 +732,18 @@ static int hist_iter__top_callback(struct hist_entry_iter *iter, EXCLUSIVE_LOCKS_REQUIRED(iter->he->hists->lock) { struct perf_top *top = arg; - struct evsel *evsel = iter->evsel; if (perf_hpp_list.sym && single) - perf_top__record_precise_ip(top, iter->he, iter->sample, evsel, al->addr); + perf_top__record_precise_ip(top, iter->he, iter->sample, al->addr); hist__account_cycles(iter->sample->branch_stack, al, iter->sample, !(top->record_opts.branch_stack & PERF_SAMPLE_BRANCH_ANY), - NULL, evsel); + /*total_cycles=*/NULL); return 0; } static void perf_event__process_sample(const struct perf_tool *tool, const union perf_event *event, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine) { @@ -768,7 +766,7 @@ static void perf_event__process_sample(const struct perf_tool *tool, if (!machine) { pr_err("%u unprocessable samples recorded.\r", - top->session->evlist->stats.nr_unprocessable_samples++); + evlist__stats(top->session->evlist)->nr_unprocessable_samples++); return; } @@ -830,10 +828,10 @@ static void perf_event__process_sample(const struct perf_tool *tool, } } - if (al.sym == NULL || !al.sym->idle) { - struct hists *hists = evsel__hists(evsel); + if (al.sym == NULL || + !symbol__is_idle(al.sym, al.map ? map__dso(al.map) : NULL, machine->env)) { + struct hists *hists = evsel__hists(sample->evsel); struct hist_entry_iter iter = { - .evsel = evsel, .sample = sample, .add_entry_cb = hist_iter__top_callback, }; @@ -861,7 +859,7 @@ perf_top__process_lost(struct perf_top *top, union perf_event *event, { top->lost += event->lost.lost; top->lost_total += event->lost.lost; - evsel->evlist->stats.total_lost += event->lost.lost; + evlist__stats(evsel->evlist)->total_lost += event->lost.lost; } static void @@ -871,7 +869,7 @@ perf_top__process_lost_samples(struct perf_top *top, { top->lost += event->lost_samples.lost; top->lost_total += event->lost_samples.lost; - evsel->evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evsel->evlist)->total_lost_samples += event->lost_samples.lost; } static u64 last_timestamp; @@ -883,7 +881,7 @@ static void perf_top__mmap_read_idx(struct perf_top *top, int idx) struct mmap *md; union perf_event *event; - md = opts->overwrite ? &evlist->overwrite_mmap[idx] : &evlist->mmap[idx]; + md = opts->overwrite ? &evlist__overwrite_mmap(evlist)[idx] : &evlist__mmap(evlist)[idx]; if (perf_mmap__read_init(&md->core) < 0) return; @@ -920,7 +918,7 @@ static void perf_top__mmap_read(struct perf_top *top) if (overwrite) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_DATA_PENDING); - for (i = 0; i < top->evlist->core.nr_mmaps; i++) + for (i = 0; i < evlist__core(top->evlist)->nr_mmaps; i++) perf_top__mmap_read_idx(top, i); if (overwrite) { @@ -1065,7 +1063,7 @@ static int perf_top__start_counters(struct perf_top *top) goto out_err; } - if (evlist__mmap(evlist, opts->mmap_pages) < 0) { + if (evlist__do_mmap(evlist, opts->mmap_pages) < 0) { ui__error("Failed to mmap with %d (%s)\n", errno, str_error_r(errno, msg, sizeof(msg))); goto out_err; @@ -1167,7 +1165,9 @@ static int deliver_event(struct ordered_events *qe, goto next_event; } - evsel = evlist__id2evsel(session->evlist, sample.id); + evsel = sample.evsel; + if (!evsel) + evsel = evlist__id2evsel(session->evlist, sample.id); assert(evsel != NULL); if (event->header.type == PERF_RECORD_SAMPLE) { @@ -1211,17 +1211,17 @@ static int deliver_event(struct ordered_events *qe, } if (event->header.type == PERF_RECORD_SAMPLE) { - perf_event__process_sample(&top->tool, event, evsel, + perf_event__process_sample(&top->tool, event, &sample, machine); } else if (event->header.type == PERF_RECORD_LOST) { perf_top__process_lost(top, event, evsel); } else if (event->header.type == PERF_RECORD_LOST_SAMPLES) { perf_top__process_lost_samples(top, event, evsel); } else if (event->header.type < PERF_RECORD_MAX) { - events_stats__inc(&session->evlist->stats, event->header.type); + events_stats__inc(evlist__stats(session->evlist), event->header.type); machine__process_event(machine, event, &sample); } else - ++session->evlist->stats.nr_unknown_events; + ++evlist__stats(session->evlist)->nr_unknown_events; ret = 0; next_event: @@ -1296,7 +1296,7 @@ static int __cmd_top(struct perf_top *top) pr_debug("Couldn't synthesize cgroup events.\n"); machine__synthesize_threads(&top->session->machines.host, &opts->target, - top->evlist->core.threads, true, false, + evlist__core(top->evlist)->threads, true, false, top->nr_threads_synthesize); perf_set_multithreaded(); @@ -1652,14 +1652,14 @@ int cmd_top(int argc, const char **argv) perf_env__init(&host_env); status = perf_config(perf_top_config, &top); if (status) - goto out_delete_evlist; + goto out_put_evlist; /* * Since the per arch annotation init routine may need the cpuid, read * it here, since we are not getting this from the perf.data header. */ status = perf_env__set_cmdline(&host_env, argc, argv); if (status) - goto out_delete_evlist; + goto out_put_evlist; status = perf_env__read_cpuid(&host_env); if (status) { @@ -1680,30 +1680,30 @@ int cmd_top(int argc, const char **argv) annotate_opts.disassembler_style = strdup(disassembler_style); if (!annotate_opts.disassembler_style) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (objdump_path) { annotate_opts.objdump_path = strdup(objdump_path); if (!annotate_opts.objdump_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } if (addr2line_path) { symbol_conf.addr2line_path = strdup(addr2line_path); if (!symbol_conf.addr2line_path) { status = -ENOMEM; - goto out_delete_evlist; + goto out_put_evlist; } } status = symbol__validate_sym_arguments(); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (annotate_check_args() < 0) - goto out_delete_evlist; + goto out_put_evlist; status = target__validate(target); if (status) { @@ -1714,19 +1714,19 @@ int cmd_top(int argc, const char **argv) if (target__none(target)) target->system_wide = true; - if (!top.evlist->core.nr_entries) { + if (!evlist__nr_entries(top.evlist)) { struct evlist *def_evlist = evlist__new_default(target, callchain_param.enabled); if (!def_evlist) - goto out_delete_evlist; + goto out_put_evlist; - evlist__splice_list_tail(top.evlist, &def_evlist->core.entries); - evlist__delete(def_evlist); + evlist__splice_list_tail(top.evlist, &evlist__core(def_evlist)->entries); + evlist__put(def_evlist); } status = evswitch__init(&top.evswitch, top.evlist, stderr); if (status) - goto out_delete_evlist; + goto out_put_evlist; if (symbol_conf.report_hierarchy) { /* disable incompatible options */ @@ -1737,18 +1737,18 @@ int cmd_top(int argc, const char **argv) pr_err("Error: --hierarchy and --fields options cannot be used together\n"); parse_options_usage(top_usage, options, "fields", 0); parse_options_usage(NULL, options, "hierarchy", 0); - goto out_delete_evlist; + goto out_put_evlist; } } if (top.stitch_lbr && !(callchain_param.record_mode == CALLCHAIN_LBR)) { pr_err("Error: --stitch-lbr must be used with --call-graph lbr\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (nr_cgroups > 0 && opts->record_cgroup) { pr_err("--cgroup and --all-cgroups cannot be used together\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (branch_call_mode) { @@ -1772,7 +1772,7 @@ int cmd_top(int argc, const char **argv) status = perf_env__read_core_pmu_caps(&host_env); if (status) { pr_err("PMU capability data is not available\n"); - goto out_delete_evlist; + goto out_put_evlist; } } @@ -1795,9 +1795,9 @@ int cmd_top(int argc, const char **argv) if (IS_ERR(top.session)) { status = PTR_ERR(top.session); top.session = NULL; - goto out_delete_evlist; + goto out_put_evlist; } - top.evlist->session = top.session; + evlist__set_session(top.evlist, top.session); if (setup_sorting(top.evlist, perf_session__env(top.session)) < 0) { if (sort_order) @@ -1805,7 +1805,7 @@ int cmd_top(int argc, const char **argv) if (field_order) parse_options_usage(sort_order ? NULL : top_usage, options, "fields", 0); - goto out_delete_evlist; + goto out_put_evlist; } if (top.uid_str) { @@ -1814,18 +1814,18 @@ int cmd_top(int argc, const char **argv) if (uid == UINT_MAX) { ui__error("Invalid User: %s", top.uid_str); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } status = parse_uid_filter(top.evlist, uid); if (status) - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__create_maps(top.evlist, target) < 0) { ui__error("Couldn't create thread/CPU maps: %s\n", errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf))); status = -errno; - goto out_delete_evlist; + goto out_put_evlist; } if (top.delay_secs < 1) @@ -1833,7 +1833,7 @@ int cmd_top(int argc, const char **argv) if (record_opts__config(opts)) { status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } top.sym_evsel = evlist__first(top.evlist); @@ -1848,14 +1848,14 @@ int cmd_top(int argc, const char **argv) status = symbol__annotation_init(); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; annotation_config__init(); symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL); status = symbol__init(NULL); if (status < 0) - goto out_delete_evlist; + goto out_put_evlist; sort__setup_elide(stdout); @@ -1875,13 +1875,13 @@ int cmd_top(int argc, const char **argv) if (top.sb_evlist == NULL) { pr_err("Couldn't create side band evlist.\n."); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } if (evlist__add_bpf_sb_event(top.sb_evlist, &host_env)) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); status = -EINVAL; - goto out_delete_evlist; + goto out_put_evlist; } } #endif @@ -1896,8 +1896,8 @@ int cmd_top(int argc, const char **argv) if (!opts->no_bpf_event) evlist__stop_sb_thread(top.sb_evlist); -out_delete_evlist: - evlist__delete(top.evlist); +out_put_evlist: + evlist__put(top.evlist); perf_session__delete(top.session); annotation_options__exit(); perf_env__exit(&host_env); diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 48615ddccd93..845a9fe47f0b 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -60,12 +60,13 @@ #include "callchain.h" #include "print_binary.h" #include "string2.h" -#include "syscalltbl.h" +#include "trace/beauty/syscalltbl.h" #include "../perf.h" #include "trace_augment.h" #include "dwarf-regs.h" #include +#include #include #include #include @@ -217,6 +218,7 @@ struct trace { bool kernel_syscallchains; s16 args_alignment; bool show_tstamp; + bool show_cpu; bool show_duration; bool show_zeros; bool show_arg_names; @@ -234,6 +236,16 @@ struct trace { const char *uid_str; }; +bool trace__show_zeros(const struct trace *trace) +{ + return trace->show_zeros; +} + +struct machine *trace__host(const struct trace *trace) +{ + return trace->host; +} + static void trace__load_vmlinux_btf(struct trace *trace __maybe_unused) { #ifdef HAVE_LIBBPF_SUPPORT @@ -448,10 +460,10 @@ static int evsel__init_tp_ptr_field(struct evsel *evsel, struct tp_field *field, ({ struct syscall_tp *sc = __evsel__syscall_tp(evsel);\ evsel__init_tp_ptr_field(evsel, &sc->name, #name); }) -static void evsel__delete_priv(struct evsel *evsel) +static void evsel__put_and_free_priv(struct evsel *evsel) { zfree(&evsel->priv); - evsel__delete(evsel); + evsel__put(evsel); } static int evsel__init_syscall_tp(struct evsel *evsel) @@ -531,16 +543,16 @@ static struct evsel *perf_evsel__raw_syscall_newtp(const char *direction, void * return evsel; out_delete: - evsel__delete_priv(evsel); + evsel__put_and_free_priv(evsel); return NULL; } -#define perf_evsel__sc_tp_uint(evsel, name, sample) \ - ({ struct syscall_tp *fields = __evsel__syscall_tp(evsel); \ +#define perf_evsel__sc_tp_uint(name, sample) \ + ({ struct syscall_tp *fields = __evsel__syscall_tp(sample->evsel); \ fields->name.integer(&fields->name, sample); }) -#define perf_evsel__sc_tp_ptr(evsel, name, sample) \ - ({ struct syscall_tp *fields = __evsel__syscall_tp(evsel); \ +#define perf_evsel__sc_tp_ptr(name, sample) \ + ({ struct syscall_tp *fields = __evsel__syscall_tp(sample->evsel); \ fields->name.pointer(&fields->name, sample); }) size_t strarray__scnprintf_suffix(struct strarray *sa, char *bf, size_t size, const char *intfmt, bool show_suffix, int val) @@ -771,10 +783,6 @@ static const char *bpf_cmd[] = { }; static DEFINE_STRARRAY(bpf_cmd, "BPF_"); -#include "trace/beauty/generated/fsconfig_arrays.c" - -static DEFINE_STRARRAY(fsconfig_cmds, "FSCONFIG_"); - static const char *epoll_ctl_ops[] = { "ADD", "DEL", "MOD", }; static DEFINE_STRARRAY_OFFSET(epoll_ctl_ops, "EPOLL_CTL_", 1); @@ -964,7 +972,7 @@ static bool syscall_arg__strtoul_btf_enum(char *bf, size_t size, struct syscall_ struct btf *btf = arg->trace->btf; struct btf_enum *be = btf_enum(bt); - for (int i = 0; i < btf_vlen(bt); ++i, ++be) { + for (u32 i = 0; i < btf_vlen(bt); ++i, ++be) { const char *name = btf__name_by_offset(btf, be->name_off); int max_len = max(size, strlen(name)); @@ -1009,9 +1017,9 @@ static bool syscall_arg__strtoul_btf_type(char *bf, size_t size, struct syscall_ static size_t btf_enum_scnprintf(const struct btf_type *type, struct btf *btf, char *bf, size_t size, int val) { struct btf_enum *be = btf_enum(type); - const int nr_entries = btf_vlen(type); + const unsigned int nr_entries = btf_vlen(type); - for (int i = 0; i < nr_entries; ++i, ++be) { + for (unsigned int i = 0; i < nr_entries; ++i, ++be) { if (be->val == val) { return scnprintf(bf, size, "%s", btf__name_by_offset(btf, be->name_off)); @@ -1123,21 +1131,6 @@ static bool syscall_arg__strtoul_btf_type(char *bf __maybe_unused, size_t size _ .parm = &strarray__##array, \ .show_zero = true, } -#include "trace/beauty/eventfd.c" -#include "trace/beauty/futex_op.c" -#include "trace/beauty/futex_val3.c" -#include "trace/beauty/mmap.c" -#include "trace/beauty/mode_t.c" -#include "trace/beauty/msg_flags.c" -#include "trace/beauty/open_flags.c" -#include "trace/beauty/perf_event_open.c" -#include "trace/beauty/pid.c" -#include "trace/beauty/sched_policy.c" -#include "trace/beauty/seccomp.c" -#include "trace/beauty/signum.c" -#include "trace/beauty/socket_type.c" -#include "trace/beauty/waitid_options.c" - static const struct syscall_fmt syscall_fmts[] = { { .name = "access", .arg = { [1] = { .scnprintf = SCA_ACCMODE, /* mode */ }, }, }, @@ -1528,6 +1521,7 @@ static size_t fprintf_duration(unsigned long t, bool calculated, FILE *fp) */ struct thread_trace { u64 entry_time; + u32 entry_cpu; bool entry_pending; unsigned long nr_events; unsigned long pfmaj, pfmin; @@ -1890,6 +1884,27 @@ static size_t trace__fprintf_tstamp(struct trace *trace, u64 tstamp, FILE *fp) return fprintf(fp, " ? "); } +/** + * trace__fprintf_cpu - Print the CPU ID to a given file stream + * @cpu: The CPU ID to print + * @fp: The file stream to write to + * + * Formats and prints the specified CPU ID enclosed in brackets + * (e.g., "[003] ") to the provided file pointer. It is used to + * align and display the CPU ID consistently within the trace output. + * + * Return: The number of characters printed. + */ +static size_t trace__fprintf_cpu(u32 cpu, FILE *fp) +{ + size_t printed = 0; + + if (cpu != (u32)-1) + printed += fprintf(fp, "[%03u] ", cpu); + + return printed; +} + static pid_t workload_pid = -1; static volatile sig_atomic_t done = false; static volatile sig_atomic_t interrupted = false; @@ -1920,12 +1935,15 @@ static size_t trace__fprintf_comm_tid(struct trace *trace, struct thread *thread } static size_t trace__fprintf_entry_head(struct trace *trace, struct thread *thread, - u64 duration, bool duration_calculated, u64 tstamp, FILE *fp) + u64 duration, bool duration_calculated, + u64 tstamp, u32 cpu, FILE *fp) { size_t printed = 0; if (trace->show_tstamp) printed = trace__fprintf_tstamp(trace, tstamp, fp); + if (trace->show_cpu && cpu != (u32)-1) + printed += trace__fprintf_cpu(cpu, fp); if (trace->show_duration) printed += fprintf_duration(duration, duration_calculated, fp); return printed + trace__fprintf_comm_tid(trace, thread, fp); @@ -2005,7 +2023,7 @@ static int trace__symbols_init(struct trace *trace, int argc, const char **argv, goto out; err = __machine__synthesize_threads(trace->host, &trace->tool, &trace->opts.target, - evlist->core.threads, trace__tool_process, + evlist__core(evlist)->threads, trace__tool_process, /*needs_mmap=*/callchain_param.enabled && !trace->summary_only, /*mmap_data=*/false, @@ -2579,7 +2597,7 @@ static struct syscall *trace__find_syscall(struct trace *trace, int e_machine, i return sc; } -typedef int (*tracepoint_handler)(struct trace *trace, struct evsel *evsel, +typedef int (*tracepoint_handler)(struct trace *trace, union perf_event *event, struct perf_sample *sample); @@ -2704,7 +2722,9 @@ static int trace__printf_interrupted_entry(struct trace *trace) if (!ttrace->entry_pending) return 0; - printed = trace__fprintf_entry_head(trace, trace->current, 0, false, ttrace->entry_time, trace->output); + printed = trace__fprintf_entry_head(trace, trace->current, 0, false, + ttrace->entry_time, ttrace->entry_cpu, + trace->output); printed += len = fprintf(trace->output, "%s)", ttrace->entry_str); if (len < trace->args_alignment - 4) @@ -2718,8 +2738,8 @@ static int trace__printf_interrupted_entry(struct trace *trace) return printed; } -static int trace__fprintf_sample(struct trace *trace, struct evsel *evsel, - struct perf_sample *sample, struct thread *thread) +static int trace__fprintf_sample(struct trace *trace, struct perf_sample *sample, + struct thread *thread) { int printed = 0; @@ -2727,7 +2747,7 @@ static int trace__fprintf_sample(struct trace *trace, struct evsel *evsel, double ts = (double)sample->time / NSEC_PER_MSEC; printed += fprintf(trace->output, "%22s %10.3f %s %d/%d [%d]\n", - evsel__name(evsel), ts, + evsel__name(sample->evsel), ts, thread__comm_str(thread), sample->pid, sample->tid, sample->cpu); } @@ -2774,15 +2794,16 @@ static void *syscall__augmented_args(struct syscall *sc, struct perf_sample *sam return NULL; } -static int trace__sys_enter(struct trace *trace, struct evsel *evsel, +static int trace__sys_enter(struct trace *trace, union perf_event *event __maybe_unused, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; char *msg; void *args; int printed = 0; struct thread *thread; - int id = perf_evsel__sc_tp_uint(evsel, id, sample), err = -1; + int id = perf_evsel__sc_tp_uint(id, sample), err = -1; int augmented_args_size = 0, e_machine; void *augmented_args = NULL; struct syscall *sc; @@ -2797,9 +2818,9 @@ static int trace__sys_enter(struct trace *trace, struct evsel *evsel, if (ttrace == NULL) goto out_put; - trace__fprintf_sample(trace, evsel, sample, thread); + trace__fprintf_sample(trace, sample, thread); - args = perf_evsel__sc_tp_ptr(evsel, args, sample); + args = perf_evsel__sc_tp_ptr(args, sample); if (ttrace->entry_str == NULL) { ttrace->entry_str = malloc(trace__entry_str_size); @@ -2822,6 +2843,7 @@ static int trace__sys_enter(struct trace *trace, struct evsel *evsel, if (evsel != trace->syscalls.events.sys_enter) augmented_args = syscall__augmented_args(sc, sample, &augmented_args_size, trace->raw_augmented_syscalls_args_size); ttrace->entry_time = sample->time; + ttrace->entry_cpu = sample->cpu; msg = ttrace->entry_str; printed += scnprintf(msg + printed, trace__entry_str_size - printed, "%s(", sc->name); @@ -2832,7 +2854,9 @@ static int trace__sys_enter(struct trace *trace, struct evsel *evsel, if (!(trace->duration_filter || trace->summary_only || trace->failure_only || trace->min_stack)) { int alignment = 0; - trace__fprintf_entry_head(trace, thread, 0, false, ttrace->entry_time, trace->output); + trace__fprintf_entry_head(trace, thread, 0, false, + ttrace->entry_time, + sample->cpu, trace->output); printed = fprintf(trace->output, "%s)", ttrace->entry_str); if (trace->args_alignment > printed) alignment = trace->args_alignment - printed; @@ -2854,12 +2878,11 @@ static int trace__sys_enter(struct trace *trace, struct evsel *evsel, return err; } -static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, - struct perf_sample *sample) +static int trace__fprintf_sys_enter(struct trace *trace, struct perf_sample *sample) { struct thread_trace *ttrace; struct thread *thread; - int id = perf_evsel__sc_tp_uint(evsel, id, sample), err = -1; + int id = perf_evsel__sc_tp_uint(id, sample), err = -1; struct syscall *sc; char msg[1024]; void *args, *augmented_args = NULL; @@ -2869,7 +2892,7 @@ static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, thread = machine__findnew_thread(trace->host, sample->pid, sample->tid); e_machine = thread__e_machine(thread, trace->host, /*e_flags=*/NULL); - sc = trace__syscall_info(trace, evsel, e_machine, id); + sc = trace__syscall_info(trace, sample->evsel, e_machine, id); if (sc == NULL) goto out_put; ttrace = thread__trace(thread, trace); @@ -2880,7 +2903,7 @@ static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, if (ttrace == NULL) goto out_put; - args = perf_evsel__sc_tp_ptr(evsel, args, sample); + args = perf_evsel__sc_tp_ptr(args, sample); augmented_args = syscall__augmented_args(sc, sample, &augmented_args_size, trace->raw_augmented_syscalls_args_size); printed += syscall__scnprintf_args(sc, msg, sizeof(msg), args, augmented_args, augmented_args_size, trace, thread); fprintf(trace->output, "%.*s", (int)printed, msg); @@ -2890,10 +2913,11 @@ static int trace__fprintf_sys_enter(struct trace *trace, struct evsel *evsel, return err; } -static int trace__resolve_callchain(struct trace *trace, struct evsel *evsel, +static int trace__resolve_callchain(struct trace *trace, struct perf_sample *sample, struct callchain_cursor *cursor) { + struct evsel *evsel = sample->evsel; struct addr_location al; int max_stack = evsel->core.attr.sample_max_stack ? evsel->core.attr.sample_max_stack : @@ -2904,7 +2928,7 @@ static int trace__resolve_callchain(struct trace *trace, struct evsel *evsel, if (machine__resolve(trace->host, &al, sample) < 0) goto out; - err = thread__resolve_callchain(al.thread, cursor, evsel, sample, NULL, NULL, max_stack); + err = thread__resolve_callchain(al.thread, cursor, sample, NULL, NULL, max_stack); out: addr_location__exit(&al); return err; @@ -2920,15 +2944,16 @@ static int trace__fprintf_callchain(struct trace *trace, struct perf_sample *sam return sample__fprintf_callchain(sample, 38, print_opts, get_tls_callchain_cursor(), symbol_conf.bt_stop_list, trace->output); } -static int trace__sys_exit(struct trace *trace, struct evsel *evsel, +static int trace__sys_exit(struct trace *trace, union perf_event *event __maybe_unused, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; long ret; u64 duration = 0; bool duration_calculated = false; struct thread *thread; - int id = perf_evsel__sc_tp_uint(evsel, id, sample), err = -1, callchain_ret = 0, printed = 0; + int id = perf_evsel__sc_tp_uint(id, sample), err = -1, callchain_ret = 0, printed = 0; int alignment = trace->args_alignment, e_machine; struct syscall *sc; struct thread_trace *ttrace; @@ -2942,9 +2967,9 @@ static int trace__sys_exit(struct trace *trace, struct evsel *evsel, if (ttrace == NULL) goto out_put; - trace__fprintf_sample(trace, evsel, sample, thread); + trace__fprintf_sample(trace, sample, thread); - ret = perf_evsel__sc_tp_uint(evsel, ret, sample); + ret = perf_evsel__sc_tp_uint(ret, sample); if (trace->summary) thread__update_stats(thread, ttrace, id, sample, ret, trace); @@ -2966,7 +2991,7 @@ static int trace__sys_exit(struct trace *trace, struct evsel *evsel, if (sample->callchain) { struct callchain_cursor *cursor = get_tls_callchain_cursor(); - callchain_ret = trace__resolve_callchain(trace, evsel, sample, cursor); + callchain_ret = trace__resolve_callchain(trace, sample, cursor); if (callchain_ret == 0) { if (cursor->nr < trace->min_stack) goto out; @@ -2977,7 +3002,9 @@ static int trace__sys_exit(struct trace *trace, struct evsel *evsel, if (trace->summary_only || (ret >= 0 && trace->failure_only)) goto out; - trace__fprintf_entry_head(trace, thread, duration, duration_calculated, ttrace->entry_time, trace->output); + trace__fprintf_entry_head(trace, thread, duration, + duration_calculated, ttrace->entry_time, + sample->cpu, trace->output); if (ttrace->entry_pending) { printed = fprintf(trace->output, "%s", ttrace->entry_str); @@ -3005,9 +3032,8 @@ static int trace__sys_exit(struct trace *trace, struct evsel *evsel, } else if (ret < 0) { errno_print: { char bf[STRERR_BUFSIZE]; - struct perf_env *env = evsel__env(evsel) ?: &trace->host_env; const char *emsg = str_error_r(-ret, bf, sizeof(bf)); - const char *e = perf_env__arch_strerrno(env, err); + const char *e = perf_env__arch_strerrno(e_machine, err); fprintf(trace->output, "-1 %s (%s)", e, emsg); } @@ -3058,7 +3084,7 @@ errno_print: { return err; } -static int trace__vfs_getname(struct trace *trace, struct evsel *evsel, +static int trace__vfs_getname(struct trace *trace, union perf_event *event __maybe_unused, struct perf_sample *sample) { @@ -3067,7 +3093,7 @@ static int trace__vfs_getname(struct trace *trace, struct evsel *evsel, size_t filename_len, entry_str_len, to_move; ssize_t remaining_space; char *pos; - const char *filename = evsel__rawptr(evsel, sample, "pathname"); + const char *filename = perf_sample__strval(sample, "pathname"); if (!thread) goto out; @@ -3119,11 +3145,11 @@ static int trace__vfs_getname(struct trace *trace, struct evsel *evsel, return 0; } -static int trace__sched_stat_runtime(struct trace *trace, struct evsel *evsel, +static int trace__sched_stat_runtime(struct trace *trace, union perf_event *event __maybe_unused, struct perf_sample *sample) { - u64 runtime = evsel__intval(evsel, sample, "runtime"); + u64 runtime = perf_sample__intval(sample, "runtime"); double runtime_ms = (double)runtime / NSEC_PER_MSEC; struct thread *thread = machine__findnew_thread(trace->host, sample->pid, @@ -3141,11 +3167,11 @@ static int trace__sched_stat_runtime(struct trace *trace, struct evsel *evsel, out_dump: fprintf(trace->output, "%s: comm=%s,pid=%u,runtime=%" PRIu64 ",vruntime=%" PRIu64 ")\n", - evsel->name, - evsel__strval(evsel, sample, "comm"), - (pid_t)evsel__intval(evsel, sample, "pid"), + sample->evsel->name, + perf_sample__strval(sample, "comm"), + (pid_t)perf_sample__intval(sample, "pid"), runtime, - evsel__intval(evsel, sample, "vruntime")); + perf_sample__intval(sample, "vruntime")); goto out_put; } @@ -3181,9 +3207,10 @@ static void bpf_output__fprintf(struct trace *trace, ++trace->nr_events_printed; } -static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, struct perf_sample *sample, +static size_t trace__fprintf_tp_fields(struct trace *trace, struct perf_sample *sample, struct thread *thread, void *augmented_args, int augmented_args_size) { + struct evsel *evsel = sample->evsel; char bf[2048]; size_t size = sizeof(bf); const struct tep_event *tp_format = evsel__tp_format(evsel); @@ -3234,6 +3261,27 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, if (val == 0 && !trace->show_zeros && !arg->show_zero && arg->strtoul != STUL_BTF_TYPE) continue; + /* + * __probe_ip is implicitly added to bare dynamic probes. + * Suppress it by default to avoid cluttering the output. + * If verbose mode is enabled, ensure it is formatted as a + * hexadecimal memory address rather than a signed integer. + */ + if (evsel__is_probe(evsel) && !strcmp(field->name, "__probe_ip")) { + if (!verbose) + continue; + + printed += scnprintf(bf + printed, size - printed, + "%s", printed ? ", " : ""); + if (trace->show_arg_names) + printed += scnprintf(bf + printed, size - printed, + "%s: ", field->name); + + printed += scnprintf(bf + printed, size - printed, "%#016llx", + (unsigned long long)val); + continue; + } + printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); if (trace->show_arg_names) @@ -3251,10 +3299,11 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, return fprintf(trace->output, "%.*s", (int)printed, bf); } -static int trace__event_handler(struct trace *trace, struct evsel *evsel, +static int trace__event_handler(struct trace *trace, union perf_event *event __maybe_unused, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; struct thread *thread; int callchain_ret = 0; @@ -3266,7 +3315,7 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, if (sample->callchain) { struct callchain_cursor *cursor = get_tls_callchain_cursor(); - callchain_ret = trace__resolve_callchain(trace, evsel, sample, cursor); + callchain_ret = trace__resolve_callchain(trace, sample, cursor); if (callchain_ret == 0) { if (cursor->nr < trace->min_stack) goto out; @@ -3277,6 +3326,9 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, trace__printf_interrupted_entry(trace); trace__fprintf_tstamp(trace, sample->time, trace->output); + if (trace->show_cpu) + trace__fprintf_cpu(sample->cpu, trace->output); + if (trace->trace_syscalls && trace->show_duration) fprintf(trace->output, "( ): "); @@ -3284,7 +3336,7 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, trace__fprintf_comm_tid(trace, thread, trace->output); if (evsel == trace->syscalls.events.bpf_output) { - int id = perf_evsel__sc_tp_uint(evsel, id, sample); + int id = perf_evsel__sc_tp_uint(id, sample); int e_machine = thread ? thread__e_machine(thread, trace->host, /*e_flags=*/NULL) : EM_HOST; @@ -3292,7 +3344,7 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, if (sc) { fprintf(trace->output, "%s(", sc->name); - trace__fprintf_sys_enter(trace, evsel, sample); + trace__fprintf_sys_enter(trace, sample); fputc(')', trace->output); goto newline; } @@ -3312,13 +3364,13 @@ static int trace__event_handler(struct trace *trace, struct evsel *evsel, const struct tep_event *tp_format = evsel__tp_format(evsel); if (tp_format && (strncmp(tp_format->name, "sys_enter_", 10) || - trace__fprintf_sys_enter(trace, evsel, sample))) { + trace__fprintf_sys_enter(trace, sample))) { if (trace->libtraceevent_print) { event_format__fprintf(tp_format, sample->cpu, sample->raw_data, sample->raw_size, trace->output); } else { - trace__fprintf_tp_fields(trace, evsel, sample, thread, NULL, 0); + trace__fprintf_tp_fields(trace, sample, thread, NULL, 0); } } } @@ -3360,7 +3412,6 @@ static void print_location(FILE *f, struct perf_sample *sample, } static int trace__pgfault(struct trace *trace, - struct evsel *evsel, union perf_event *event __maybe_unused, struct perf_sample *sample) { @@ -3377,7 +3428,7 @@ static int trace__pgfault(struct trace *trace, if (sample->callchain) { struct callchain_cursor *cursor = get_tls_callchain_cursor(); - callchain_ret = trace__resolve_callchain(trace, evsel, sample, cursor); + callchain_ret = trace__resolve_callchain(trace, sample, cursor); if (callchain_ret == 0) { if (cursor->nr < trace->min_stack) goto out_put; @@ -3389,7 +3440,7 @@ static int trace__pgfault(struct trace *trace, if (ttrace == NULL) goto out_put; - if (evsel->core.attr.config == PERF_COUNT_SW_PAGE_FAULTS_MAJ) { + if (sample->evsel->core.attr.config == PERF_COUNT_SW_PAGE_FAULTS_MAJ) { ttrace->pfmaj++; trace->pfmaj++; } else { @@ -3402,10 +3453,11 @@ static int trace__pgfault(struct trace *trace, thread__find_symbol(thread, sample->cpumode, sample->ip, &al); - trace__fprintf_entry_head(trace, thread, 0, true, sample->time, trace->output); + trace__fprintf_entry_head(trace, thread, 0, true, sample->time, + sample->cpu, trace->output); fprintf(trace->output, "%sfault [", - evsel->core.attr.config == PERF_COUNT_SW_PAGE_FAULTS_MAJ ? + sample->evsel->core.attr.config == PERF_COUNT_SW_PAGE_FAULTS_MAJ ? "maj" : "min"); print_location(trace->output, sample, &al, false, true); @@ -3430,7 +3482,8 @@ static int trace__pgfault(struct trace *trace, if (callchain_ret > 0) trace__fprintf_callchain(trace, sample); else if (callchain_ret < 0) - pr_err("Problem processing %s callchain, skipping...\n", evsel__name(evsel)); + pr_err("Problem processing %s callchain, skipping...\n", + evsel__name(sample->evsel)); ++trace->nr_events_printed; out: @@ -3442,7 +3495,6 @@ static int trace__pgfault(struct trace *trace, } static void trace__set_base_time(struct trace *trace, - struct evsel *evsel, struct perf_sample *sample) { /* @@ -3454,17 +3506,17 @@ static void trace__set_base_time(struct trace *trace, * appears in our event stream (vfs_getname comes to mind). */ if (trace->base_time == 0 && !trace->full_time && - (evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) + (sample->evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) trace->base_time = sample->time; } static int trace__process_sample(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct trace *trace = container_of(tool, struct trace, tool); + struct evsel *evsel = sample->evsel; struct thread *thread; int err = 0; @@ -3474,11 +3526,11 @@ static int trace__process_sample(const struct perf_tool *tool, if (thread && thread__is_filtered(thread)) goto out; - trace__set_base_time(trace, evsel, sample); + trace__set_base_time(trace, sample); if (handler) { ++trace->nr_events; - handler(trace, evsel, event, sample); + handler(trace, event, sample); } out: thread__put(thread); @@ -3581,7 +3633,7 @@ static bool evlist__add_vfs_getname(struct evlist *evlist) list_del_init(&evsel->core.node); evsel->evlist = NULL; - evsel__delete(evsel); + evsel__put(evsel); } return found; @@ -3620,32 +3672,37 @@ static void evlist__free_syscall_tp_fields(struct evlist *evlist) static void trace__handle_event(struct trace *trace, union perf_event *event, struct perf_sample *sample) { const u32 type = event->header.type; - struct evsel *evsel; if (type != PERF_RECORD_SAMPLE) { trace__process_event(trace, trace->host, event, sample); return; } - evsel = evlist__id2evsel(trace->evlist, sample->id); - if (evsel == NULL) { + if (sample->evsel == NULL) { + sample->evsel = evlist__id2evsel(trace->evlist, sample->id); + if (sample->evsel) + evsel__get(sample->evsel); + } + + if (sample->evsel == NULL) { fprintf(trace->output, "Unknown tp ID %" PRIu64 ", skipping...\n", sample->id); return; } - if (evswitch__discard(&trace->evswitch, evsel)) + if (evswitch__discard(&trace->evswitch, sample->evsel)) return; - trace__set_base_time(trace, evsel, sample); + trace__set_base_time(trace, sample); - if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT && + if (sample->evsel->core.attr.type == PERF_TYPE_TRACEPOINT && sample->raw_data == NULL) { fprintf(trace->output, "%s sample with no payload for tid: %d, cpu %d, raw_size=%d, skipping...\n", - evsel__name(evsel), sample->tid, + evsel__name(sample->evsel), sample->tid, sample->cpu, sample->raw_size); } else { - tracepoint_handler handler = evsel->handler; - handler(trace, evsel, event, sample); + tracepoint_handler handler = sample->evsel->handler; + + handler(trace, event, sample); } if (trace->nr_events_printed >= trace->max_events && trace->max_events != ULONG_MAX) @@ -3695,9 +3752,9 @@ static int trace__add_syscall_newtp(struct trace *trace) return ret; out_delete_sys_exit: - evsel__delete_priv(sys_exit); + evsel__put_and_free_priv(sys_exit); out_delete_sys_enter: - evsel__delete_priv(sys_enter); + evsel__put_and_free_priv(sys_enter); goto out; } @@ -4162,7 +4219,7 @@ static int trace__set_filter_pids(struct trace *trace) err = augmented_syscalls__set_filter_pids(trace->filter_pids.nr, trace->filter_pids.entries); } - } else if (perf_thread_map__pid(trace->evlist->core.threads, 0) == -1) { + } else if (perf_thread_map__pid(evlist__core(trace->evlist)->threads, 0) == -1) { err = trace__set_filter_loop_pids(trace); } @@ -4385,7 +4442,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_bpf) { if (trace_prepare_bpf_summary(trace->summary_mode) < 0) - goto out_delete_evlist; + goto out_put_evlist; if (trace->summary_only) goto create_maps; @@ -4453,19 +4510,19 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = evlist__create_maps(evlist, &trace->opts.target); if (err < 0) { fprintf(trace->output, "Problems parsing the target to trace, check your options!\n"); - goto out_delete_evlist; + goto out_put_evlist; } err = trace__symbols_init(trace, argc, argv, evlist); if (err < 0) { fprintf(trace->output, "Problems initializing symbol libraries!\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (trace->summary_mode == SUMMARY__BY_TOTAL && !trace->summary_bpf) { trace->syscall_stats = alloc_syscall_stats(); if (!trace->syscall_stats) - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &trace->opts, &callchain_param); @@ -4474,9 +4531,9 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = evlist__prepare_workload(evlist, &trace->opts.target, argv, false, NULL); if (err < 0) { fprintf(trace->output, "Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } - workload_pid = evlist->workload.pid; + workload_pid = evlist__workload_pid(evlist); } err = evlist__open(evlist); @@ -4522,13 +4579,13 @@ static int trace__run(struct trace *trace, int argc, const char **argv) err = trace__expand_filters(trace, &evsel); if (err) - goto out_delete_evlist; + goto out_put_evlist; err = evlist__apply_filters(evlist, &evsel, &trace->opts.target); if (err < 0) goto out_error_apply_filters; if (!trace->summary_only || !trace->summary_bpf) { - err = evlist__mmap(evlist, trace->opts.mmap_pages); + err = evlist__do_mmap(evlist, trace->opts.mmap_pages); if (err < 0) goto out_error_mmap; } @@ -4547,8 +4604,8 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_bpf) trace_start_bpf_summary(); - trace->multiple_threads = perf_thread_map__pid(evlist->core.threads, 0) == -1 || - perf_thread_map__nr(evlist->core.threads) > 1 || + trace->multiple_threads = perf_thread_map__pid(evlist__core(evlist)->threads, 0) == -1 || + perf_thread_map__nr(evlist__core(evlist)->threads) > 1 || evlist__first(evlist)->core.attr.inherit; /* @@ -4565,11 +4622,11 @@ static int trace__run(struct trace *trace, int argc, const char **argv) again: before = trace->nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -4639,12 +4696,12 @@ static int trace__run(struct trace *trace, int argc, const char **argv) } } -out_delete_evlist: +out_put_evlist: trace_cleanup_bpf_summary(); delete_syscall_stats(trace->syscall_stats); trace__symbols__exit(trace); evlist__free_syscall_tp_fields(evlist); - evlist__delete(evlist); + evlist__put(evlist); cgroup__put(trace->cgroup); trace->evlist = NULL; trace->live = false; @@ -4669,21 +4726,21 @@ static int trace__run(struct trace *trace, int argc, const char **argv) out_error: fprintf(trace->output, "%s\n", errbuf); - goto out_delete_evlist; + goto out_put_evlist; out_error_apply_filters: fprintf(trace->output, "Failed to set filter \"%s\" on event %s: %m\n", evsel->filter, evsel__name(evsel)); - goto out_delete_evlist; + goto out_put_evlist; } out_error_mem: fprintf(trace->output, "Not enough memory to run!\n"); - goto out_delete_evlist; + goto out_put_evlist; out_errno: fprintf(trace->output, "%m\n"); - goto out_delete_evlist; + goto out_put_evlist; } static int trace__replay(struct trace *trace) @@ -4887,7 +4944,9 @@ static size_t syscall__dump_stats(struct trace *trace, int e_machine, FILE *fp, for (e = 0; e < stats->max_errno; ++e) { if (stats->errnos[e] != 0) - fprintf(fp, "\t\t\t\t%s: %d\n", perf_env__arch_strerrno(trace->host->env, e + 1), stats->errnos[e]); + fprintf(fp, "\t\t\t\t%s: %d\n", + perf_env__arch_strerrno(e_machine, e + 1), + stats->errnos[e]); } } lines++; @@ -5269,7 +5328,7 @@ static int trace__parse_cgroups(const struct option *opt, const char *str, int u { struct trace *trace = opt->value; - if (!list_empty(&trace->evlist->core.entries)) { + if (!list_empty(&evlist__core(trace->evlist)->entries)) { struct option o = { .value = &trace->evlist, }; @@ -5361,7 +5420,7 @@ static void trace__exit(struct trace *trace) zfree(&trace->syscalls.table); } zfree(&trace->perfconfig_events); - evlist__delete(trace->evlist); + evlist__put(trace->evlist); trace->evlist = NULL; ordered_events__free(&trace->oe.data); #ifdef HAVE_LIBBPF_SUPPORT @@ -5429,6 +5488,7 @@ int cmd_trace(int argc, const char **argv) OPT_CALLBACK('m', "mmap-pages", &trace.opts.mmap_pages, "pages", "number of mmap data pages", evlist__parse_mmap_pages), OPT_STRING('u', "uid", &trace.uid_str, "user", "user to profile"), + OPT_BOOLEAN(0, "show-cpu", &trace.show_cpu, "show cpu id"), OPT_CALLBACK(0, "duration", &trace, "float", "show only events with duration > N.M ms", trace__set_duration), @@ -5542,7 +5602,7 @@ int cmd_trace(int argc, const char **argv) * .perfconfig trace.add_events, and filter those out. */ if (!trace.trace_syscalls && !trace.trace_pgfaults && - trace.evlist->core.nr_entries == 0 /* Was --events used? */) { + evlist__nr_entries(trace.evlist) == 0 /* Was --events used? */) { trace.trace_syscalls = true; } /* @@ -5563,6 +5623,9 @@ int cmd_trace(int argc, const char **argv) goto out; } + if (trace.show_cpu) + trace.opts.sample_cpu = true; + if ((nr_cgroups || trace.cgroup) && !trace.opts.target.system_wide) { usage_with_options_msg(trace_usage, trace_options, "cgroup monitoring only available in system-wide mode"); @@ -5625,7 +5688,7 @@ int cmd_trace(int argc, const char **argv) symbol_conf.use_callchain = true; } - if (trace.evlist->core.nr_entries > 0) { + if (evlist__nr_entries(trace.evlist) > 0) { bool use_btf = false; evlist__set_default_evsel_handler(trace.evlist, trace__event_handler); diff --git a/tools/perf/pmu-events/Build b/tools/perf/pmu-events/Build index dc1df2d57ddc..372773b998e4 100644 --- a/tools/perf/pmu-events/Build +++ b/tools/perf/pmu-events/Build @@ -1,7 +1,12 @@ EMPTY_PMU_EVENTS_C = pmu-events/empty-pmu-events.c # pmu-events.c will be generated by jevents.py or copied from EMPTY_PMU_EVENTS_C PMU_EVENTS_C = $(OUTPUT)pmu-events/pmu-events.c +PMU_EVENTS_STRING_C = $(OUTPUT)pmu-events/pmu-events-string.c + pmu-events-y += pmu-events.o +ifneq ($(NO_JEVENTS),1) +pmu-events-y += pmu-events-string.o +endif # pmu-events.c file is generated in the OUTPUT directory so it needs a # separate rule to depend on it properly @@ -9,6 +14,10 @@ $(OUTPUT)pmu-events/pmu-events.o: $(PMU_EVENTS_C) $(call rule_mkdir) $(call if_changed_dep,cc_o_c) +$(OUTPUT)pmu-events/pmu-events-string.o: $(PMU_EVENTS_STRING_C) + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + # Message for $(call echo-cmd,cp), possibly remove the src file from # the destination to save space in the build log. quiet_cmd_cp = COPY $(patsubst %$<,%,$@) <- $< @@ -47,12 +56,12 @@ $(LEGACY_CACHE_JSON): $(LEGACY_CACHE_PY) # Python to generate architectural metrics GEN_METRIC_DEPS := pmu-events/metric.py pmu-events/common_metrics.py # Functions to extract the model from an extra-metrics.json or extra-metricgroups.json path. -model_name = $(shell echo $(1)|sed -e 's@.\+/\(.*\)/extra-metric.*\.json@\1@') -vendor_name = $(shell echo $(1)|sed -e 's@.\+/\(.*\)/[^/]*/extra-metric.*\.json@\1@') +model_name = $(notdir $(patsubst %/,%,$(dir $(1)))) +vendor_name = $(notdir $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $(1)))))) ifeq ($(JEVENTS_ARCH),$(filter $(JEVENTS_ARCH),x86 all)) # Generate AMD Json -ZENS = $(shell ls -d pmu-events/arch/x86/amdzen*) +ZENS := $(shell ls -d pmu-events/arch/x86/amdzen*) ZEN_METRICS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metrics.json) ZEN_METRICGROUPS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metricgroups.json) GEN_JSON += $(ZEN_METRICS) $(ZEN_METRICGROUPS) @@ -69,7 +78,7 @@ endif ifeq ($(JEVENTS_ARCH),$(filter $(JEVENTS_ARCH),arm64 all)) # Generate ARM Json -ARMS = $(shell ls -d pmu-events/arch/arm64/arm/*|grep -v cmn) +ARMS := $(shell ls -d pmu-events/arch/arm64/arm/*|grep -v cmn) ARM_METRICS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metrics.json) ARM_METRICGROUPS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metricgroups.json) GEN_JSON += $(ARM_METRICS) $(ARM_METRICGROUPS) @@ -86,7 +95,7 @@ endif ifeq ($(JEVENTS_ARCH),$(filter $(JEVENTS_ARCH),x86 all)) # Generate Intel Json -INTELS = $(shell ls -d pmu-events/arch/x86/*|grep -v amdzen|grep -v mapfile.csv) +INTELS := $(shell ls -d pmu-events/arch/x86/*|grep -v amdzen|grep -v mapfile.csv) INTEL_METRICS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metrics.json) INTEL_METRICGROUPS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metricgroups.json) GEN_JSON += $(INTEL_METRICS) $(INTEL_METRICGROUPS) @@ -118,6 +127,7 @@ CUR_OUT_JSON := $(shell [ -d $(OUT_DIR) ] && find $(OUT_DIR) -type f) # Things in the OUTPUT directory but shouldn't be there as computed by # OUT_JSON and GEN_JSON. + ORPHAN_FILES := $(filter-out $(OUT_JSON) $(GEN_JSON),$(CUR_OUT_JSON)) # Message for $(call echo-cmd,mkd). There is already a mkdir message @@ -224,6 +234,10 @@ endif # and inputs are dependencies. $(PMU_EVENTS_C): $(JEVENTS_DEPS) $(call rule_mkdir) - $(Q)$(call echo-cmd,gen)$(PYTHON) $(JEVENTS_PY) $(JEVENTS_ARCH) $(JEVENTS_MODEL) $(OUT_DIR) $@ + $(Q)$(call echo-cmd,gen)$(PYTHON) $(JEVENTS_PY) $(JEVENTS_ARCH) $(JEVENTS_MODEL) \ + $(OUT_DIR) $(PMU_EVENTS_C) $(PMU_EVENTS_STRING_C) + +$(PMU_EVENTS_STRING_C): $(PMU_EVENTS_C) + @: endif # ifeq ($(NO_JEVENTS),1) diff --git a/tools/perf/pmu-events/amd_metrics.py b/tools/perf/pmu-events/amd_metrics.py index e2defaffde3e..dccfcacaf148 100755 --- a/tools/perf/pmu-events/amd_metrics.py +++ b/tools/perf/pmu-events/amd_metrics.py @@ -265,10 +265,66 @@ def AmdDtlb() -> Optional[MetricGroup]: ], description="Data TLB metrics") +def AmdIotlb() -> Optional[MetricGroup]: + global _zen_model + if _zen_model < 2: + return None + + # On AMD, the pde events cover both 2M and 1G pages. + total_hit = Event("amd_iommu/mem_iommu_tlb_pte_hit/") + Event( + "amd_iommu/mem_iommu_tlb_pde_hit/" + ) + total_miss = Event("amd_iommu/mem_iommu_tlb_pte_mis/") + Event( + "amd_iommu/mem_iommu_tlb_pde_mis/" + ) + miss_rate = d_ratio(total_miss, total_miss + total_hit) + + interrupt_cache_hit = Event("amd_iommu/int_dte_hit/") + interrupt_cache_miss = Event("amd_iommu/int_dte_mis/") + interrupt_cache_lookup = interrupt_cache_hit + interrupt_cache_miss + interrupt_cache_miss_rate = d_ratio( + interrupt_cache_miss, interrupt_cache_miss + interrupt_cache_hit + ) + + return MetricGroup( + "iotlb", + [ + Metric("iotlb_total_hit", "IOTLB total hit", total_hit, "hits"), + Metric("iotlb_total_miss", "IOTLB total miss", total_miss, "misses"), + Metric("iotlb_miss_rate", "IOTLB miss rate", miss_rate, "100%"), + Metric( + "iotlb_interrupt_cache_hit", + "IOTLB interrupt cache hit", + interrupt_cache_hit, + "hits", + ), + Metric( + "iotlb_interrupt_cache_miss", + "IOTLB interrupt cache miss", + interrupt_cache_miss, + "misses", + ), + Metric( + "iotlb_interrupt_cache_lookup", + "IOTLB interrupt cache lookup", + interrupt_cache_lookup, + "lookups", + ), + Metric( + "iotlb_interrupt_cache_miss_rate", + "IOTLB interrupt cache miss rate", + interrupt_cache_miss_rate, + "100%", + ), + ], + description="IOMMU TLB metrics", + ) + + def AmdItlb(): global _zen_model l2h = Event("bp_l1_tlb_miss_l2_tlb_hit", "bp_l1_tlb_miss_l2_hit") - l2m = Event("l2_itlb_misses") + l2m = Event("bp_l1_tlb_miss_l2_tlb_miss.all", "l2_itlb_misses",) l2r = l2h + l2m itlb_l1_mg = None @@ -473,6 +529,7 @@ def dir_path(path: str) -> str: AmdBr(), AmdCtxSw(), AmdDtlb(), + AmdIotlb(), AmdItlb(), AmdLdSt(), AmdUpc(), diff --git a/tools/perf/pmu-events/arch/x86/alderlake/cache.json b/tools/perf/pmu-events/arch/x86/alderlake/cache.json index 5d0d824f3e7e..e44e6b651d55 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/cache.json @@ -150,6 +150,16 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts the number of L2 Cache accesses that resulted in a miss. Counts on a per core basis.", "Counter": "0,1,2,3,4,5", @@ -259,6 +269,16 @@ "UMask": "0x21", "Unit": "cpu_core" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf", + "Unit": "cpu_core" + }, { "BriefDescription": "L2_RQSTS.HWPF_MISS", "Counter": "0,1,2,3", @@ -338,6 +358,16 @@ "UMask": "0x40", "Unit": "cpu_core" }, + { + "BriefDescription": "Cycles when L1D is locked", + "Counter": "0,1,2,3", + "EventCode": "0x42", + "EventName": "LOCK_CYCLES.CACHE_LOCK_DURATION", + "PublicDescription": "This event counts the number of cycles when the L1D is locked. It is a superset of the 0x1 mask (BUS_LOCK_CLOCKS.BUS_LOCK_DURATION).", + "SampleAfterValue": "2000003", + "UMask": "0x2", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.", "Counter": "0,1,2,3,4,5", @@ -853,6 +883,17 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of memory uops retired.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "PublicDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)", + "SampleAfterValue": "200003", + "UMask": "0x83", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of load uops retired.", "Counter": "0,1,2,3,4,5", @@ -875,6 +916,18 @@ "UMask": "0x82", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024. Only counts with PEBS enabled.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_1024", + "MSRIndex": "0x3F6", + "MSRValue": "0x400", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128. Only counts with PEBS enabled.", "Counter": "0,1", @@ -899,6 +952,18 @@ "UMask": "0x5", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048. Only counts with PEBS enabled.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_2048", + "MSRIndex": "0x3F6", + "MSRValue": "0x800", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256. Only counts with PEBS enabled.", "Counter": "0,1", @@ -981,6 +1046,16 @@ "UMask": "0x21", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of memory uops retired that were splits.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT", + "SampleAfterValue": "200003", + "UMask": "0x43", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of retired split load uops.", "Counter": "0,1,2,3,4,5", @@ -991,6 +1066,16 @@ "UMask": "0x41", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of retired split store uops.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT_STORES", + "SampleAfterValue": "200003", + "UMask": "0x42", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the total number of load and store uops retired that missed in the second level TLB.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/memory.json b/tools/perf/pmu-events/arch/x86/alderlake/memory.json index a0260d5b8619..f482c06ac728 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/memory.json @@ -9,6 +9,15 @@ "UMask": "0x6", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.ANY", + "SampleAfterValue": "1000003", + "UMask": "0x7f", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.", "Counter": "0,1,2,3,4,5", @@ -27,6 +36,15 @@ "UMask": "0xf4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.", "Counter": "0,1,2,3,4,5", @@ -36,6 +54,16 @@ "UMask": "0x81", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", "Counter": "0,1,2,3,4,5", @@ -46,6 +74,15 @@ "UMask": "0xc0", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.PGWALK", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.", "Counter": "0,1,2,3,4,5", @@ -55,6 +92,15 @@ "UMask": "0xa0", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store data forward block.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.ST_ADDR", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.", "Counter": "0,1,2,3,4,5", @@ -64,6 +110,24 @@ "UMask": "0x84", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json index 80cad3c49d20..1c292f29b0aa 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json @@ -349,6 +349,15 @@ "UMask": "0xfd", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of near relative JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.REL_JMP", + "SampleAfterValue": "200003", + "UMask": "0xdf", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_INST_RETIRED.NEAR_RETURN", "Counter": "0,1,2,3,4,5", @@ -359,6 +368,15 @@ "UMask": "0xf7", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of taken branch instructions retired.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_INST_RETIRED.COND_TAKEN", "Counter": "0,1,2,3,4,5", @@ -560,6 +578,15 @@ "UMask": "0xfe", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the total number of BTCLEARS.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xe8", + "EventName": "BTCLEAR.ANY", + "PublicDescription": "Counts the total number of BTCLEARS which occurs when the Branch Target Buffer (BTB) predicts a taken branch.", + "SampleAfterValue": "200003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Core clocks when the thread is in the C0.1 light-weight slower wakeup time but more power saving optimized state.", "Counter": "0,1,2,3,4,5,6,7", @@ -1214,6 +1241,24 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FAST", + "SampleAfterValue": "20003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of machines clears due to memory renaming.", "Counter": "0,1,2,3,4,5", @@ -1410,6 +1455,15 @@ "UMask": "0x4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "200003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/virtual-memory.json b/tools/perf/pmu-events/arch/x86/alderlake/virtual-memory.json index 132ce48af6d9..115bbc000a45 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/virtual-memory.json @@ -250,6 +250,15 @@ "UMask": "0x10", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.DTLB_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/cache.json b/tools/perf/pmu-events/arch/x86/alderlaken/cache.json index 1f97a4dc6fb1..0ffad953e752 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/cache.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/cache.json @@ -225,6 +225,16 @@ "SampleAfterValue": "20003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of memory uops retired.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "PublicDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)", + "SampleAfterValue": "200003", + "UMask": "0x83" + }, { "BriefDescription": "Counts the number of load uops retired.", "Counter": "0,1,2,3,4,5", @@ -245,6 +255,17 @@ "SampleAfterValue": "200003", "UMask": "0x82" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024. Only counts with PEBS enabled.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_1024", + "MSRIndex": "0x3F6", + "MSRValue": "0x400", + "SampleAfterValue": "1000003", + "UMask": "0x5" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128. Only counts with PEBS enabled.", "Counter": "0,1", @@ -267,6 +288,17 @@ "SampleAfterValue": "1000003", "UMask": "0x5" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048. Only counts with PEBS enabled.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_2048", + "MSRIndex": "0x3F6", + "MSRValue": "0x800", + "SampleAfterValue": "1000003", + "UMask": "0x5" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256. Only counts with PEBS enabled.", "Counter": "0,1", @@ -342,6 +374,15 @@ "SampleAfterValue": "200003", "UMask": "0x21" }, + { + "BriefDescription": "Counts the number of memory uops retired that were splits.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT", + "SampleAfterValue": "200003", + "UMask": "0x43" + }, { "BriefDescription": "Counts the number of retired split load uops.", "Counter": "0,1,2,3,4,5", @@ -351,6 +392,15 @@ "SampleAfterValue": "200003", "UMask": "0x41" }, + { + "BriefDescription": "Counts the number of retired split store uops.", + "Counter": "0,1,2,3,4,5", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT_STORES", + "SampleAfterValue": "200003", + "UMask": "0x42" + }, { "BriefDescription": "Counts the total number of load and store uops retired that missed in the second level TLB.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/memory.json b/tools/perf/pmu-events/arch/x86/alderlaken/memory.json index 049c5e2630d7..a0dd82ddd58a 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/memory.json @@ -1,4 +1,12 @@ [ + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.ANY", + "SampleAfterValue": "1000003", + "UMask": "0x7f" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.", "Counter": "0,1,2,3,4,5", @@ -15,6 +23,14 @@ "SampleAfterValue": "1000003", "UMask": "0xf4" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.", "Counter": "0,1,2,3,4,5", @@ -23,6 +39,15 @@ "SampleAfterValue": "1000003", "UMask": "0x81" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", "Counter": "0,1,2,3,4,5", @@ -32,6 +57,14 @@ "SampleAfterValue": "1000003", "UMask": "0xc0" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.PGWALK", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.", "Counter": "0,1,2,3,4,5", @@ -40,6 +73,14 @@ "SampleAfterValue": "1000003", "UMask": "0xa0" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store data forward block.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.ST_ADDR", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.", "Counter": "0,1,2,3,4,5", @@ -48,6 +89,22 @@ "SampleAfterValue": "1000003", "UMask": "0x84" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json index a13851071624..ad3acb109bd8 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json @@ -175,6 +175,14 @@ "SampleAfterValue": "200003", "UMask": "0xfd" }, + { + "BriefDescription": "Counts the number of near relative JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.REL_JMP", + "SampleAfterValue": "200003", + "UMask": "0xdf" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_INST_RETIRED.NEAR_RETURN", "Counter": "0,1,2,3,4,5", @@ -184,6 +192,14 @@ "SampleAfterValue": "200003", "UMask": "0xf7" }, + { + "BriefDescription": "Counts the number of taken branch instructions retired.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_INST_RETIRED.COND_TAKEN", "Counter": "0,1,2,3,4,5", @@ -293,6 +309,14 @@ "SampleAfterValue": "200003", "UMask": "0xfe" }, + { + "BriefDescription": "Counts the total number of BTCLEARS.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xe8", + "EventName": "BTCLEAR.ANY", + "PublicDescription": "Counts the total number of BTCLEARS which occurs when the Branch Target Buffer (BTB) predicts a taken branch.", + "SampleAfterValue": "200003" + }, { "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.THREAD]", "Counter": "Fixed counter 1", @@ -400,6 +424,22 @@ "SampleAfterValue": "20003", "UMask": "0x8" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FAST", + "SampleAfterValue": "20003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40" + }, { "BriefDescription": "Counts the number of machines clears due to memory renaming.", "Counter": "0,1,2,3,4,5", @@ -482,6 +522,14 @@ "SampleAfterValue": "200003", "UMask": "0x4" }, + { + "BriefDescription": "Counts the number issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "200003", + "UMask": "0x8" + }, { "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/virtual-memory.json b/tools/perf/pmu-events/arch/x86/alderlaken/virtual-memory.json index d9c737a17df0..56c1bfc38024 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/virtual-memory.json @@ -42,6 +42,14 @@ "SampleAfterValue": "200003", "UMask": "0xe" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x05", + "EventName": "LD_HEAD.DTLB_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/cache.json b/tools/perf/pmu-events/arch/x86/arrowlake/cache.json index 4c3aa1fab5a8..142f62c59531 100644 --- a/tools/perf/pmu-events/arch/x86/arrowlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/arrowlake/cache.json @@ -1,6 +1,6 @@ [ { - "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.", + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0x31", "EventName": "CORE_REJECT_L2Q.ANY", @@ -8,6 +8,15 @@ "SampleAfterValue": "1000003", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x31", + "EventName": "CORE_REJECT_L2Q.ANY", + "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q. It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link. The L2Q may also reject transactions from a core to insure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops. (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.", + "SampleAfterValue": "200003", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of L1D cacheline (dirty) evictions caused by load misses, stores, and prefetches.", "Counter": "0,1,2,3,4,5,6,7", @@ -310,6 +319,15 @@ "SampleAfterValue": "1000003", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x30", + "EventName": "L2_REJECT_XQ.ANY", + "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link. The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).", + "SampleAfterValue": "200003", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of L2 Cache Accesses Counts the total number of L2 Cache Accesses - sum of hits, misses, rejects front door requests for CRd/DRd/RFO/ItoM/L2 Prefetches only, per core event", "Counter": "0,1,2,3,4,5,6,7", @@ -339,6 +357,16 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts the number of L2 Cache Accesses that resulted in a Hit from a front door request only (does not include rejects or recycles), per core event", "Counter": "0,1,2,3,4,5,6,7", @@ -464,6 +492,16 @@ "UMask": "0x21", "Unit": "cpu_core" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Read requests with true-miss in L2 cache [This event is alias to L2_REQUEST.MISS]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -1126,6 +1164,15 @@ "UMask": "0x20", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of retired load ops that hit in the L3 cache in which a snoop was required and modified data was forwarded", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_HITM", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of retired load ops with an unknown source", "Counter": "0,1,2,3,4,5,6,7", @@ -1353,6 +1400,16 @@ "UMask": "0x83", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "SampleAfterValue": "200003", + "UMask": "0x83", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of load uops retired.", "Counter": "0,1,2,3,4,5,6,7", @@ -1393,6 +1450,18 @@ "UMask": "0x82", "Unit": "cpu_lowpower" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_1024", + "MSRIndex": "0x3F6", + "MSRValue": "0x400", + "SampleAfterValue": "200003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024. Only counts with PEBS enabled.", "Counter": "0,1", @@ -1453,6 +1522,18 @@ "UMask": "0x5", "Unit": "cpu_lowpower" }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048.", + "Counter": "0,1", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_2048", + "MSRIndex": "0x3F6", + "MSRValue": "0x800", + "SampleAfterValue": "200003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048. Only counts with PEBS enabled.", "Counter": "0,1", diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json b/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json index 3e68c2468f11..8dc3a11350c5 100644 --- a/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json @@ -510,6 +510,15 @@ "UMask": "0x1f", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on all floating point ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xf", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.", "Counter": "0,1,2,3,4,5,6,7", @@ -519,6 +528,15 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.", "Counter": "0,1,2,3,4,5,6,7", @@ -528,6 +546,15 @@ "UMask": "0x4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.", "Counter": "0,1,2,3,4,5,6,7", @@ -537,6 +564,15 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 3.", "Counter": "0,1,2,3,4,5,6,7", @@ -555,6 +591,15 @@ "UMask": "0x1e", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0xe", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", "Counter": "0,1,2,3,4,5,6,7", @@ -564,6 +609,15 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.STD", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of floating point operations retired that required microcode assist.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/memory.json b/tools/perf/pmu-events/arch/x86/arrowlake/memory.json index 05cc46518232..44922186c2b0 100644 --- a/tools/perf/pmu-events/arch/x86/arrowlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/arrowlake/memory.json @@ -173,6 +173,15 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", "Counter": "0,1,2,3,4,5,6,7", @@ -182,6 +191,15 @@ "UMask": "0x82", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of memory ordering machine clears triggered due to a snoop from an external agent. Does not count internally generated machine clears such as those due to disambiguations.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json b/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json index fb973c75be57..bdfee0347cc5 100644 --- a/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json @@ -209,7 +209,6 @@ "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD", "PublicDescription": "Counts taken forward conditional branch instructions retired. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x102", "Unit": "cpu_core" }, { @@ -376,7 +375,7 @@ "Unit": "cpu_lowpower" }, { - "BriefDescription": "Counts the number of taken branch instructions retired", + "BriefDescription": "Counts the number of near taken branch instructions retired", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0xc4", "EventName": "BR_INST_RETIRED.NEAR_TAKEN", @@ -422,6 +421,15 @@ "UMask": "0xfd", "Unit": "cpu_lowpower" }, + { + "BriefDescription": "Counts the number of relative JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.REL_JMP", + "SampleAfterValue": "200003", + "UMask": "0xdf", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of near relative JMP branch instructions retired.", "Counter": "0,1,2,3,4,5,6,7", @@ -431,6 +439,25 @@ "UMask": "0xdf", "Unit": "cpu_lowpower" }, + { + "BriefDescription": "Counts the number of taken branch instructions retired", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of taken branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "Errata": "ARL011", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -580,7 +607,7 @@ "EventName": "BR_MISP_RETIRED.COND_TAKEN_BWD_COST", "PublicDescription": "number of branch instructions retired that were mispredicted and taken backward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x8001", + "UMask": "0x41", "Unit": "cpu_core" }, { @@ -609,7 +636,7 @@ "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD_COST", "PublicDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x8002", + "UMask": "0x140", "Unit": "cpu_core" }, { @@ -745,11 +772,11 @@ "Unit": "cpu_core" }, { - "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS", + "BriefDescription": "This event counts the number of mispredicted ret instructions retired.", "Counter": "0,1,2,3,4,5,6,7,8,9", "EventCode": "0xc5", "EventName": "BR_MISP_RETIRED.RET", - "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. Available PDIST counters: 0,1", + "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0,1", "SampleAfterValue": "100007", "UMask": "0x8", "Unit": "cpu_core" @@ -1298,6 +1325,15 @@ "UMask": "0xff", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on all Integer ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xff", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on a load port.", "Counter": "0,1,2,3,4,5,6,7", @@ -1308,6 +1344,16 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on a load port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.LD", + "PublicDescription": "Counts the number of uops executed on a load port. This event counts for integer uops even if the destination is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on integer port 0.", "Counter": "0,1,2,3,4,5,6,7", @@ -1317,6 +1363,15 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on integer port 1.", "Counter": "0,1,2,3,4,5,6,7", @@ -1326,6 +1381,15 @@ "UMask": "0x10", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on integer port 2.", "Counter": "0,1,2,3,4,5,6,7", @@ -1335,6 +1399,15 @@ "UMask": "0x20", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on integer port 3.", "Counter": "0,1,2,3,4,5,6,7", @@ -1344,6 +1417,15 @@ "UMask": "0x40", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on integer port 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P3", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on integer port 0,1, 2, 3.", "Counter": "0,1,2,3,4,5,6,7", @@ -1353,6 +1435,15 @@ "UMask": "0x78", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0,1, 2, 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0x78", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on a Store address port.", "Counter": "0,1,2,3,4,5,6,7", @@ -1363,6 +1454,16 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on a Store address port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STA", + "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.", "Counter": "0,1,2,3,4,5,6,7", @@ -1372,6 +1473,15 @@ "UMask": "0x4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STD_JMP", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Number of vector integer instructions retired of 128-bit vector-width.", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -1663,6 +1773,33 @@ "UMask": "0x88", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FAST", + "SampleAfterValue": "20003", + "UMask": "0x10", + "Unit": "cpu_lowpower" + }, + { + "BriefDescription": "Counts number of virtual trap actually taken (e.g. highest priority event during retirement). It can count virtual trap from FPC port 0 or port 1 (x87/SSE) equally in a single counter.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of nukes due to memory renaming", "Counter": "0,1,2,3,4,5,6,7", @@ -1672,6 +1809,15 @@ "UMask": "0x10", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machines clears due to memory renaming.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x80", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", "Counter": "0,1,2,3,4,5,6,7", @@ -1969,6 +2115,15 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "200003", + "UMask": "0x8", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "Counts the number of issue slots where no uop could issue due to an IQ scoreboard that stalls allocation until a specified older uop retires or (in the case of jump scoreboard) executes. Commonly executed instructions with IQ scoreboards include LFENCE and MFENCE.", "Counter": "0,1,2,3,4,5,6,7", @@ -1988,6 +2143,16 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.NON_C01_MS_SCB", + "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.", + "SampleAfterValue": "200003", + "UMask": "0x2", + "Unit": "cpu_lowpower" + }, { "BriefDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.", "Counter": "0,1,2,3,4,5,6,7,8,9", diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/cache.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/cache.json index ecb7dc252208..480874ccbd09 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/cache.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/cache.json @@ -1,4 +1,140 @@ [ + { + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x31", + "EventName": "CORE_REJECT_L2Q.ANY", + "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full condition which likely indicates back pressure from L2Q. It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link. The L2Q may also reject transactions from a core to ensure fairness between cores, or to delay a core's dirty eviction when the address conflicts with incoming external snoops. Note that L2 prefetcher requests that are dropped are not counted by this event. Counts on a per core basis.", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x30", + "EventName": "L2_REJECT_XQ.ANY", + "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link. The IDI link is a central on-die protocol for memory and coherence traffic between the core and the uncore. It is highly optimized for efficiency, bandwidth, and coherency, using multiple channels and flow control mechanisms to manage high-throughput, low-latency data and protocol communication within the chip. The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door requests for Code Read, Data Read, RFO, ITOM, and L2 Prefetches. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.ALL", + "SampleAfterValue": "1000003", + "UMask": "0x1ff" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Code Read requests. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_CODE_RD", + "SampleAfterValue": "1000003", + "UMask": "0xc4" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Code Read requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_CODE_RD_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x84" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Code Read requests that resulted in a Miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_CODE_RD_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x44" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Data Read requests. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_DATA_RD", + "SampleAfterValue": "1000003", + "UMask": "0xc1" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Data Read requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_DATA_RD_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x81" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Data Read requests that resulted in a Miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_DATA_RD_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x41" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand RFO requests. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_RFO", + "SampleAfterValue": "1000003", + "UMask": "0xc2" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand RFO requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_RFO_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x82" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand RFO requests that resulted in a Miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_RFO_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x42" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door requests that resulted in a Hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "SampleAfterValue": "1000003", + "UMask": "0x1bf" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests, including hits and misses. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HWPF", + "SampleAfterValue": "1000003", + "UMask": "0xc8" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests that result in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HWPF_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x88" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests that result in a miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HWPF_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x48" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door requests that resulted in a Miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.MISS", + "SampleAfterValue": "1000003", + "UMask": "0x17f" + }, { "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.", "Counter": "0,1,2,3,4,5,6,7", @@ -17,6 +153,231 @@ "SampleAfterValue": "1000003", "UMask": "0x4f" }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which hit in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.L2_HIT", + "PublicDescription": "Counts the number of cycles a core is stalled due to a demand load which hit in the L2 cache.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x6" + }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC, no snoop was required. LLC provided data.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_HIT_NOSNOOP", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC, a snoop was required, the snoop misses or the snoop hits but no fwd. LLC provides the data.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_HIT_SNOOP", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which missed all the caches. DRAM, MMIO or other LOCAL memory type provides the data.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_MISS_LOCALMEM", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which missed all the caches, a snoop was required, and hits in other core or module on same die. Another core provides the data with a fwd, no fwd, or hitM.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_MISS_OTHERMOD", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the total number of load ops retired that miss the L3 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.ALL", + "PublicDescription": "Counts the total number of load ops retired that miss the L3 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xff" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in DRAM", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM", + "PublicDescription": "Counts the number of load ops retired that miss the L3 cache and hit in DRAM Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote DRAM", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_OR_NOFWD", + "PublicDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote DRAM, OR had a Remote snoop miss/no fwd and hit in the Local DRAM. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote Cache and modified data was forwarded", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_HITM", + "PublicDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote Cache and modified data was forwarded Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote Cache and non-modified data was forwarded", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_NONM", + "PublicDescription": "Counts the number of load ops retired that miss the L3 cache and hit in a Remote Cache and non-modified data was forwarded Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and modified data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_HITM", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and modified data was forwarded. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and no data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_NO_FWD", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and no data was forwarded. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and non-modified data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_WITH_FWD", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and non-modified data was forwarded. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit the L1 data cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L1_HIT", + "PublicDescription": "Counts the number of load ops retired that hit the L1 data cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss in the L1 data cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L1_MISS", + "PublicDescription": "Counts the number of load ops retired that miss in the L1 data cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L2_HIT", + "PublicDescription": "Counts the number of load ops retired that hit in the L2 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of load ops retired that miss in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L2_MISS", + "PublicDescription": "Counts the number of load ops retired that miss in the L2 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L3_HIT", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1c" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which no snoop was required.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L3_HIT_NO_SNOOP", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache in which no snoop was required. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and it hit and forwarded data, it hit and did not forward data, or it hit and the forwarded data was modified.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_UOPS_RETIRED.L3_HIT_SNOOP_HIT", + "PublicDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and it hit and forwarded data, it hit and did not forward data, or it hit and the forwarded data was modified. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of cycles that uops are blocked for any of the following reasons: load buffer, store buffer or RSV full.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x04", + "EventName": "MEM_SCHEDULER_BLOCK.ALL", + "SampleAfterValue": "1000003", + "UMask": "0x7" + }, + { + "BriefDescription": "Counts the number of cycles that uops are blocked due to a load buffer full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x04", + "EventName": "MEM_SCHEDULER_BLOCK.LD_BUF", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of cycles that uops are blocked due to an RSV full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x04", + "EventName": "MEM_SCHEDULER_BLOCK.RSV", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of cycles that uops are blocked due to a store buffer full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x04", + "EventName": "MEM_SCHEDULER_BLOCK.ST_BUF", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "PublicDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST). Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x83" + }, { "BriefDescription": "Counts the number of load ops retired.", "Counter": "0,1,2,3,4,5,6,7", @@ -36,121 +397,213 @@ "UMask": "0x82" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_1024", "MSRIndex": "0x3F6", "MSRValue": "0x400", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_128", "MSRIndex": "0x3F6", "MSRValue": "0x80", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 16.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_16", "MSRIndex": "0x3F6", "MSRValue": "0x10", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 16. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_2048", "MSRIndex": "0x3F6", "MSRValue": "0x800", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 2048. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_256", "MSRIndex": "0x3F6", "MSRValue": "0x100", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 32.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_32", "MSRIndex": "0x3F6", "MSRValue": "0x20", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 32. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 4.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_4", "MSRIndex": "0x3F6", "MSRValue": "0x4", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 4. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 512.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_512", "MSRIndex": "0x3F6", "MSRValue": "0x200", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 512. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 64.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_64", "MSRIndex": "0x3F6", "MSRValue": "0x40", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 64. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", - "Counter": "0,1", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 8.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_8", "MSRIndex": "0x3F6", "MSRValue": "0x8", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 8. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5" }, { - "BriefDescription": "Counts the number of stores uops retired same as MEM_UOPS_RETIRED.ALL_STORES", + "BriefDescription": "Counts the number of load uops retired that performed one or more locks", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOCK_LOADS", + "PublicDescription": "Counts the number of load uops retired that performed one or more locks Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x21" + }, + { + "BriefDescription": "Counts the number of load uops retired that were correctly predicated by the memory renaming (MRN) feature.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.MRN_LOADS", + "PublicDescription": "Counts the number of load uops retired that were correctly predicated by the memory renaming (MRN) feature. MRN loads are a part of the memory renaming process that optimizes data forwarding between stores and loads, reducing latency and improving performance. This involves tagging stores that forward to loads and copying the physical source (the actual data) directly from the store to the load. The MRN'd load can complete without accessing the memory, thus reducing load-to-use latency. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x9" + }, + { + "BriefDescription": "Counts the number of store uops retired that were correctly tagged by the memory renaming (MRN) feature.", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.MRN_STORES", + "PublicDescription": "Counts the number of store uops retired that were correctly tagged by the memory renaming (MRN) feature. MRN stores are a part of the memory renaming process that optimizes data forwarding between stores and loads, reducing latency and improving performance. This involves tagging stores that forward to loads and copying the physical source (the actual data) directly from the store to the load. The MRN store ensures that it's data is accurately forwarded to matching subsequent loads. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xa" + }, + { + "BriefDescription": "Counts the number of memory uops retired that were splits.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT", + "PublicDescription": "Counts the number of memory uops retired that were splits. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x43" + }, + { + "BriefDescription": "Counts the number of retired split load uops.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT_LOADS", + "PublicDescription": "Counts the number of retired split load uops. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x41" + }, + { + "BriefDescription": "Counts the number of retired split store uops.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.SPLIT_STORES", + "PublicDescription": "Counts the number of retired split store uops. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x42" + }, + { + "BriefDescription": "Counts the number of memory uops retired that missed in the second level TLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.STLB_MISS", + "PublicDescription": "Counts the number of memory uops retired that missed in the second level TLB. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x13" + }, + { + "BriefDescription": "Counts the number of load uops retired that miss in the second Level TLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.STLB_MISS_LOADS", + "PublicDescription": "Counts the number of load uops retired that miss in the second Level TLB. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x11" + }, + { + "BriefDescription": "Counts the number of store uops retired that miss in the second level TLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.STLB_MISS_STORES", + "PublicDescription": "Counts the number of store uops retired that miss in the second level TLB. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x12" + }, + { + "BriefDescription": "Counts the number of stores uops retired.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", + "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.STORE_LATENCY", - "PublicDescription": "Counts the number of stores uops retired same as MEM_UOPS_RETIRED.ALL_STORES Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of stores uops retired. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x6" }, @@ -175,5 +628,13 @@ "PublicDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response. Available PDIST counters: 0", "SampleAfterValue": "100003", "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to an icache miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.ICACHE", + "SampleAfterValue": "1000003", + "UMask": "0x20" } ] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/counter.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/counter.json index a0eaf5b6f2bc..03c2e5f77204 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/counter.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/counter.json @@ -1,7 +1,82 @@ [ { "Unit": "core", - "CountersNumFixed": "3", - "CountersNumGeneric": "39" + "CountersNumFixed": "7", + "CountersNumGeneric": "8" + }, + { + "Unit": "B2CMI", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "CHA", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "IMC", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "B2HOT", + "CountersNumFixed": "0", + "CountersNumGeneric": 4 + }, + { + "Unit": "IIO", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "IRP", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "UPI", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "B2UPI", + "CountersNumFixed": "0", + "CountersNumGeneric": 4 + }, + { + "Unit": "B2CXL", + "CountersNumFixed": "0", + "CountersNumGeneric": 4 + }, + { + "Unit": "UBOX", + "CountersNumFixed": "0", + "CountersNumGeneric": "2" + }, + { + "Unit": "PCU", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "CHACMS", + "CountersNumFixed": "0", + "CountersNumGeneric": "4" + }, + { + "Unit": "MDF", + "CountersNumFixed": "0", + "CountersNumGeneric": 4 + }, + { + "Unit": "CXLCM", + "CountersNumFixed": "0", + "CountersNumGeneric": 8 + }, + { + "Unit": "CXLDP", + "CountersNumFixed": "0", + "CountersNumGeneric": 4 } ] \ No newline at end of file diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/cwf-metrics.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/cwf-metrics.json new file mode 100644 index 000000000000..bff8841096cd --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/cwf-metrics.json @@ -0,0 +1,310 @@ +[ + { + "BriefDescription": "C1 residency percent per core", + "MetricExpr": "cstate_core@c1\\-residency@ / msr@tsc@", + "MetricGroup": "Power", + "MetricName": "C1_Core_Residency", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "C2 residency percent per package", + "MetricExpr": "cstate_pkg@c2\\-residency@ / msr@tsc@", + "MetricGroup": "Power", + "MetricName": "C2_Pkg_Residency", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "C6 residency percent per core", + "MetricExpr": "cstate_core@c6\\-residency@ / msr@tsc@", + "MetricGroup": "Power", + "MetricName": "C6_Core_Residency", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "C6 residency percent per package", + "MetricExpr": "cstate_pkg@c6\\-residency@ / msr@tsc@", + "MetricGroup": "Power", + "MetricName": "C6_Pkg_Residency", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles", + "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", + "MetricName": "cpi", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "The average number of cores that are in cstate C0 as observed by the power control unit (PCU)", + "MetricExpr": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C0 / pcu_0@UNC_P_CLOCKTICKS@ * #num_packages", + "MetricName": "cpu_cstate_c0" + }, + { + "BriefDescription": "The average number of cores are in cstate C6 as observed by the power control unit (PCU)", + "MetricExpr": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C6 / pcu_0@UNC_P_CLOCKTICKS@ * #num_packages", + "MetricName": "cpu_cstate_c6" + }, + { + "BriefDescription": "CPU operating frequency (in GHz)", + "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ / 1e9", + "MetricName": "cpu_operating_frequency", + "ScaleUnit": "1GHz" + }, + { + "BriefDescription": "Percentage of time spent in the active CPU power state C0", + "MetricExpr": "CPU_CLK_UNHALTED.REF_TSC / msr@tsc@", + "MetricName": "cpu_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", + "MetricName": "dtlb_2nd_level_load_mpi", + "PublicDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions", + "MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", + "MetricName": "dtlb_2nd_level_store_mpi", + "PublicDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Bandwidth observed by the integrated I/O traffic controller (IIO) of IO reads that are initiated by end device controllers that are requesting memory from the CPU", + "MetricExpr": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.ALL_PARTS * 4 / 1e6 / duration_time", + "MetricName": "iio_bandwidth_read", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth observed by the integrated I/O traffic controller (IIO) of IO writes that are initiated by end device controllers that are writing memory to the CPU", + "MetricExpr": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.ALL_PARTS * 4 / 1e6 / duration_time", + "MetricName": "iio_bandwidth_write", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_read", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of inbound IO reads that are initiated by end device controllers that are requesting memory from the CPU and miss the L3 cache", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_read_l3_miss", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the local CPU socket", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR_LOCAL * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_read_local", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from a remote CPU socket", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR_REMOTE * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_read_remote", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_ITOM + UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR) * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_write", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of inbound IO writes that are initiated by end device controllers that are writing memory to the CPU", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR) * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_write_l3_miss", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the local CPU socket", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_ITOM_LOCAL + UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR_LOCAL) * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_write_local", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to a remote CPU socket", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_ITOM_REMOTE + UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR_REMOTE) * 64 / 1e6 / duration_time", + "MetricName": "io_bandwidth_write_remote", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "The percent of inbound full cache line writes initiated by IO that miss the L3 cache", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_MISS_ITOM / UNC_CHA_TOR_INSERTS.IO_ITOM", + "MetricName": "io_full_write_l3_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The number of times per second that ownership of a cacheline was stolen from the integrated IO controller before it was able to write back the modified line", + "MetricExpr": "(UNC_I_MISC1.LOST_FWD + UNC_I_MISC1.SEC_RCVD_INVLD) / duration_time", + "MetricName": "io_lost_fwd", + "ScaleUnit": "1per_sec" + }, + { + "BriefDescription": "Message Signaled Interrupts (MSI) per second sent by the integrated I/O traffic controller (IIO) to System Configuration Controller (Ubox)", + "MetricExpr": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX_POSTED / duration_time", + "MetricName": "io_msi", + "ScaleUnit": "1per_sec" + }, + { + "BriefDescription": "", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_RFO) / duration_time", + "MetricName": "io_number_of_partial_pci_writes", + "ScaleUnit": "1per_sec" + }, + { + "BriefDescription": "The percent of inbound partial writes initiated by IO that miss the L3 cache", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_RFO) / (UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_RFO)", + "MetricName": "io_partial_write_l3_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The percent of inbound reads initiated by IO that miss the L3 cache", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR / UNC_CHA_TOR_INSERTS.IO_PCIRDCUR", + "MetricName": "io_read_l3_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions", + "MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", + "MetricName": "itlb_2nd_level_mpi", + "PublicDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of code read requests missing in L1 instruction cache (includes prefetches) to the total number of completed instructions", + "MetricExpr": "ICACHE.MISSES / INST_RETIRED.ANY", + "MetricName": "l1_i_code_read_misses_with_prefetches_per_instr", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of demand load requests hitting in L1 data cache to the total number of completed instructions", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L1_HIT / INST_RETIRED.ANY", + "MetricName": "l1d_demand_data_read_hits_per_instr", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of completed demand load requests hitting in L2 cache to the total number of completed instructions", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_HIT / INST_RETIRED.ANY", + "MetricName": "l2_demand_data_read_hits_per_instr", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of completed data read request missing L2 cache to the total number of completed instructions", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", + "MetricName": "l2_demand_data_read_mpi", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IA_MISS_CRD + UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF) / INST_RETIRED.ANY", + "MetricName": "llc_code_read_mpi_demand_plus_prefetch", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Ratio of number of data read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF + UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA) / INST_RETIRED.ANY", + "MetricName": "llc_data_read_mpi_demand_plus_prefetch", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) in nano seconds", + "MetricExpr": "1e9 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT) / (UNC_CHA_CLOCKTICKS / (source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT) * #num_packages)) * duration_time", + "MetricName": "llc_demand_data_read_miss_latency", + "ScaleUnit": "1ns" + }, + { + "BriefDescription": "Bandwidth (MB/sec) of read requests that miss the last level cache (LLC) and go to local memory", + "MetricExpr": "UNC_CHA_REQUESTS.READS_LOCAL * 64 / 1e6 / duration_time", + "MetricName": "llc_miss_local_memory_bandwidth_read", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth (MB/sec) of write requests that miss the last level cache (LLC) and go to local memory", + "MetricExpr": "UNC_CHA_REQUESTS.WRITES_LOCAL * 64 / 1e6 / duration_time", + "MetricName": "llc_miss_local_memory_bandwidth_write", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth (MB/sec) of read requests that miss the last level cache (LLC) and go to remote memory", + "MetricExpr": "UNC_CHA_REQUESTS.READS_REMOTE * 64 / 1e6 / duration_time", + "MetricName": "llc_miss_remote_memory_bandwidth_read", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Bandwidth (MB/sec) of write requests that miss the last level cache (LLC) and go to remote memory", + "MetricExpr": "UNC_CHA_REQUESTS.WRITES_REMOTE * 64 / 1e6 / duration_time", + "MetricName": "llc_miss_remote_memory_bandwidth_write", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "", + "MetricExpr": "MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY", + "MetricName": "loads_retired_per_instr", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "DDR memory read bandwidth (MB/sec)", + "MetricExpr": "(UNC_M_CAS_COUNT_SCH0.RD + UNC_M_CAS_COUNT_SCH1.RD) * 64 / 1e6 / duration_time", + "MetricName": "memory_bandwidth_read", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "DDR memory bandwidth (MB/sec)", + "MetricExpr": "(UNC_M_CAS_COUNT_SCH0.RD + UNC_M_CAS_COUNT_SCH1.RD + UNC_M_CAS_COUNT_SCH0.WR + UNC_M_CAS_COUNT_SCH1.WR) * 64 / 1e6 / duration_time", + "MetricName": "memory_bandwidth_total", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "DDR memory write bandwidth (MB/sec)", + "MetricExpr": "(UNC_M_CAS_COUNT_SCH0.WR + UNC_M_CAS_COUNT_SCH1.WR) * 64 / 1e6 / duration_time", + "MetricName": "memory_bandwidth_write", + "ScaleUnit": "1MB/s" + }, + { + "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_LOCAL) / (UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_REMOTE)", + "MetricName": "numa_reads_addressed_to_local_dram", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_REMOTE) / (UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_REMOTE)", + "MetricName": "numa_reads_addressed_to_remote_dram", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Percentage of cycles spent in System Management Interrupts.", + "MetricExpr": "((msr@aperf@ - cycles) / msr@aperf@ if msr@smi@ > 0 else 0)", + "MetricGroup": "smi", + "MetricName": "smi_cycles", + "MetricThreshold": "smi_cycles > 0.1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Number of SMI interrupts.", + "MetricExpr": "msr@smi@", + "MetricGroup": "smi", + "MetricName": "smi_num", + "ScaleUnit": "1SMI#" + }, + { + "BriefDescription": "", + "MetricExpr": "MEM_UOPS_RETIRED.ALL_STORES / INST_RETIRED.ANY", + "MetricName": "stores_retired_per_instr", + "ScaleUnit": "1per_instr" + }, + { + "BriefDescription": "Uncore operating frequency in GHz", + "MetricExpr": "UNC_CHA_CLOCKTICKS / (source_count(UNC_CHA_CLOCKTICKS) * #num_packages) / 1e9 / duration_time", + "MetricName": "uncore_frequency", + "ScaleUnit": "1GHz" + }, + { + "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)", + "MetricExpr": "UNC_UPI_TxL_FLITS.ALL_DATA * 7.111111111111111 / 1e6 / duration_time", + "MetricName": "upi_data_transmit_bw", + "ScaleUnit": "1MB/s" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/floating-point.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/floating-point.json new file mode 100644 index 000000000000..34d7a50d2fd8 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/floating-point.json @@ -0,0 +1,191 @@ +[ + { + "BriefDescription": "Counts the number of cycles when any of the floating point dividers are active.", + "Counter": "0,1,2,3,4,5,6,7", + "CounterMask": "1", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of floating point dividers per cycle in the loop stage.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of floating point divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of all types of floating point operations per uop with all default weighting", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc8", + "EventName": "FP_FLOPS_RETIRED.ALL", + "PublicDescription": "Counts the number of all types of floating point operations per uop with all default weighting Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts the number of floating point operations that produce 32 bit single precision results", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc8", + "EventName": "FP_FLOPS_RETIRED.FP32", + "PublicDescription": "Counts the number of floating point operations that produce 32 bit single precision results Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of floating point operations that produce 64 bit double precision results", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc8", + "EventName": "FP_FLOPS_RETIRED.FP64", + "PublicDescription": "Counts the number of floating point operations that produce 64 bit double precision results Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a packed 128 bit double precision floating point. This may be SSE or AVX.128 operations.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.128B_DP", + "PublicDescription": "Counts the number of retired instructions whose sources are a packed 128 bit double precision floating point. This may be SSE or AVX.128 operations. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a packed 128 bit single precision floating point. This may be SSE or AVX.128 operations.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.128B_SP", + "PublicDescription": "Counts the number of retired instructions whose sources are a packed 128 bit single precision floating point. This may be SSE or AVX.128 operations. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a packed 256 bit double precision floating point.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.256B_DP", + "PublicDescription": "Counts the number of retired instructions whose sources are a packed 256 bit double precision floating point. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a packed 256 bit single precision floating point.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.256B_SP", + "PublicDescription": "Counts the number of retired instructions whose sources are a packed 256 bit single precision floating point. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a scalar 32bit single precision floating point.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.32B_SP", + "PublicDescription": "Counts the number of retired instructions whose sources are a scalar 32bit single precision floating point. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of retired instructions whose sources are a scalar 64 bit double precision floating point.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.64B_DP", + "PublicDescription": "Counts the number of retired instructions whose sources are a scalar 64 bit double precision floating point. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the total number of floating point retired instructions.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc7", + "EventName": "FP_INST_RETIRED.ALL", + "PublicDescription": "Counts the total number of floating point retired instructions. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x3f" + }, + { + "BriefDescription": "Counts the number of uops executed on all floating point ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0x1f" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P3", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2, 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0x1e" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.STD", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of floating point operations retired that required microcode assist.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FP_ASSIST", + "PublicDescription": "Counts the number of floating point operations retired that required microcode assist, which is not a reflection of the number of FP operations, instructions or uops.", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of floating point divide uops retired (x87 and sse, including x87 sqrt).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.FPDIV", + "PublicDescription": "Counts the number of floating point divide uops retired (x87 and sse, including x87 sqrt). Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/frontend.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/frontend.json index 7a9250e5c8f2..b71d6ae1abe8 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/frontend.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/frontend.json @@ -1,4 +1,206 @@ [ + { + "BriefDescription": "Counts the total number of BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe6", + "EventName": "BACLEARS.ANY", + "PublicDescription": "Counts the total number of BACLEARS, which occur when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend. Includes BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of BACLEARS due to a conditional jump.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe6", + "EventName": "BACLEARS.COND", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of BACLEARS due to an indirect branch.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe6", + "EventName": "BACLEARS.INDIRECT", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of BACLEARS due to a return branch.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe6", + "EventName": "BACLEARS.RETURN", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of BACLEARS due to a direct, unconditional jump.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe6", + "EventName": "BACLEARS.UNCOND", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged with having preceded with frontend bound behavior", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ALL", + "PublicDescription": "Counts the number of instructions retired that were tagged with having preceded with frontend bound behavior Available PDIST counters: 0,1", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a baclear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.BRANCH_DETECT", + "PublicDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a baclear Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles /empty issue slots due to a btclear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.BRANCH_RESTEER", + "PublicDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles /empty issue slots due to a btclear Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged following an ms flow due to the bubble/wasted issue slot from exiting long ms flow", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.CISC", + "PublicDescription": "Counts the number of instructions retired that were tagged following an ms flow due to the bubble/wasted issue slot from exiting long ms flow Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged every cycle the decoder is unable to send 3 uops per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.DECODE", + "PublicDescription": "Counts the number of instructions retired that were tagged every cycle the decoder is unable to send 3 uops per cycle. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ICACHE", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ITLB_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of instruction retired tagged after a wasted issue slot if none of the previous events occurred", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.OTHER", + "PublicDescription": "Counts the number of instruction retired tagged after a wasted issue slot if none of the previous events occurred Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a predecode wrong.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.PREDECODE", + "PublicDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a predecode wrong. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L2_HIT", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L2 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L2_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xe" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x6" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and did not have to snoop another core.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT_NO_SNOOP", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and did not have to snoop another core. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and required a snoop (that resulted in a snoop miss, snoop hitm, snoop hit with fwd, or snoop hit with no fwd).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT_WITH_SNOOP", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and required a snoop (that resulted in a snoop miss, snoop hitm, snoop hit with fwd, or snoop hit with no fwd). Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 and L3 caches.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 and L3 caches. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss that hit in the second level TLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ITLB_STLB_HIT", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss that hit in the second level TLB. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss that also missed the second level TLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ITLB_STLB_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss that also missed the second level TLB. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of instructions retired that were preceded by empty issue slots at allocation due to an Instruction L1 cache miss, that matched a current outstanding prefetch request initiated by a Code SWPF stream.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.LATE_SWPF", + "PublicDescription": "Counts the number of instructions retired that were preceded by empty issue slots at allocation due to an Instruction L1 cache miss, that matched a current outstanding prefetch request initiated by a Code SWPF stream. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, { "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump.", "Counter": "0,1,2,3,4,5,6,7", @@ -7,6 +209,14 @@ "SampleAfterValue": "1000003", "UMask": "0x3" }, + { + "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are present.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x80", + "EventName": "ICACHE.HIT", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are not present. -", "Counter": "0,1,2,3,4,5,6,7", @@ -14,5 +224,21 @@ "EventName": "ICACHE.MISSES", "SampleAfterValue": "1000003", "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of cycles that the micro-sequencer is busy.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe7", + "EventName": "MS_DECODED.MS_BUSY", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of times entered into a ucode flow in the FEC. Includes inserted flows due to front-end detected faults or assists.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe7", + "EventName": "MS_DECODED.MS_ENTRY", + "SampleAfterValue": "1000003", + "UMask": "0x1" } ] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json index 58e543550279..ae9095a090e5 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json @@ -1,4 +1,96 @@ [ + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a core bound stall including a store address match, a DTLB miss or a page walk that detains the load from retiring.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_BOUND_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0xf4" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x81" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER_AT_RET", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0xc0" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82" + }, + { + "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MEMORY_ORDERING_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8002" + }, + { + "BriefDescription": "Counts misaligned loads that are 4K page splits.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x13", + "EventName": "MISALIGN_MEM_REF.LOAD_PAGE_SPLIT", + "PublicDescription": "Counts misaligned loads that are 4K page splits. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts misaligned stores that are 4K page splits.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x13", + "EventName": "MISALIGN_MEM_REF.STORE_PAGE_SPLIT", + "PublicDescription": "Counts misaligned stores that are 4K page splits. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/other.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/other.json new file mode 100644 index 000000000000..2c8721903610 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/other.json @@ -0,0 +1,20 @@ +[ + { + "BriefDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe4", + "EventName": "LBR_INSERTS.ANY", + "PublicDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the total number of BTCLEARS.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe8", + "EventName": "PREDICTION.BTCLEAR", + "PublicDescription": "Counts the total number of BTCLEARS which occurs when the Branch Target Buffer (BTB) predicts a taken branch.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json index 26bd12fefa3d..3d10dcffb8fc 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json @@ -1,4 +1,38 @@ [ + { + "BriefDescription": "Counts the number of cycles when any of the floating point or integer dividers are active.", + "Counter": "0,1,2,3,4,5,6,7", + "CounterMask": "1", + "EventCode": "0xcd", + "EventName": "ARITH.DIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts the number of cycles when any of the integer dividers are active.", + "Counter": "0,1,2,3,4,5,6,7", + "CounterMask": "1", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts number of active integer dividers per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of integer divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the total number of branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -7,6 +41,211 @@ "PublicDescription": "Counts the total number of instructions in which the instruction pointer (IP) of the processor is resteered due to a branch instruction and the branch instruction successfully retires. All branch type instructions are accounted for. Available PDIST counters: 0,1", "SampleAfterValue": "1000003" }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.ALL_NEAR_IND", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_OR_RETURN]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.ALL_NEAR_IND_OR_RET", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_OR_RETURN] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x58" + }, + { + "BriefDescription": "Counts the number of conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND", + "PublicDescription": "Counts the number of conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x7" + }, + { + "BriefDescription": "Counts the number of not taken conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_NTAKEN", + "PublicDescription": "Counts the number of not taken conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of taken conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_TAKEN", + "PublicDescription": "Counts the number of taken conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts the number of taken backward conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_TAKEN_BWD", + "PublicDescription": "Counts the number of taken backward conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of taken forward conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD", + "PublicDescription": "Counts the number of taken forward conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of near CALL branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_CALL", + "PublicDescription": "Counts the number of near CALL branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x30" + }, + { + "BriefDescription": "Counts the number of near direct CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIRECT_CALL", + "PublicDescription": "Counts the number of near direct CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of near direct JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIRECT_JMP", + "PublicDescription": "Counts the number of near direct JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIR_CALL", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIR_JMP", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of near indirect JMP and near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.ALL_NEAR_IND]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT", + "PublicDescription": "Counts the number of near indirect JMP and near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.ALL_NEAR_IND] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, + { + "BriefDescription": "Counts the number of near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_CALL", + "PublicDescription": "Counts the number of near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of near indirect JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_JMP", + "PublicDescription": "Counts the number of near indirect JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of near indirect JMP, near indirect CALL, and RET branch instructions retired. [This event is alias to BR_INST_RETIRED.ALL_NEAR_IND_OR_RET]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_OR_RETURN", + "PublicDescription": "Counts the number of near indirect JMP, near indirect CALL, and RET branch instructions retired. [This event is alias to BR_INST_RETIRED.ALL_NEAR_IND_OR_RET] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x58" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_IND_CALL", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_IND_JMP", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of near JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_JMP", + "PublicDescription": "Counts the number of near JMP branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xc0" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_RETURN]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_RET", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_RETURN] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of near RET branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_RET]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_RETURN", + "PublicDescription": "Counts the number of near RET branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_RET] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of near taken branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_TAKEN", + "PublicDescription": "Counts the number of near taken branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xfb" + }, { "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -16,10 +255,47 @@ "SampleAfterValue": "1000003" }, { - "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles.", + "BriefDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_INDIRECT]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.ALL_NEAR_IND", + "PublicDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_INDIRECT] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, + { + "BriefDescription": "Counts the number of mispredicted not taken conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND_NTAKEN", + "PublicDescription": "Counts the number of mispredicted not taken conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of mispredicted taken conditional branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND_TAKEN", + "PublicDescription": "Counts the number of mispredicted taken conditional branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts the number of mispredicted near indirect JMP and near indirect CALL branch instructions retired. [This event is alias to BR_MISP_RETIRED.ALL_NEAR_IND]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT", + "PublicDescription": "Counts the number of mispredicted near indirect JMP and near indirect CALL branch instructions retired. [This event is alias to BR_MISP_RETIRED.ALL_NEAR_IND] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.THREAD]", "Counter": "Fixed counter 1", "EventName": "CPU_CLK_UNHALTED.CORE", - "SampleAfterValue": "1000003", + "SampleAfterValue": "2000003", "UMask": "0x2" }, { @@ -27,13 +303,13 @@ "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0x3c", "EventName": "CPU_CLK_UNHALTED.CORE_P", - "SampleAfterValue": "1000003" + "SampleAfterValue": "2000003" }, { "BriefDescription": "Fixed Counter: Counts the number of unhalted reference clock cycles.", "Counter": "Fixed counter 2", "EventName": "CPU_CLK_UNHALTED.REF_TSC", - "SampleAfterValue": "1000003", + "SampleAfterValue": "2000003", "UMask": "0x3" }, { @@ -42,14 +318,14 @@ "EventCode": "0x3c", "EventName": "CPU_CLK_UNHALTED.REF_TSC_P", "PublicDescription": "Counts the number of reference cycles that the core is not in a halt state. The core enters the halt state when it is running the HLT instruction. This event is not affected by core frequency changes and increments at a fixed frequency that is also used for the Time Stamp Counter (TSC). This event uses a programmable general purpose performance counter.", - "SampleAfterValue": "1000003", + "SampleAfterValue": "2000003", "UMask": "0x1" }, { - "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles.", + "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.CORE]", "Counter": "Fixed counter 1", "EventName": "CPU_CLK_UNHALTED.THREAD", - "SampleAfterValue": "1000003", + "SampleAfterValue": "2000003", "UMask": "0x2" }, { @@ -57,14 +333,14 @@ "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0x3c", "EventName": "CPU_CLK_UNHALTED.THREAD_P", - "SampleAfterValue": "1000003" + "SampleAfterValue": "2000003" }, { "BriefDescription": "Fixed Counter: Counts the number of instructions retired.", "Counter": "Fixed counter 0", "EventName": "INST_RETIRED.ANY", "PublicDescription": "Fixed Counter: Counts the number of instructions retired. Available PDIST counters: 32", - "SampleAfterValue": "1000003", + "SampleAfterValue": "2000003", "UMask": "0x1" }, { @@ -73,15 +349,339 @@ "EventCode": "0xc0", "EventName": "INST_RETIRED.ANY_P", "PublicDescription": "Counts the number of instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "2000003" + }, + { + "BriefDescription": "Counts the number of uops executed on secondary integer ports 0,1,2,3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.2ND", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of uops executed on all Integer ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xff" + }, + { + "BriefDescription": "Counts the number of uops executed on a load port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.LD", + "PublicDescription": "Counts the number of uops executed on a load port. This event counts for integer uops even if the destination is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P3", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0,1, 2, 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0x78" + }, + { + "BriefDescription": "Counts the number of uops executed on a Store address port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STA", + "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STD_JMP", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.ADDRESS_ALIAS", + "PublicDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of retired loads that are blocked for any of the following reasons: DTLB miss, address alias, store forward or data unknown (includes memory disambiguation blocks and ESP consuming load blocks) and others.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.ALL", + "PublicDescription": "Counts the number of retired loads that are blocked for any of the following reasons: DTLB miss, address alias, store forward or data unknown (includes memory disambiguation blocks and ESP consuming load blocks) and others. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1f" + }, + { + "BriefDescription": "Counts the number of retired loads that are blocked because its address exactly matches an older store whose data is not ready.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.DATA_UNKNOWN", + "PublicDescription": "Counts the number of retired loads that are blocked because its address exactly matches an older store whose data is not ready. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of retired loads that are blocked because its address partially overlapped with an older store.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.STORE_FORWARD", + "PublicDescription": "Counts the number of retired loads that are blocked because its address partially overlapped with an older store. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the total number of machine clears for any reason including, but not limited to, memory ordering, memory disambiguation, SMC, and FP assist.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY", "SampleAfterValue": "1000003" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x80ff" + }, + { + "BriefDescription": "Counts the number of machine clears due to memory ordering in which an internal load passes an older store within the same CPU.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.DISAMBIGUATION", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.DISAMBIGUATION_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8008" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of machine clears due to memory renaming.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8010" + }, + { + "BriefDescription": "Counts the number of machine clears due to a page fault. Counts both I-Side and D-Side (Loads/Stores) page faults. A page fault occurs when either the page is not present, or an access violation occurs.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.PAGE_FAULT", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of machine clears due to program modifying data (self modifying code) within 1K of a recently fetched code page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.SMC", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of machine clears due to program modifying data (cross-core modifying code) within 1K of a recently fetched code page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.XMC", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xe4", + "EventName": "MISC_RETIRED.LBR_INSERTS", + "PublicDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe0", + "EventName": "MISC_RETIRED1.CL_INST", + "PublicDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xff" + }, + { + "BriefDescription": "Counts the number of LFENCE instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe0", + "EventName": "MISC_RETIRED1.LFENCE", + "PublicDescription": "Counts the number of LFENCE instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of accesses to KeyLocker cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe1", + "EventName": "MISC_RETIRED2.KEYLOCKER_ACCESS", + "PublicDescription": "Counts the number of accesses to KeyLocker cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of misses to KeyLocker cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe1", + "EventName": "MISC_RETIRED2.KEYLOCKER_MISS", + "PublicDescription": "Counts the number of misses to KeyLocker cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x11" + }, + { + "BriefDescription": "Counts the number of issue slots in a UMWAIT or TPAUSE instruction where no uop issues due to the instruction putting the CPU into the C0.1 activity state. For Tremont, UMWAIT and TPAUSE will only put the CPU into C0.1 activity state (not C0.2 activity state).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.C01_MS_SCB", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of issue slots where no uop could issue due to an IQ scoreboard that stalls allocation until a specified older uop retires or (in the case of jump scoreboard) executes. Commonly executed instructions with IQ scoreboards include LFENCE and MFENCE.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.IQ_JEU_SCB", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.NON_C01_MS_SCB", + "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, { "BriefDescription": "Fixed Counter: Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear.", - "Counter": "36", + "Counter": "Fixed counter 4", "EventName": "TOPDOWN_BAD_SPECULATION.ALL", "SampleAfterValue": "1000003", "UMask": "0x5" }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.ALL_P", + "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. Only issue slots wasted due to fast nukes such as memory ordering nukes are counted. Other nukes are not accounted for. Counts all issue slots blocked during this recovery window, including relevant microcode flows, and while uops are not yet available in the instruction queue (IQ) or until an FE_BOUND event occurs besides OTHER and CISC. Also includes the issue slots that were consumed by the backend but were thrown away because they were younger than the mispredict or machine clear.", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to Fast Nukes such as Memory Ordering Machine clears and MRN nukes", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.FASTNUKE", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to a branch mispredict that resulted in LSD exit.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.LSD_MISPREDICT", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a machine clear (nuke) of any kind including memory ordering and memory disambiguation.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.MACHINE_CLEARS", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to Branch Mispredict", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.MISPREDICT", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to a machine clear (nuke).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of retirement slots not consumed due to backend stalls. [This event is alias to TOPDOWN_BE_BOUND.ALL_P]", "Counter": "0,1,2,3,4,5,6,7", @@ -90,6 +690,14 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to due to certain allocation restrictions.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.ALLOC_RESTRICTIONS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of retirement slots not consumed due to backend stalls. [This event is alias to TOPDOWN_BE_BOUND.ALL]", "Counter": "0,1,2,3,4,5,6,7", @@ -98,18 +706,215 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to LSD entry.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.LSD", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to memory reservation stall (scheduler not being able to accept another uop). This could be caused by RSV full or load/store buffer block.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.MEM_SCHEDULER", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to IEC and FPC RAT stalls - which can be due to the FIQ and IEC reservation station stall (integer, FP and SIMD scheduler not being able to accept another uop).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.NON_MEM_SCHEDULER", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to mrbl stall. A marble refers to a physical register file entry, also known as the physical destination (PDST).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.REGISTER", + "PublicDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to mrbl stall. A 'marble' refers to a physical register file entry, also known as the physical destination (PDST).", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to ROB full", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.REORDER_BUFFER", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to iq/jeu scoreboards or ms scb", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.SERIALIZATION", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, { "BriefDescription": "Fixed Counter: Counts the number of retirement slots not consumed due to front end stalls.", - "Counter": "37", + "Counter": "Fixed counter 5", "EventName": "TOPDOWN_FE_BOUND.ALL", "SampleAfterValue": "1000003", "UMask": "0x6" }, + { + "BriefDescription": "Counts the number of retirement slots not consumed due to front end stalls.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x9c", + "EventName": "TOPDOWN_FE_BOUND.ALL_P", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to BAClear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.BRANCH_DETECT", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to BTClear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.BRANCH_RESTEER", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to ms", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.CISC", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to decode stall", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.DECODE", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.FRONTEND_BANDWIDTH", + "SampleAfterValue": "1000003", + "UMask": "0x8d" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to latency related stalls including BACLEARs, BTCLEARs, ITLB misses, and ICache misses.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.FRONTEND_LATENCY", + "SampleAfterValue": "1000003", + "UMask": "0x72" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to itlb miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.ITLB_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend that do not categorize into any other common frontend stall", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.OTHER", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to predecode wrong", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.PREDECODE", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Fixed Counter: Counts the number of consumed retirement slots.", - "Counter": "38", + "Counter": "Fixed counter 6", "EventName": "TOPDOWN_RETIRING.ALL", "SampleAfterValue": "1000003", "UMask": "0x7" + }, + { + "BriefDescription": "Counts the number of consumed retirement slots.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "TOPDOWN_RETIRING.ALL_P", + "PublicDescription": "Counts the number of consumed retirement slots. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of uops issued by the front end every cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x0e", + "EventName": "UOPS_ISSUED.ANY", + "PublicDescription": "Counts the number of uops issued by the front end every cycle. When 4-uops are requested and only 2-uops are delivered, the event counts 2. Uops_issued correlates to the number of ROB entries. If uop takes 2 ROB slots it counts as 2 uops_issued.", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of uops retired that were dual destination.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.DUAL_DEST", + "PublicDescription": "Counts the number of uops retired that were dual destination. Dual destination (dual dest) micro-operations require writing to two separate destination registers or memory addresses upon execution. These uops consume two entries in the ROB and tracked separately because they involve more complex operations that necessitate multiple results. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of uops retired that are the last uop of a macro-instruction.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.EOM", + "PublicDescription": "Counts the number of uops retired that are the last uop of a macro-instruction. EOM uops indicate the 'end of a macro-instruction' and play a crucial role in the processor's control flow and recovery mechanisms.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of integer divide uops retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.IDIV", + "PublicDescription": "Counts the number of integer divide uops retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of uops retired that originated from a loop stream detector.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.LSD", + "PublicDescription": "Counts the number of uops retired that originated from a loop stream detector. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS). This includes uops from flows due to complex instructions, faults, assists, and inserted flows.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.MS", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of x87 uops retired, includes those in ms flows.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.X87", + "SampleAfterValue": "1000003" } ] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cache.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cache.json new file mode 100644 index 000000000000..278837f2068a --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cache.json @@ -0,0 +1,3413 @@ +[ + { + "BriefDescription": "Clockticks for CMS units attached to CHA", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_CHACMS_CLOCKTICKS", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "CHACMS" + }, + { + "BriefDescription": "UNC_CHACMS_DISTRESS_ASSERTED", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHACMS_DISTRESS_ASSERTED", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "CHACMS" + }, + { + "BriefDescription": "Counts the number of cycles FAST trigger is received from the global FAST distress wire.", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHACMS_RING_SRC_THRTL", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "CHACMS" + }, + { + "BriefDescription": "Number of CHA clock cycles while the event is enabled", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_CHA_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Clockticks of the uncore caching and home agent (CHA)", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts transactions that looked into the multi-socket cacheline Directory state, and therefore did not send a snoop because the Directory indicated it was not needed.", + "Counter": "0,1,2,3", + "EventCode": "0x53", + "EventName": "UNC_CHA_DIR_LOOKUP.NO_SNP", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts transactions that looked into the multi-socket cacheline Directory state, and sent one or more snoops, because the Directory indicated it was needed.", + "Counter": "0,1,2,3", + "EventCode": "0x53", + "EventName": "UNC_CHA_DIR_LOOKUP.SNP", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts only multi-socket cacheline Directory state updates memory writes issued from the HA pipe. This does not include memory write requests which are for I (Invalid) or E (Exclusive) cachelines.", + "Counter": "0,1,2,3", + "EventCode": "0x54", + "EventName": "UNC_CHA_DIR_UPDATE.HA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts only multi-socket cacheline Directory state updates due to memory writes issued from the TOR pipe which are the result of remote transaction hitting the SF/LLC and returning data Core2Core. This does not include memory write requests which are for I (Invalid) or E (Exclusive) cachelines.", + "Counter": "0,1,2,3", + "EventCode": "0x54", + "EventName": "UNC_CHA_DIR_UPDATE.TOR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Distress signal assertion for dynamic prefetch throttle (DPT). Threshold for distress signal assertion reached in TOR or IRQ (immediate cause for triggering).", + "Counter": "0,1,2,3", + "EventCode": "0x59", + "EventName": "UNC_CHA_DISTRESS_ASSERTED.DPT_ANY", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x3", + "Unit": "CHA" + }, + { + "BriefDescription": "Distress signal assertion for dynamic prefetch throttle (DPT). Threshold for distress signal assertion reached in IRQ (immediate cause for triggering).", + "Counter": "0,1,2,3", + "EventCode": "0x59", + "EventName": "UNC_CHA_DISTRESS_ASSERTED.DPT_IRQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Distress signal assertion for dynamic prefetch throttle (DPT). Threshold for distress signal assertion reached in TOR (immediate cause for triggering).", + "Counter": "0,1,2,3", + "EventCode": "0x59", + "EventName": "UNC_CHA_DISTRESS_ASSERTED.DPT_TOR", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts when a normal (Non-Isochronous) full line write is issued from the CHA to the any of the memory controller channels.", + "Counter": "0,1,2,3", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.FULL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : ISOCH Full Line : Counts the total number of full line writes issued from the HA into the memory controller.", + "Counter": "0,1,2,3", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.FULL_PRIORITY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : Partial Non-ISOCH : Counts the total number of full line writes issued from the HA into the memory controller.", + "Counter": "0,1,2,3", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.PARTIAL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : ISOCH Partial : Counts the total number of full line writes issued from the HA into the memory controller.", + "Counter": "0,1,2,3", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.PARTIAL_PRIORITY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: All Requests to Remotely Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.ALL_REMOTE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : All transactions from Remote Agents", + "UMask": "0x17e0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: CRd Requests", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.CODE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests", + "UMask": "0x1bd0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests and Read Prefetches", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1bc1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests, Read Prefetches, and Snoops", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Data Reads", + "UMask": "0x1fc1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Demand Data Reads, Core and LLC prefetches", + "UMask": "0x841ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests, Read Prefetches, and Snoops which miss the Cache", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_MISS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Data Read Misses", + "UMask": "0x1fc101", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: All Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCALLY_HOMED_ADDRESS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed locally", + "UMask": "0xbdfff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Code Read Requests and Code Read Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_CODE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests", + "UMask": "0x19d0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests and Read Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DATA_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x19c1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Code Read Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_CODE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests", + "UMask": "0x1850ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_DATA_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1841ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: RFO Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_RFO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests", + "UMask": "0x1848ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: LLC Prefetch Requests to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_LLC_PF", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x189dff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: All Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x199dff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Code Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_CODE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests", + "UMask": "0x1910ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Read Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_DATA_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1981ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: RFO Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_RFO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests", + "UMask": "0x1908ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: RFO Requests and RFO Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_RFO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests", + "UMask": "0x19c8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: All Requests to Remotely Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTELY_HOMED_ADDRESS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed remotely : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Transaction whose address resides in a remote MC", + "UMask": "0x15dfff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Code Read/Prefetch Requests from a Remote Socket", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_CODE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests", + "UMask": "0x1a10ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Data Read/Prefetch Requests from a Remote Socket", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_DATA_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1a01ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: RFO Requests/Prefetches from a Remote Socket", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_RFO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests", + "UMask": "0x1a08ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Snoop Requests from a Remote Socket", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_SNP", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed", + "UMask": "0x1c19ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: All RFO and RFO Prefetches", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.RFO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : All RFOs - Demand and Prefetches", + "UMask": "0x1bc8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: RFO Requests and RFO Prefetches to Locally Homed Memory", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.RFO_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Locally HOMed RFOs - Demand and Prefetches", + "UMask": "0x9c8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Writes to Locally Homed Memory (includes writebacks from L1/L2)", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.WRITE_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Writes", + "UMask": "0x842ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups: Writes to Remotely Homed Memory (includes writebacks from L1/L2)", + "Counter": "0,1,2,3", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.WRITE_REMOTE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Remote Writes", + "UMask": "0x17c2ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : All Lines Victimized", + "UMask": "0xf", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : IA traffic : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : IO traffic : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local - All Lines", + "UMask": "0x200f", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_E", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local - Lines in E State", + "UMask": "0x2002", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_F", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local - Lines in F State", + "UMask": "0x2008", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_M", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local - Lines in M State", + "UMask": "0x2001", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_S", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local - Lines in S State", + "UMask": "0x2004", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Remote - All Lines", + "UMask": "0x800f", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_E", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Remote - Lines in E State", + "UMask": "0x8002", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_M", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Remote - Lines in M State", + "UMask": "0x8001", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_S", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Remote - Lines in S State", + "UMask": "0x8004", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_E", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in E state", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_M", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in M state", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_S", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in S State", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts when a RFO (the Read for Ownership issued before a write) request hit a cacheline in the S (Shared) state.", + "Counter": "0,1,2,3", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.RFO_HIT_S", + "PerPkg": "1", + "PublicDescription": "Cbo Misc : RFO HitS", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Local InvItoE : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "Counter": "0,1,2,3", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.LOCAL_INVITOE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Local Rd : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "Counter": "0,1,2,3", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.LOCAL_READ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Off : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "Counter": "0,1,2,3", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.OFF_PWRHEURISTIC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Remote Rd : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "Counter": "0,1,2,3", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.REMOTE_READ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : RFO HitS Snoop Broadcast : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "Counter": "0,1,2,3", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.RFO_HITS_SNP_BCAST", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.ALLOC_EXCLUSIVE", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.ALLOC_EXCLUSIVE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.ALLOC_SHARED", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.ALLOC_SHARED", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.DEALLOC_EVCTCLN", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.DEALLOC_EVCTCLN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.DIRBACKED_ONLY", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.DIRBACKED_ONLY", + "Experimental": "1", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.HIT_EXCLUSIVE", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.HIT_EXCLUSIVE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.HIT_SHARED", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.HIT_SHARED", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.INCLUSIVE_ONLY", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.INCLUSIVE_ONLY", + "Experimental": "1", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.MISS", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.MISS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.UPDATE_EXCLUSIVE", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.UPDATE_EXCLUSIVE", + "Experimental": "1", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.UPDATE_SHARED", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.UPDATE_SHARED", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.VICTIM_EXCLUSIVE", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.VICTIM_EXCLUSIVE", + "Experimental": "1", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_REMOTE_SF.VICTIM_SHARED", + "Counter": "0,1,2,3", + "EventCode": "0x69", + "EventName": "UNC_CHA_REMOTE_SF.VICTIM_SHARED", + "Experimental": "1", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the total number of requests coming from a unit on this socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE", + "PerPkg": "1", + "PublicDescription": "HA Read and Write Requests : InvalItoE", + "UMask": "0x30", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the total number of requests coming from a unit on this socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE_LOCAL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the total number of requests coming from a remote socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE_REMOTE", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts read requests made into this CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write) .", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS", + "PerPkg": "1", + "PublicDescription": "HA Read and Write Requests : Reads", + "UMask": "0x3", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts read requests coming from a unit on this socket made into this CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write).", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS_LOCAL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts read requests coming from a remote socket made into the CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write).", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS_REMOTE", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts write requests made into the CHA, including streaming, evictions, HitM (Reads from another core to a Modified cacheline), etc.", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES", + "PerPkg": "1", + "PublicDescription": "HA Read and Write Requests : Writes", + "UMask": "0xc", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts write requests coming from a unit on this socket made into this CHA, including streaming, evictions, HitM (Reads from another core to a Modified cacheline), etc.", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES_LOCAL", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the total number of read requests made into the Home Agent. Reads include all read opcodes (including RFO). Writes include all writes (streaming, evictions, HitM, etc).", + "Counter": "0,1,2,3", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES_REMOTE", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : IRQ : Counts number of allocations per cycle into the specified Ingress queue.", + "Counter": "0,1,2,3", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.IRQ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Occupancy : IRQ : Counts number of entries in the specified Ingress queue in each cycle.", + "Counter": "0", + "EventCode": "0x11", + "EventName": "UNC_CHA_RxC_OCCUPANCY.IRQ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts snoop filter capacity evictions for entries tracking exclusive lines in the core's cache. Snoop filter capacity evictions occur when the snoop filter is full and evicts an existing entry to track a new entry. Does not count clean evictions such as when a core's cache replaces a tracked cacheline with a new cacheline.", + "Counter": "0,1,2,3", + "EventCode": "0x3d", + "EventName": "UNC_CHA_SF_EVICTION.E_STATE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Snoop Filter Capacity Evictions : E state", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts snoop filter capacity evictions for entries tracking modified lines in the core's cache. Snoop filter capacity evictions occur when the snoop filter is full and evicts an existing entry to track a new entry. Does not count clean evictions such as when a core's cache replaces a tracked cacheline with a new cacheline.", + "Counter": "0,1,2,3", + "EventCode": "0x3d", + "EventName": "UNC_CHA_SF_EVICTION.M_STATE", + "PerPkg": "1", + "PublicDescription": "Snoop Filter Capacity Evictions : M state", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts snoop filter capacity evictions for entries tracking shared lines in the core's cache. Snoop filter capacity evictions occur when the snoop filter is full and evicts an existing entry to track a new entry. Does not count clean evictions such as when a core's cache replaces a tracked cacheline with a new cacheline.", + "Counter": "0,1,2,3", + "EventCode": "0x3d", + "EventName": "UNC_CHA_SF_EVICTION.S_STATE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Snoop Filter Capacity Evictions : S state", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR Inserts", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All", + "UMask": "0xc001ffff", + "Unit": "CHA" + }, + { + "BriefDescription": "CLFlush transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_CLFLUSH", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8c7fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "FsRdCur transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_FSRDCUR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8effd20", + "Unit": "CHA" + }, + { + "BriefDescription": "FsRdCurPtl transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_FSRDCURPTL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c9effd20", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_ITOM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc47fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMWr transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_ITOMWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc4ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "MemPushWr transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_MEMPUSHWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc6ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "WCiL transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_WCIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c86ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "WcilF transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_WCILF", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c867fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "WiL transactions from a CXL device which hit in the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_HIT_WIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c87ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "CLFlush transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_CLFLUSH", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8c7fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "FsRdCur transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_FSRDCUR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8effe20", + "Unit": "CHA" + }, + { + "BriefDescription": "FsRdCurPtl transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_FSRDCURPTL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c9effe20", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_ITOM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc47fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMWr transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_ITOMWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc4ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "MemPushWr transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_MEMPUSHWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc6ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "WCiL transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_WCIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c86ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "WcilF transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_WCILF", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c867fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "WiL transactions from a CXL device which miss the L3.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.CXL_MISS_WIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c87ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "All locally initiated requests from IA Cores", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from iA Cores", + "UMask": "0xc001ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "CLFlush events that are initiated from the Core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CLFlushes issued by iA Cores", + "UMask": "0xc8c7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "CLFlushOpt events that are initiated from the Core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CLFLUSHOPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CLFlushOpts issued by iA Cores", + "UMask": "0xc8d7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRDs issued by iA Cores", + "UMask": "0xc80fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read prefetch from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opts issued by iA Cores", + "UMask": "0xc827ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt prefetch from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opt_Prefs issued by iA Cores", + "UMask": "0xc8a7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "All locally initiated requests from IA Cores which hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from iA Cores that Hit the LLC", + "UMask": "0xc001fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRds issued by iA Cores that Hit the LLC", + "UMask": "0xc80ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read prefetch from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that hit the LLC", + "UMask": "0xc88ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "All requests issued from IA cores to CXL accelerator memory regions that hit the LLC.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018101", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opts issued by iA Cores that hit the LLC", + "UMask": "0xc827fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt prefetch from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opt_Prefs issued by iA Cores that hit the LLC", + "UMask": "0xc8a7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM requests from local IA cores that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by iA Cores that Hit LLC", + "UMask": "0xcc47fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch code read from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefCode issued by iA Cores that hit the LLC", + "UMask": "0xcccffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch data read from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefData issued by iA Cores that hit the LLC", + "UMask": "0xccd7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch read for ownership from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefRFO issued by iA Cores that hit the LLC", + "UMask": "0xccc7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by iA Cores that Hit the LLC", + "UMask": "0xc807fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership prefetch from local IA that hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFO_Prefs issued by iA Cores that Hit the LLC", + "UMask": "0xc887fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM events that are initiated from the Core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by iA Cores", + "UMask": "0xcc47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNear requests from local IA cores", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears issued by iA Cores", + "UMask": "0xcd47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch code read from local IA.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefCode issued by iA Cores", + "UMask": "0xcccfff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch data read from local IA.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefData issued by iA Cores", + "UMask": "0xccd7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch read for ownership from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefRFO issued by iA Cores", + "UMask": "0xccc7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "All locally initiated requests from IA Cores which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from iA Cores that Missed the LLC", + "UMask": "0xc001fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRds issued by iA Cores that Missed the LLC", + "UMask": "0xc80ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "CRDs from local IA cores to locally homed memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc80efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Code read prefetch from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that Missed the LLC", + "UMask": "0xc88ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "CRD Prefetches from local IA cores to locally homed memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc88efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "CRD Prefetches from local IA cores to remotely homed memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc88f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "CRDs from local IA cores to remotely homed memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CRd issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc80f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "All requests issued from IA cores to CXL accelerator memory regions that miss the LLC.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018201", + "Unit": "CHA" + }, + { + "BriefDescription": "DRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 memory expander card.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_ACC", + "PerPkg": "1", + "PublicDescription": "DRds issued from an IA core which miss the L3 and target memory in a CXL type 2 memory expander card.", + "UMask": "0x10c8178201", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opt issued by iA Cores that missed the LLC", + "UMask": "0xc827fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd_Opt, and which target local memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opt issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc826fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Data read opt prefetch from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd_Opt_Prefs issued by iA Cores that missed the LLC", + "UMask": "0xc8a7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRD_PREF_OPT, and target local memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Data read opt prefetch from local iA that missed the LLC targeting local memory", + "UMask": "0xc8a6fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRD_PREF_OPT, and target remote memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Data read opt prefetch from local iA that missed the LLC targeting remote memory", + "UMask": "0xc8a77e01", + "Unit": "CHA" + }, + { + "BriefDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd_Opt, and target remote memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Data read opt from local iA that missed the LLC targeting remote memory", + "UMask": "0xc8277e01", + "Unit": "CHA" + }, + { + "BriefDescription": "L2 data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8978201", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM requests from local IA cores that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by iA Cores that Missed LLC", + "UMask": "0xcc47fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch code read from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefCode issued by iA Cores that missed the LLC", + "UMask": "0xcccffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch data read from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefData issued by iA Cores that missed the LLC", + "UMask": "0xccd7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "LLC data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccd78201", + "Unit": "CHA" + }, + { + "BriefDescription": "Last level cache prefetch read for ownership from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : LLCPrefRFO issued by iA Cores that missed the LLC", + "UMask": "0xccc7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8878201", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to locally homed DDR addresses that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "UMask": "0xc8668601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to locally homed PMM addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "UMask": "0xc8668a01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from local IA cores to locally homed DDR addresses that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "UMask": "0xc86e8601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from local IA cores to locally homed PMM addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "UMask": "0xc86e8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to remotely homed DDR addresses that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "UMask": "0xc8670601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to remotely homed PMM addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "UMask": "0xc8670a01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from local IA cores to remotely homed DDR addresses that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "UMask": "0xc86f0601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from local IA cores to remotely homed PMM addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "UMask": "0xc86f0a01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by iA Cores that Missed the LLC", + "UMask": "0xc807fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "RFOs issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8078201", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership from local IA that miss the LLC targeting local memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc806fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership prefetch from local IA that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFO_Prefs issued by iA Cores that Missed the LLC", + "UMask": "0xc887fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "LLC RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccc78201", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership prefetch from local IA that miss the LLC targeting local memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFO_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc886fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership prefetch from local IA that miss the LLC targeting remote memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFO_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc8877e01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership from local IA that miss the LLC targeting remote memory", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc8077e01", + "Unit": "CHA" + }, + { + "BriefDescription": "UCRDF requests from local IA cores that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_UCRDF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : UCRdFs issued by iA Cores that Missed LLC", + "UMask": "0xc877de01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from a local IA core that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores that Missed the LLC", + "UMask": "0xc86ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA core that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLF issued by iA Cores that Missed the LLC", + "UMask": "0xc867fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to DDR homed addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC", + "UMask": "0xc8678601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA cores to PMM homed addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC", + "UMask": "0xc8678a01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from local IA cores to DDR homed addresses which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC", + "UMask": "0xc86f8601", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from a local IA core to PMM homed addresses that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC", + "UMask": "0xc86f8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "WIL requests from local IA cores that miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WIL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WiLs issued by iA Cores that Missed LLC", + "UMask": "0xc87fde01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by iA Cores", + "UMask": "0xc807ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "Read for ownership prefetch from local IA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFO_Prefs issued by iA Cores", + "UMask": "0xc887ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "SpecItoM events that are initiated from the Core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_SPECITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : SpecItoMs issued by iA Cores", + "UMask": "0xcc57ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WbEFtoEs issued by iA Cores. (Non Modified Write Backs)", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBEFTOE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc3fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WbEFtoIs issued by iA Cores . (Non Modified Write Backs)", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBEFTOI", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc37ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WbMtoEs issued by iA Cores . (Modified Write Backs)", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBMTOE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc2fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WbMtoI requests from local IA cores", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WbMtoIs issued by iA Cores", + "UMask": "0xcc27ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WbStoIs issued by iA Cores . (Non Modified Write Backs)", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBSTOI", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc67ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCIL requests from a local IA core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLs issued by iA Cores", + "UMask": "0xc86fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "WCILF requests from local IA core", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WCiLF issued by iA Cores", + "UMask": "0xc867ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR inserts from local IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from IO Devices", + "UMask": "0xc001ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "CLFlush requests from IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : CLFlushes issued by IO Devices", + "UMask": "0xc8c3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR inserts from local IO devices which hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from IO Devices that hit the LLC", + "UMask": "0xc001fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMs from local IO devices which hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "UMask": "0xcd43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "PCIRDCURs issued by IO devices which hit the LLC", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that hit the LLC", + "UMask": "0xc8f3fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "RFOs from local IO devices which hit the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by IO Devices that hit the LLC", + "UMask": "0xc803fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR ItoM inserts from local IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices", + "UMask": "0xcc43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNears, indicating a partial write request, from IO Devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices", + "UMask": "0xcd43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNear (partial write) transactions from an IO device that addresses memory on the local socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that address memory on the local socket", + "UMask": "0xcd42ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNear (partial write) transactions from an IO device that addresses memory on a remote socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that address memory on a remote socket", + "UMask": "0xcd437f04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM (write) transactions from an IO device that addresses memory on the local socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOM_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoM, indicating a write request, from IO Devices that address memory on the local socket", + "UMask": "0xcc42ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoM (write) transactions from an IO device that addresses memory on a remote socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOM_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoM, indicating a write request, from IO Devices that address memory on a remote socket", + "UMask": "0xcc437f04", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR inserts from local IO devices which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from IO Devices that missed the LLC", + "UMask": "0xc001fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR ItoM inserts from local IO devices which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMs issued by IO Devices that missed the LLC", + "UMask": "0xcc43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "UMask": "0xcd43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "PCIRDCURs issued by IO devices which miss the LLC", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that missed the LLC", + "UMask": "0xc8f3fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "All TOR RFO inserts from local IO devices which miss the cache", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by IO Devices that missed the LLC", + "UMask": "0xc803fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "PCIRDCURs issued by IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices", + "UMask": "0xc8f3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "PCIRDCUR (read) transactions from an IO device that addresses memory on the local socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that addresses memory on the local socket", + "UMask": "0xc8f2ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "PCIRDCUR (read) transactions from an IO device that addresses memory on a remote socket", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that addresses memory on a remote socket", + "UMask": "0xc8f37f04", + "Unit": "CHA" + }, + { + "BriefDescription": "RFOs from local IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by IO Devices", + "UMask": "0xc803ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "WBMtoI requests from IO devices", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WbMtoIs issued by IO Devices", + "UMask": "0xcc23ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for SF or LLC Evictions", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LLC_OR_SF_EVICTIONS", + "PerPkg": "1", + "PublicDescription": "TOR allocation occurred as a result of SF/LLC evictions (came from the ISMQ)", + "UMask": "0xc001ff02", + "Unit": "CHA" + }, + { + "BriefDescription": "All locally initiated requests", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local iA and IO", + "UMask": "0xc000ff05", + "Unit": "CHA" + }, + { + "BriefDescription": "All from Local iA", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_IA", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local iA", + "UMask": "0xc000ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "All from Local IO", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_IO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local IO", + "UMask": "0xc000ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "All remote requests (e.g. snoops, writebacks) that came from remote sockets", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.REM_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All Remote Requests", + "UMask": "0xc001ffc8", + "Unit": "CHA" + }, + { + "BriefDescription": "All snoops to this LLC that came from remote sockets", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.REM_SNPS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All Snoops from Remote", + "UMask": "0xc001ff08", + "Unit": "CHA" + }, + { + "BriefDescription": "Occupancy for all TOR entries", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All", + "UMask": "0xc001ffff", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CLFlush transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_CLFLUSH", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8c7fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for FsRdCur transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_FSRDCUR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8effd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for FsRdCurPtl transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_FSRDCURPTL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c9effd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_ITOM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc47fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMWr transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_ITOMWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc4ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for MemPushWr transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_MEMPUSHWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc6ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCiL transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_WCIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c86ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WcilF transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_WCILF", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c867fd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WiL transactions from a CXL device which hit in the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_HIT_WIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c87ffd20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CLFlush transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_CLFLUSH", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8c7fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for FsRdCur transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_FSRDCUR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c8effe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for FsRdCurPtl transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_FSRDCURPTL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c9effe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_ITOM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc47fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMWr transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_ITOMWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc4ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for MemPushWr transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_MEMPUSHWR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78cc6ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCiL transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_WCIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c86ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WcilF transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_WCILF", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c867fe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WiL transactions from a CXL device which miss the L3.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.CXL_MISS_WIL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78c87ffe20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All locally initiated requests from IA Cores", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores", + "UMask": "0xc001ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CLFlush events that are initiated from the Core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushes issued by iA Cores", + "UMask": "0xc8c7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CLFlushOpt events that are initiated from the Core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CLFLUSHOPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushOpts issued by iA Cores", + "UMask": "0xc8d7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRDs issued by iA Cores", + "UMask": "0xc80fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opts issued by iA Cores", + "UMask": "0xc827ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD_OPT_PREF", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opt_Prefs issued by iA Cores", + "UMask": "0xc8a7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All locally initiated requests from IA Cores which hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores that Hit the LLC", + "UMask": "0xc001fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRds issued by iA Cores that Hit the LLC", + "UMask": "0xc80ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read prefetch from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that hit the LLC", + "UMask": "0xc88ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All requests issued from IA cores to CXL accelerator memory regions that hit the LLC.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018101", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opts issued by iA Cores that hit the LLC", + "UMask": "0xc827fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt prefetch from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opt_Prefs issued by iA Cores that hit the LLC", + "UMask": "0xc8a7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM requests from local IA cores that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores that Hit LLC", + "UMask": "0xcc47fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch code read from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefCode issued by iA Cores that hit the LLC", + "UMask": "0xcccffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch data read from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefData issued by iA Cores that hit the LLC", + "UMask": "0xccd7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch read for ownership from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefRFO issued by iA Cores that hit the LLC", + "UMask": "0xccc7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by iA Cores that Hit the LLC", + "UMask": "0xc807fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership prefetch from local IA that hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFO_Prefs issued by iA Cores that Hit the LLC", + "UMask": "0xc887fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM events that are initiated from the Core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores", + "UMask": "0xcc47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNear requests from local IA cores", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears issued by iA Cores", + "UMask": "0xcd47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch code read from local IA.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefCode issued by iA Cores", + "UMask": "0xcccfff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch data read from local IA.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefData issued by iA Cores", + "UMask": "0xccd7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefRFO issued by iA Cores", + "UMask": "0xccc7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All locally initiated requests from IA Cores which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores that Missed the LLC", + "UMask": "0xc001fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRds issued by iA Cores that Missed the LLC", + "UMask": "0xc80ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CRDs from local IA cores to locally homed memory", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc80efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Code read prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC", + "UMask": "0xc88ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CRD Prefetches from local IA cores to locally homed memory", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc88efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CRD Prefetches from local IA cores to remotely homed memory", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc88f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CRDs from local IA cores to remotely homed memory", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc80f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All requests issued from IA cores to CXL accelerator memory regions that miss the LLC.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 memory expander card.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8178201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opt issued by iA Cores that missed the LLC", + "UMask": "0xc827fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Data read opt prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Opt_Prefs issued by iA Cores that missed the LLC", + "UMask": "0xc8a7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for L2 data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8978201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM requests from local IA cores that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores that Missed LLC", + "UMask": "0xcc47fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch code read from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefCode issued by iA Cores that missed the LLC", + "UMask": "0xcccffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch data read from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefData issued by iA Cores that missed the LLC", + "UMask": "0xccd7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for LLC data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccd78201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Last level cache prefetch read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : LLCPrefRFO issued by iA Cores that missed the LLC", + "UMask": "0xccc7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8878201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to locally homed DDR addresses that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "UMask": "0xc8668601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to locally homed PMM addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "UMask": "0xc8668a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from local IA cores to locally homed DDR addresses that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "UMask": "0xc86e8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from local IA cores to locally homed PMM addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "UMask": "0xc86e8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to remotely homed DDR addresses that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "UMask": "0xc8670601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to remotely homed PMM addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "UMask": "0xc8670a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from local IA cores to remotely homed DDR addresses that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "UMask": "0xc86f0601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from local IA cores to remotely homed PMM addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "UMask": "0xc86f0a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by iA Cores that Missed the LLC", + "UMask": "0xc807fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for RFOs issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8078201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc806fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFO_Prefs issued by iA Cores that Missed the LLC", + "UMask": "0xc887fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for LLC RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccc78201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFO_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "UMask": "0xc886fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFO_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc8877e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by iA Cores that Missed the LLC - HOMed remotely", + "UMask": "0xc8077e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for UCRDF requests from local IA cores that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_UCRDF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : UCRdFs issued by iA Cores that Missed LLC", + "UMask": "0xc877de01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from a local IA core that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores that Missed the LLC", + "UMask": "0xc86ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA core that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLF issued by iA Cores that Missed the LLC", + "UMask": "0xc867fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to DDR homed addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC", + "UMask": "0xc8678601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA cores to PMM homed addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC", + "UMask": "0xc8678a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from local IA cores to DDR homed addresses which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC", + "UMask": "0xc86f8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from a local IA core to PMM homed addresses that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL_PMM", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC", + "UMask": "0xc86f8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WIL requests from local IA cores that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WiLs issued by iA Cores that Missed LLC", + "UMask": "0xc87fde01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by iA Cores", + "UMask": "0xc807ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for Read for ownership prefetch from local IA that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFO_Prefs issued by iA Cores", + "UMask": "0xc887ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for SpecItoM events that are initiated from the Core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_SPECITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : SpecItoMs issued by iA Cores", + "UMask": "0xcc57ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WbMtoI requests from local IA cores", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WbMtoIs issued by iA Cores", + "UMask": "0xcc27ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCIL requests from a local IA core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores", + "UMask": "0xc86fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WCILF requests from local IA core", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLF issued by iA Cores", + "UMask": "0xc867ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR inserts from local IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices", + "UMask": "0xc001ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CLFlush requests from IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushes issued by IO Devices", + "UMask": "0xc8c3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR inserts from local IO devices which hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices that hit the LLC", + "UMask": "0xc001fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMs from local IO devices which hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that Hit the LLC", + "UMask": "0xcc43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "UMask": "0xcd43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for PCIRDCURs issued by IO devices which hit the LLC", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that hit the LLC", + "UMask": "0xc8f3fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for RFOs from local IO devices which hit the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices that hit the LLC", + "UMask": "0xc803fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR ItoM inserts from local IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices", + "UMask": "0xcc43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNears, indicating a partial write request, from IO Devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices", + "UMask": "0xcd43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR inserts from local IO devices which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices that missed the LLC", + "UMask": "0xc001fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR ItoM inserts from local IO devices which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that missed the LLC", + "UMask": "0xcc43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "UMask": "0xcd43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNear transactions from an IO device on the local socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOMCACHENEAR_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "UMask": "0xcd42fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoMCacheNear transactions from an IO device on a remote socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOMCACHENEAR_REMOTE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "UMask": "0xcd437e04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM transactions from an IO device on the local socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOM_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that missed the LLC", + "UMask": "0xcc42fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for ItoM transactions from an IO device on a remote socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOM_REMOTE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that missed the LLC", + "UMask": "0xcc437e04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for PCIRDCURs issued by IO devices which miss the LLC", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that missed the LLC", + "UMask": "0xc8f3fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for PCIRDCUR transactions from an IO device on the local socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_PCIRDCUR_LOCAL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that missed the LLC", + "UMask": "0xc8f2fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for PCIRDCUR transactions from an IO device on a remote socket that miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_PCIRDCUR_REMOTE", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that missed the LLC", + "UMask": "0xc8f37e04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All TOR RFO inserts from local IO devices which miss the cache", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices that missed the LLC", + "UMask": "0xc803fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for PCIRDCURs issued by IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices", + "UMask": "0xc8f3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for RFOs from local IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices", + "UMask": "0xc803ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for WBMtoI requests from IO devices", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WbMtoIs issued by IO Devices", + "UMask": "0xcc23ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All locally initiated requests", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local iA and IO", + "UMask": "0xc000ff05", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All from Local iA", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_IA", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local iA", + "UMask": "0xc000ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All from Local IO", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_IO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local IO", + "UMask": "0xc000ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All remote requests (e.g. snoops, writebacks) that came from remote sockets", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.REM_ALL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All Remote Requests", + "UMask": "0xc001ffc8", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All snoops to this LLC that came from remote sockets", + "Counter": "0", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.REM_SNPS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All Snoops from Remote", + "UMask": "0xc001ff08", + "Unit": "CHA" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cxl.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cxl.json new file mode 100644 index 000000000000..383a5ba5a697 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-cxl.json @@ -0,0 +1,31 @@ +[ + { + "BriefDescription": "B2CXL Clockticks", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_B2CXL_CLOCKTICKS", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "B2CXL" + }, + { + "BriefDescription": "Number of Allocation to Mem Data Packing buffer", + "Counter": "4,5,6,7", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.MEM_DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to M2S Data AGF", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.M2S_DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLDP" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json new file mode 100644 index 000000000000..251e5d20fefe --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json @@ -0,0 +1,1625 @@ +[ + { + "BriefDescription": "Clockticks of the mesh to memory (B2CMI)", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_B2CMI_CLOCKTICKS", + "PerPkg": "1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of time D2C was not honoured by egress due to directory state constraints", + "Counter": "0,1,2,3", + "EventCode": "0x17", + "EventName": "UNC_B2CMI_DIRECT2CORE_NOT_TAKEN_DIRSTATE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of times B2CMI egress did D2C (direct to core)", + "Counter": "0,1,2,3", + "EventCode": "0x16", + "EventName": "UNC_B2CMI_DIRECT2CORE_TAKEN", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of times D2C wasn't honoured even though the incoming request had d2c set for non cisgress txn", + "Counter": "0,1,2,3", + "EventCode": "0x18", + "EventName": "UNC_B2CMI_DIRECT2CORE_TXN_OVERRIDE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of d2k wasn't done due to credit constraints", + "Counter": "0,1,2,3", + "EventCode": "0x1B", + "EventName": "UNC_B2CMI_DIRECT2UPI_NOT_TAKEN_CREDITS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Direct to UPI Transactions - Ignored due to lack of credits : All : Counts the number of d2k wasn't done due to credit constraints", + "Counter": "0,1,2,3", + "EventCode": "0x1B", + "EventName": "UNC_B2CMI_DIRECT2UPI_NOT_TAKEN_CREDITS.EGRESS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of time D2K was not honoured by egress due to directory state constraints", + "Counter": "0,1,2,3", + "EventCode": "0x1A", + "EventName": "UNC_B2CMI_DIRECT2UPI_NOT_TAKEN_DIRSTATE", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Egress Ignored D2U : Counts the number of time D2K was not honoured by egress due to directory state constraints", + "Counter": "0,1,2,3", + "EventCode": "0x1A", + "EventName": "UNC_B2CMI_DIRECT2UPI_NOT_TAKEN_DIRSTATE.EGRESS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of times egress did D2K (Direct to KTI)", + "Counter": "0,1,2,3", + "EventCode": "0x19", + "EventName": "UNC_B2CMI_DIRECT2UPI_TAKEN", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of times D2K wasn't honoured even though the incoming request had d2k set for non cisgress txn", + "Counter": "0,1,2,3", + "EventCode": "0x1C", + "EventName": "UNC_B2CMI_DIRECT2UPI_TXN_OVERRIDE", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit Clean", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x38", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in A State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.CLEAN_A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in I State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.CLEAN_I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in S State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.CLEAN_S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit Dirty (modified)", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in A State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.DIRTY_A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in I State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.DIRTY_I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in S State", + "Counter": "0,1,2,3", + "EventCode": "0x1D", + "EventName": "UNC_B2CMI_DIRECTORY_HIT.DIRTY_S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of 1lm or 2lm hit read data returns to egress with any directory to non persistent memory", + "Counter": "0,1,2,3", + "EventCode": "0x20", + "EventName": "UNC_B2CMI_DIRECTORY_LOOKUP.ANY", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of 1lm or 2lm hit read data returns to egress with directory A to non persistent memory", + "Counter": "0,1,2,3", + "EventCode": "0x20", + "EventName": "UNC_B2CMI_DIRECTORY_LOOKUP.STATE_A", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of 1lm or 2lm hit read data returns to egress with directory I to non persistent memory", + "Counter": "0,1,2,3", + "EventCode": "0x20", + "EventName": "UNC_B2CMI_DIRECTORY_LOOKUP.STATE_I", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the number of 1lm or 2lm hit read data returns to egress with directory S to non persistent memory", + "Counter": "0,1,2,3", + "EventCode": "0x20", + "EventName": "UNC_B2CMI_DIRECTORY_LOOKUP.STATE_S", + "PerPkg": "1", + "PublicDescription": "Counts the number of 1lm or 2lm hit read data returns to egress with directory S to non persistent memory", + "UMask": "0x4", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss Clean", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x38", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in A State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.CLEAN_A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in I State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.CLEAN_I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in S State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.CLEAN_S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss Dirty (modified)", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in A State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.DIRTY_A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in I State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.DIRTY_I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in S State", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_B2CMI_DIRECTORY_MISS.DIRTY_S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any A2I Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.A2I", + "PerPkg": "1", + "UMask": "0x320", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any A2S Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.A2S", + "PerPkg": "1", + "UMask": "0x340", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts cisgress directory updates", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.ANY", + "PerPkg": "1", + "UMask": "0x301", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts any 1lm or 2lm hit data return that would result in directory update to non persistent memory (DRAM)", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.HIT_ANY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x101", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in near memory to the A state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.HIT_X2A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x114", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in near memory to the I state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.HIT_X2I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x128", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in near memory to the S state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.HIT_X2S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x142", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any I2A Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.I2A", + "PerPkg": "1", + "UMask": "0x304", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any I2S Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.I2S", + "PerPkg": "1", + "UMask": "0x302", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in far memory to the A state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.MISS_X2A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x214", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in far memory to the I state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.MISS_X2I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x228", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update in far memory to the S state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.MISS_X2S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x242", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any S2A Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.S2A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x310", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Any S2I Transition", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.S2I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x308", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update to the A state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.X2A", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x314", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update to the I state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.X2I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x328", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Directory update to the S state", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_B2CMI_DIRECTORY_UPDATE.X2S", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x342", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts any read", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "UNC_B2CMI_IMC_READS.ALL", + "PerPkg": "1", + "UMask": "0x104", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts normal reads issue to CMI", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "UNC_B2CMI_IMC_READS.NORMAL", + "PerPkg": "1", + "UMask": "0x101", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Count reads to NM region", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "UNC_B2CMI_IMC_READS.TO_DDR_AS_CACHE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x110", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts reads to 1lm non persistent memory regions", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "UNC_B2CMI_IMC_READS.TO_DDR_AS_MEM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x108", + "Unit": "B2CMI" + }, + { + "BriefDescription": "All Writes - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.ALL", + "PerPkg": "1", + "UMask": "0x110", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Full Non-ISOCH - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.FULL", + "PerPkg": "1", + "UMask": "0x101", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Non-Inclusive - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.NI", + "Experimental": "1", + "PerPkg": "1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Non-Inclusive Miss - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.NI_MISS", + "Experimental": "1", + "PerPkg": "1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Partial Non-ISOCH - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.PARTIAL", + "PerPkg": "1", + "UMask": "0x102", + "Unit": "B2CMI" + }, + { + "BriefDescription": "DDR, acting as Cache - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.TO_DDR_AS_CACHE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x140", + "Unit": "B2CMI" + }, + { + "BriefDescription": "DDR - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x25", + "EventName": "UNC_B2CMI_IMC_WRITES.TO_DDR_AS_MEM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x120", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - Ch 0", + "Counter": "0,1,2,3", + "EventCode": "0x56", + "EventName": "UNC_B2CMI_PREFCAM_INSERTS.CH0_UPI", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - Ch 0", + "Counter": "0,1,2,3", + "EventCode": "0x56", + "EventName": "UNC_B2CMI_PREFCAM_INSERTS.CH0_XPT", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x56", + "EventName": "UNC_B2CMI_PREFCAM_INSERTS.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT -All Channels", + "Counter": "0,1,2,3", + "EventCode": "0x56", + "EventName": "UNC_B2CMI_PREFCAM_INSERTS.XPT_ALLCH", + "PerPkg": "1", + "PublicDescription": "Prefetch CAM Inserts : XPT - All Channels", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : Channel 0", + "Counter": "0,1,2,3", + "EventCode": "0x54", + "EventName": "UNC_B2CMI_PREFCAM_OCCUPANCY.CH0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm reads and WRNI which were a hit", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_B2CMI_TAG_HIT.ALL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm reads which were a hit clean", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_B2CMI_TAG_HIT.RD_CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm reads which were a hit dirty", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_B2CMI_TAG_HIT.RD_DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm WRNI which were a hit clean", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_B2CMI_TAG_HIT.WR_CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm WRNI which were a hit dirty", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_B2CMI_TAG_HIT.WR_DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm second way read miss for a WrNI", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm second way read miss for a WrNI", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm second way read miss for a Rd", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.RD_2WAY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm reads which were a miss and the cache line is unmodified", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.RD_CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm reads which were a miss and the cache line is modified", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.RD_DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm second way read miss for a WrNI", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.WR_2WAY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm WRNI which were a miss and the cache line is unmodified", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.WR_CLEAN", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Counts the 2lm WRNI which were a miss and the cache line is modified", + "Counter": "0,1,2,3", + "EventCode": "0x4B", + "EventName": "UNC_B2CMI_TAG_MISS.WR_DIRTY", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Tracker Inserts : Channel 0", + "Counter": "0,1,2,3", + "EventCode": "0x32", + "EventName": "UNC_B2CMI_TRACKER_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x104", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Tracker Occupancy : Channel 0", + "Counter": "0,1,2,3", + "EventCode": "0x33", + "EventName": "UNC_B2CMI_TRACKER_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "Write Tracker Inserts : Channel 0", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_B2CMI_WR_TRACKER_INSERTS.CH0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2CMI" + }, + { + "BriefDescription": "UNC_B2HOT_CLOCKTICKS", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_B2HOT_CLOCKTICKS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "B2HOT" + }, + { + "BriefDescription": "Number of uclks in domain", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_B2UPI_CLOCKTICKS", + "PerPkg": "1", + "Unit": "B2UPI" + }, + { + "BriefDescription": "Total Write Cache Occupancy : Mem", + "Counter": "0,1,2,3", + "EventCode": "0x0F", + "EventName": "UNC_I_CACHE_TOTAL_OCCUPANCY.MEM", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": "IRP Clockticks", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_I_CLOCKTICKS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "Inbound read requests received by the IRP and inserted into the FAF queue", + "Counter": "0,1,2,3", + "EventCode": "0x18", + "EventName": "UNC_I_FAF_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "FAF occupancy", + "Counter": "0,1,2,3", + "EventCode": "0x19", + "EventName": "UNC_I_FAF_OCCUPANCY", + "Experimental": "1", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_I_MISC0.FAST_REJ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Requests", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_I_MISC0.FAST_REQ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Lost Forward : Snoop pulled away ownership before a write was committed", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_I_MISC1.LOST_FWD", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Received Invalid : Secondary received a transfer that did not have sufficient MESI state", + "Counter": "0,1,2,3", + "EventCode": "0x1F", + "EventName": "UNC_I_MISC1.SEC_RCVD_INVLD", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Hit E/S responses", + "Counter": "0,1,2,3", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_ES", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x74", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Hit I responses", + "Counter": "0,1,2,3", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_I", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x72", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Hit M responses", + "Counter": "0,1,2,3", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_M", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x78", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop miss responses", + "Counter": "0,1,2,3", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_MISS", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x71", + "Unit": "IRP" + }, + { + "BriefDescription": "Inbound write (fast path) requests to coherent memory, received by the IRP resulting in write ownership requests issued by IRP to the mesh.", + "Counter": "0,1,2,3", + "EventCode": "0x11", + "EventName": "UNC_I_TRANSACTIONS.WR_PREF", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "IRP" + }, + { + "BriefDescription": "MDF Clockticks", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_MDF_CLOCKTICKS", + "PerPkg": "1", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of UPI LL clock cycles while the event is enabled", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_UPI_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of kfclks", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L1 : Number of UPI qfclk cycles spent in L1 power mode. L1 is a mode that totally shuts down a UPI link. Use edge detect to count the number of instances when the UPI link entered L1. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another. Because L1 totally shuts down the link, it takes a good amount of time to exit this mode.", + "Counter": "0,1,2,3", + "EventCode": "0x21", + "EventName": "UNC_UPI_L1_POWER_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCB", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xe", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCB_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10e", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCS", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCS_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10f", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Request", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.REQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Request, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.REQ_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x108", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - Conflict", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSPCNFLT", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1aa", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - Invalid", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSPI", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x12a", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - Data", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSP_DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xc", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - Data, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSP_DATA_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10c", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - No Data", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSP_NODATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Response - No Data, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.RSP_NODATA_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10a", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Snoop", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.SNP", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x9", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Snoop, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.SNP_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x109", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Writeback", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.WB", + "PerPkg": "1", + "UMask": "0xd", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Writeback, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.WB_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10d", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : All Data : Shows legal flit time (hides impact of L0p and L0c).", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.ALL_DATA", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Null FLITs received from any slot", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.ALL_NULL", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Null FLITs received from any slot", + "UMask": "0x27", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Data : Shows legal flit time (hides impact of L0p and L0c). : Count Data Flits (which consume all slots), but how much to count is based on Slot0-2 mask, so count can be 0-3 depending on which slots are enabled for counting..", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Idle : Shows legal flit time (hides impact of L0p and L0c).", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.IDLE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x47", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : LLCRD Not Empty : Shows legal flit time (hides impact of L0p and L0c). : Enables counting of LLCRD (with non-zero payload). This only applies to slot 2 since LLCRD is only allowed in slot 2", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.LLCRD", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : LLCTRL : Shows legal flit time (hides impact of L0p and L0c). : Equivalent to an idle packet. Enables counting of slot 0 LLCTRL messages.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.LLCTRL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : All Non Data : Shows legal flit time (hides impact of L0p and L0c).", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.NON_DATA", + "PerPkg": "1", + "UMask": "0x97", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot NULL or LLCRD Empty : Shows legal flit time (hides impact of L0p and L0c). : LLCRD with all zeros is treated as NULL. Slot 1 is not treated as NULL if slot 0 is a dual slot. This can apply to slot 0,1, or 2.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.NULL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Protocol Header : Shows legal flit time (hides impact of L0p and L0c). : Enables count of protocol headers in slot 0,1,2 (depending on slot uMask bits)", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.PROTHDR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 0 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 0 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 1 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 1 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 2 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 2 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT2", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 0 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "Counter": "0,1,2,3", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 1 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "Counter": "0,1,2,3", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 2 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "Counter": "0,1,2,3", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT2", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 0", + "Counter": "0,1,2,3", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 1", + "Counter": "0,1,2,3", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 2", + "Counter": "0,1,2,3", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT2", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L0p", + "Counter": "0,1,2,3", + "EventCode": "0x27", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_POWER_CYCLES_LL_ENTER", + "Counter": "0,1,2,3", + "EventCode": "0x28", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES_LL_ENTER", + "Experimental": "1", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_POWER_CYCLES_M3_EXIT", + "Counter": "0,1,2,3", + "EventCode": "0x29", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES_M3_EXIT", + "Experimental": "1", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCB", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xe", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCB_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10e", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCS", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCS_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10f", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Request", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.REQ", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Request, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.REQ_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x108", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - Conflict", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSPCNFLT", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1aa", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - Invalid", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSPI", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x12a", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - Data", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSP_DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xc", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - Data, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSP_DATA_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10c", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - No Data", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSP_NODATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Response - No Data, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.RSP_NODATA_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10a", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Snoop", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.SNP", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x9", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Snoop, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.SNP_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x109", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Writeback", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.WB", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xd", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Writeback, Match Opcode", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.WB_OPC", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10d", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All Data : Counts number of data flits across this UPI link.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "All Null Flits", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_NULL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Idle", + "UMask": "0x27", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Data : Shows legal flit time (hides impact of L0p and L0c). : Count Data Flits (which consume all slots), but how much to count is based on Slot0-2 mask, so count can be 0-3 depending on which slots are enabled for counting..", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.DATA", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Idle : Shows legal flit time (hides impact of L0p and L0c).", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.IDLE", + "PerPkg": "1", + "UMask": "0x47", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : LLCRD Not Empty : Shows legal flit time (hides impact of L0p and L0c). : Enables counting of LLCRD (with non-zero payload). This only applies to slot 2 since LLCRD is only allowed in slot 2", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.LLCRD", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : LLCTRL : Shows legal flit time (hides impact of L0p and L0c). : Equivalent to an idle packet. Enables counting of slot 0 LLCTRL messages.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.LLCTRL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All Non Data : Shows legal flit time (hides impact of L0p and L0c).", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.NON_DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Null FLITs transmitted to any slot", + "UMask": "0x97", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot NULL or LLCRD Empty : Shows legal flit time (hides impact of L0p and L0c). : LLCRD with all zeros is treated as NULL. Slot 1 is not treated as NULL if slot 0 is a dual slot. This can apply to slot 0,1, or 2.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.NULL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Protocol Header : Shows legal flit time (hides impact of L0p and L0c). : Enables count of protocol headers in slot 0,1,2 (depending on slot uMask bits)", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.PROTHDR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 0 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 0 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 1 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 1 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 2 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 2 - Other mask bits determine types of headers to count.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT2", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "Message Received : Doorbell", + "Counter": "0,1", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.DOORBELL_RCVD", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : Interrupt", + "Counter": "0,1", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.INT_PRIO", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Message Received : Interrupt : Interrupts", + "UMask": "0x10", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : IPI", + "Counter": "0,1", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.IPI_RCVD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Message Received : IPI : Inter Processor Interrupts", + "UMask": "0x4", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : MSI", + "Counter": "0,1", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.MSI_RCVD", + "PerPkg": "1", + "PublicDescription": "Message Received : MSI : Message Signaled Interrupts - interrupts sent by devices (including PCIe via IOxAPIC) (Socket Mode only)", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : VLW", + "Counter": "0,1", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.VLW_RCVD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Message Received : VLW : Virtual Logical Wire (legacy) message were received from Uncore.", + "UMask": "0x1", + "Unit": "UBOX" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-io.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-io.json new file mode 100644 index 000000000000..77f9c4cee952 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-io.json @@ -0,0 +1,1925 @@ +[ + { + "BriefDescription": "IIO Clockticks", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_IIO_CLOCKTICKS", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.ALL_PARTS", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIE Completion Buffer Inserts. Counts once per 64 byte read issued from this PCIE device.", + "Counter": "0,1,2,3", + "EventCode": "0xC2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.ALL_PARTS", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0xff", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "Count of allocations in the completion buffer", + "Counter": "2,3", + "EventCode": "0xD5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.ALL_PARTS", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reporting completion of Card read from Core DRAM", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.ALL_PARTS", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card.", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card.", + "Counter": "2,3", + "EventCode": "0xC0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Counts once for every 4 bytes read from this card to memory. This event does include reads to IO.", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x02", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x04", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x08", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x10", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x20", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x40", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x80", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Counts once for every 4 bytes written from this card to memory. This event does include writes to IO.", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x02", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x04", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x08", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x10", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x20", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x40", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Four byte data request of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x80", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_READ.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Counts once for every 4 bytes written from this card to a peer device's IO space.", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.ALL_PARTS", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB Hits to a 1G Page", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.1G_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB Hits to a 2M Page", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.2M_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB Hits to a 4K Page", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.4K_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB lookups all", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.ALL_LOOKUPS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Context cache hits", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.CTXT_CACHE_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Context cache lookups", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.CTXT_CACHE_LOOKUPS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB lookups first", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.FIRST_LOOKUPS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB Fills (same as IOTLB miss)", + "Counter": "0,1,2,3", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.MISSES", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "IOMMU memory access (both low and high priority)", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.NUM_MEM_ACCESSES", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0xc0", + "Unit": "IIO" + }, + { + "BriefDescription": "IOMMU high priority memory access", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.NUM_MEM_ACCESSES_HIGH", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "IOMMU low priority memory access", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.NUM_MEM_ACCESSES_LOW", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache Hit to a 1G page", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_1G_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache Hit to a 256T page", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_256T_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache Hit to a 2M page", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_2M_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache Hit to a 512G page", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_512G_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache fill", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_CACHE_FILLS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "Second Level Page Walk Cache lookup", + "Counter": "0,1,2,3", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_CACHE_LOOKUPS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Cycles PWT full", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.CYC_PWT_FULL", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Interrupt Entry cache hit", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.INT_CACHE_HITS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Interrupt Entry cache lookup", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.INT_CACHE_LOOKUPS", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "Context Cache invalidation events", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.NUM_INVAL_CTXT_CACHE", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Interrupt Entry Cache invalidation events", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.NUM_INVAL_INT_CACHE", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB invalidation events", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.NUM_INVAL_IOTLB", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PASID Cache invalidation events", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.NUM_INVAL_PASID_CACHE", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to UNC_IIO_NUM_OUTSTANDING_REQ_FROM_CPU.TO_IO]", + "Counter": "2,3", + "Deprecated": "1", + "EventCode": "0xc5", + "EventName": "UNC_IIO_NUM_OUSTANDING_REQ_FROM_CPU.TO_IO", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Occupancy of outbound request queue : To device : Counts number of outbound requests/completions IIO is currently processing [This event is alias to UNC_IIO_NUM_OUSTANDING_REQ_FROM_CPU.TO_IO]", + "Counter": "2,3", + "EventCode": "0xc5", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_FROM_CPU.TO_IO", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Passing data to be written", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.DATA", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "Issuing final read or write of line", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.FINAL_RD_WR", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Processing response from IOMMU", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.IOMMU_HIT", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Issuing to IOMMU", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.IOMMU_REQ", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Request Ownership", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.REQ_OWN", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Writing line", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_IIO_NUM_OUTSTANDING_REQ_OF_CPU.WR", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.ABORT", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.ABORT", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.CONFINED_P2P", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.CONFINED_P2P", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.LOC_P2P", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.LOC_P2P", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MCAST", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MCAST", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MEM", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MEM", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MSGB", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MSGB", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.REM_P2P", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.REM_P2P", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Posted requests sent by the integrated IO (IIO) controller to the Ubox, useful for counting message signaled interrupts (MSI).", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX_POSTED", + "FCMask": "0x01", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "All 9 bits of Page Walk Tracker Occupancy", + "Counter": "0,1,2,3", + "EventCode": "0x42", + "EventName": "UNC_IIO_PWT_OCCUPANCY", + "Experimental": "1", + "PerPkg": "1", + "PortMask": "0x000", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) reading from this card.", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "Counter": "2,3", + "EventCode": "0xC1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_READ.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART0", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x001", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART1", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x002", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART2", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x004", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART3", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x008", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART4", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x010", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART5", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x020", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART6", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x040", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "Counter": "0,1", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART7", + "Experimental": "1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x080", + "UMask": "0x2", + "Unit": "IIO" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json new file mode 100644 index 000000000000..8184c2b6b861 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json @@ -0,0 +1,883 @@ +[ + { + "BriefDescription": "DRAM Activate Count : Counts the number of DRAM Activate commands sent on this channel. Activate commands are issued to open up a page on the DRAM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_M_ACT_COUNT.ALL", + "PerPkg": "1", + "UMask": "0xf7", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Activate Count : Read transaction on Page Empty or Page Miss : Counts the number of DRAM Activate commands sent on this channel. Activate commands are issued to open up a page on the DRAM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_M_ACT_COUNT.RD", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf1", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Activate Count : Underfill Read transaction on Page Empty or Page Miss : Counts the number of DRAM Activate commands sent on this channel. Activate commands are issued to open up a page on the DRAM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_M_ACT_COUNT.UFILL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf4", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Activate Count : Write transaction on Page Empty or Page Miss : Counts the number of DRAM Activate commands sent on this channel. Activate commands are issued to open up a page on the DRAM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "Counter": "0,1,2,3", + "EventCode": "0x02", + "EventName": "UNC_M_ACT_COUNT.WR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf2", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0, all CAS operations", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.ALL", + "PerPkg": "1", + "UMask": "0xff", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0, all reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD", + "PerPkg": "1", + "UMask": "0xcf", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 regular reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_NON_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc3", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 auto-precharge reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_PRE_REG", + "PerPkg": "1", + "UMask": "0xc2", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 auto-precharge underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_PRE_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc8", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 regular reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_REG", + "PerPkg": "1", + "UMask": "0xc1", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc4", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.RD_UNDERFILL_ALL", + "PerPkg": "1", + "UMask": "0xcc", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0, all writes", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.WR", + "PerPkg": "1", + "UMask": "0xf0", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 regular writes", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.WR_NONPRE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xd0", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 0 auto-precharge writes", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT_SCH0.WR_PRE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xe0", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1, all CAS operations", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.ALL", + "PerPkg": "1", + "UMask": "0xff", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1, all reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD", + "PerPkg": "1", + "UMask": "0xcf", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 regular reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_NON_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc3", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 auto-precharge reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_PRE_REG", + "PerPkg": "1", + "UMask": "0xc2", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 auto-precharge underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_PRE_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc8", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 regular reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_REG", + "PerPkg": "1", + "UMask": "0xc1", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc4", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 underfill reads", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.RD_UNDERFILL_ALL", + "PerPkg": "1", + "UMask": "0xcc", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1, all writes", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.WR", + "PerPkg": "1", + "UMask": "0xf0", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 regular writes", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.WR_NONPRE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xd0", + "Unit": "IMC" + }, + { + "BriefDescription": "CAS count for SubChannel 1 auto-precharge writes", + "Counter": "0,1,2,3", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_COUNT_SCH1.WR_PRE", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xe0", + "Unit": "IMC" + }, + { + "BriefDescription": "Number of DRAM DCLK clock cycles while the event is enabled. DCLK is 1/4 of DRAM data rate.", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_M_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "DRAM Clockticks", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "Number of DRAM HCLK clock cycles while the event is enabled", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_M_HCLOCKTICKS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "DRAM Clockticks", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 temp readings forced 2x refresh", + "Counter": "0,1,2,3", + "EventCode": "0xA7", + "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 temp readings forced 2x refresh", + "Counter": "0,1,2,3", + "EventCode": "0xA7", + "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 temp readings forced 2x refresh", + "Counter": "0,1,2,3", + "EventCode": "0xA7", + "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM0", + "UMask": "0x4", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 temp readings forced 2x refresh", + "Counter": "0,1,2,3", + "EventCode": "0xA7", + "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM1", + "UMask": "0x8", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 MRRs was triggered/running", + "Counter": "0,1,2,3", + "EventCode": "0xA6", + "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 MRRs was triggered/running", + "Counter": "0,1,2,3", + "EventCode": "0xA6", + "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 MRRs was triggered/running", + "Counter": "0,1,2,3", + "EventCode": "0xA6", + "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM0", + "UMask": "0x4", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles MR4 MRRs was triggered/running", + "Counter": "0,1,2,3", + "EventCode": "0xA6", + "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM1", + "UMask": "0x8", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH0_RANK0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH0_RANK1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH0_RANK2", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x4", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH0_RANK3", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x8", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH1_RANK0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x10", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH1_RANK1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x20", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH1_RANK2", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x40", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode", + "Counter": "0,1,2,3", + "EventCode": "0x47", + "EventName": "UNC_M_POWERDOWN_CYCLES.SCH1_RANK3", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x80", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles a given rank is in Power Down Mode and all pages are closed", + "Counter": "0,1,2,3", + "EventCode": "0x88", + "EventName": "UNC_M_POWER_CHANNEL_PPD_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Critical level on specified DIMM and throttle level is zero.", + "Counter": "0,1,2,3", + "EventCode": "0x89", + "EventName": "UNC_M_POWER_CRITICAL_THROTTLE_CYCLES.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Critical level on specified DIMM and throttle level is zero.", + "Counter": "0,1,2,3", + "EventCode": "0x89", + "EventName": "UNC_M_POWER_CRITICAL_THROTTLE_CYCLES.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "UNC_M_POWER_THROTTLE_CYCLES.BW_SLOT0", + "Counter": "0,1,2,3", + "EventCode": "0x46", + "EventName": "UNC_M_POWER_THROTTLE_CYCLES.BW_SLOT0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "UNC_M_POWER_THROTTLE_CYCLES.BW_SLOT1", + "Counter": "0,1,2,3", + "EventCode": "0x46", + "EventName": "UNC_M_POWER_THROTTLE_CYCLES.BW_SLOT1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "MR4 temp reading is throttling", + "Counter": "0,1,2,3", + "EventCode": "0x46", + "EventName": "UNC_M_POWER_THROTTLE_CYCLES.MR4BLKEN", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x8", + "Unit": "IMC" + }, + { + "BriefDescription": "RAPL is throttling", + "Counter": "0,1,2,3", + "EventCode": "0x46", + "EventName": "UNC_M_POWER_THROTTLE_CYCLES.RAPLBLK", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x4", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.ALL", + "PerPkg": "1", + "UMask": "0xff", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.PGT", + "PerPkg": "1", + "UMask": "0xf8", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.RD", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf1", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.UFILL", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf4", + "Unit": "IMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "Counter": "0,1,2,3", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.WR", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xf2", + "Unit": "IMC" + }, + { + "BriefDescription": "Read buffer inserts on subchannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x17", + "EventName": "UNC_M_RDB_INSERTS.SCH0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "IMC" + }, + { + "BriefDescription": "Read buffer inserts on subchannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x17", + "EventName": "UNC_M_RDB_INSERTS.SCH1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "IMC" + }, + { + "BriefDescription": "Read buffer occupancy on subchannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x1a", + "EventName": "UNC_M_RDB_OCCUPANCY_SCH0", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Read buffer occupancy on subchannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x1b", + "EventName": "UNC_M_RDB_OCCUPANCY_SCH1", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue Allocations : Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.PCH0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x50", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue Allocations : Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.PCH1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xa0", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue inserts for subchannel 0, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.SCH0_PCH0", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue inserts for subchannel 0, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.SCH0_PCH1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue inserts for subchannel 1, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.SCH1_PCH0", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "IMC" + }, + { + "BriefDescription": "Read Pending Queue inserts for subchannel 1, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.SCH1_PCH1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "IMC" + }, + { + "BriefDescription": "Read pending queue occupancy for subchannel 0, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x80", + "EventName": "UNC_M_RPQ_OCCUPANCY_SCH0_PCH0", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Read pending queue occupancy for subchannel 0, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x81", + "EventName": "UNC_M_RPQ_OCCUPANCY_SCH0_PCH1", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Read pending queue occupancy for subchannel 1, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x82", + "EventName": "UNC_M_RPQ_OCCUPANCY_SCH1_PCH0", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Read pending queue occupancy for subchannel 1, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x83", + "EventName": "UNC_M_RPQ_OCCUPANCY_SCH1_PCH1", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "subevent0 - # of cycles all ranks were in SR subevent1 - # of times all ranks went into SR subevent2 -# of times ps_sr_active asserted (SRE) subevent3 - # of times ps_sr_active deasserted (SRX) subevent4 - # of times PS-&>Refresh ps_sr_req asserted (SRE) subevent5 - # of times PS-&>Refresh ps_sr_req deasserted (SRX) subevent6 - # of cycles PSCtrlr FSM was in FATAL", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_M_SELF_REFRESH.ENTER_SUCCESS", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_SELF_REFRESH.ENTER_SUCCESS", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles all ranks were in SR", + "Counter": "0,1,2,3", + "EventCode": "0x43", + "EventName": "UNC_M_SELF_REFRESH.ENTER_SUCCESS_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "-", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Critical level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Critical level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8e", + "EventName": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at High level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8d", + "EventName": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at High level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8d", + "EventName": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Normal level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8b", + "EventName": "UNC_M_THROTTLE_LOW_CYCLES.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_LOW_CYCLES.SLOT0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Normal level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8b", + "EventName": "UNC_M_THROTTLE_LOW_CYCLES.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_LOW_CYCLES.SLOT1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Mid level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8c", + "EventName": "UNC_M_THROTTLE_MID_CYCLES.SLOT0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_MID_CYCLES.SLOT0", + "UMask": "0x1", + "Unit": "IMC" + }, + { + "BriefDescription": "# of cycles Throttling at Mid level on specified DIMM", + "Counter": "0,1,2,3", + "EventCode": "0x8c", + "EventName": "UNC_M_THROTTLE_MID_CYCLES.SLOT1", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "UNC_M_THROTTLE_MID_CYCLES.SLOT1", + "UMask": "0x2", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.PCH0", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x50", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.PCH1", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0xa0", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue inserts for subchannel 0, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.SCH0_PCH0", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue inserts for subchannel 0, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.SCH0_PCH1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue inserts for subchannel 1, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.SCH1_PCH0", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "IMC" + }, + { + "BriefDescription": "Write Pending Queue inserts for subchannel 1, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x22", + "EventName": "UNC_M_WPQ_INSERTS.SCH1_PCH1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "IMC" + }, + { + "BriefDescription": "Write pending queue occupancy for subchannel 0, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x84", + "EventName": "UNC_M_WPQ_OCCUPANCY_SCH0_PCH0", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Write pending queue occupancy for subchannel 0, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x85", + "EventName": "UNC_M_WPQ_OCCUPANCY_SCH0_PCH1", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Write pending queue occupancy for subchannel 1, pseudochannel 0", + "Counter": "0,1,2,3", + "EventCode": "0x86", + "EventName": "UNC_M_WPQ_OCCUPANCY_SCH1_PCH0", + "PerPkg": "1", + "Unit": "IMC" + }, + { + "BriefDescription": "Write pending queue occupancy for subchannel 1, pseudochannel 1", + "Counter": "0,1,2,3", + "EventCode": "0x87", + "EventName": "UNC_M_WPQ_OCCUPANCY_SCH1_PCH1", + "PerPkg": "1", + "Unit": "IMC" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-power.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-power.json new file mode 100644 index 000000000000..9ea852ef190e --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-power.json @@ -0,0 +1,109 @@ +[ + { + "BriefDescription": "PCU Clockticks", + "Counter": "0,1,2,3", + "EventCode": "0x01", + "EventName": "UNC_P_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "PCU Clockticks: The PCU runs off a fixed 1 GHz clock. This event counts the number of pclk cycles measured while the counter was enabled. The pclk, like the Memory Controller's dclk, counts at a constant rate making it a good measure of actual wall time.", + "Unit": "PCU" + }, + { + "BriefDescription": "Thermal Strongest Upper Limit Cycles", + "Counter": "0,1,2,3", + "EventCode": "0x04", + "EventName": "UNC_P_FREQ_MAX_LIMIT_THERMAL_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Thermal Strongest Upper Limit Cycles : Number of cycles any frequency is reduced due to a thermal limit. Count only if throttling is occurring.", + "Unit": "PCU" + }, + { + "BriefDescription": "Power Strongest Upper Limit Cycles", + "Counter": "0,1,2,3", + "EventCode": "0x05", + "EventName": "UNC_P_FREQ_MAX_POWER_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Power Strongest Upper Limit Cycles : Counts the number of cycles when power is the upper limit on frequency.", + "Unit": "PCU" + }, + { + "BriefDescription": "Cycles spent changing Frequency", + "Counter": "0,1,2,3", + "EventCode": "0x74", + "EventName": "UNC_P_FREQ_TRANS_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Cycles spent changing Frequency : Counts the number of cycles when the system is changing frequency. This can not be filtered by thread ID. One can also use it with the occupancy counter that monitors number of threads in C0 to estimate the performance impact that frequency transitions had on the system.", + "Unit": "PCU" + }, + { + "BriefDescription": "Package C State Residency - C2E", + "Counter": "0,1,2,3", + "EventCode": "0x2b", + "EventName": "UNC_P_PKG_RESIDENCY_C2E_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Package C State Residency - C2E : Counts the number of cycles when the package was in C2E. This event can be used in conjunction with edge detect to count C2E entrances (or exits using invert). Residency events do not include transition times.", + "Unit": "PCU" + }, + { + "BriefDescription": "Package C State Residency - C6", + "Counter": "0,1,2,3", + "EventCode": "0x2d", + "EventName": "UNC_P_PKG_RESIDENCY_C6_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Package C State Residency - C6 : Counts the number of cycles when the package was in C6. This event can be used in conjunction with edge detect to count C6 entrances (or exits using invert). Residency events do not include transition times.", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C0", + "Counter": "0,1,2,3", + "EventCode": "0x35", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C0", + "PerPkg": "1", + "PublicDescription": "Number of cores in C0 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C3", + "Counter": "0,1,2,3", + "EventCode": "0x36", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C3", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Number of cores in C3 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C6", + "Counter": "0,1,2,3", + "EventCode": "0x37", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C6", + "PerPkg": "1", + "PublicDescription": "Number of cores in C6 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "External Prochot", + "Counter": "0,1,2,3", + "EventCode": "0x0a", + "EventName": "UNC_P_PROCHOT_EXTERNAL_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "External Prochot : Counts the number of cycles that we are in external PROCHOT mode. This mode is triggered when a sensor off the die determines that something off-die (like DRAM) is too hot and must throttle to avoid damaging the chip.", + "Unit": "PCU" + }, + { + "BriefDescription": "Internal Prochot", + "Counter": "0,1,2,3", + "EventCode": "0x09", + "EventName": "UNC_P_PROCHOT_INTERNAL_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Internal Prochot : Counts the number of cycles that we are in Internal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.", + "Unit": "PCU" + } +] diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/virtual-memory.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/virtual-memory.json index 78f2b835c1fa..a6a8733de235 100644 --- a/tools/perf/pmu-events/arch/x86/clearwaterforest/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/virtual-memory.json @@ -1,4 +1,20 @@ [ + { + "BriefDescription": "Counts the number of page walks initiated by a demand load that missed the first and second level TLBs.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x08", + "EventName": "DTLB_LOAD_MISSES.MISS_CAUSED_WALK", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of first level TLB misses but second level hits due to a demand load that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x08", + "EventName": "DTLB_LOAD_MISSES.STLB_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of page walks completed due to load DTLB misses to any page size.", "Counter": "0,1,2,3,4,5,6,7", @@ -8,6 +24,32 @@ "SampleAfterValue": "1000003", "UMask": "0xe" }, + { + "BriefDescription": "Counts the number of page walks completed due to load DTLB misses to a 4K page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x08", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts the number of page walks completed due to loads (including SW prefetches) whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages. Includes page walks that page fault.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of page walks initiated by a store that missed the first and second level TLBs.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x49", + "EventName": "DTLB_STORE_MISSES.MISS_CAUSED_WALK", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of first level TLB misses but second level hits due to stores that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x49", + "EventName": "DTLB_STORE_MISSES.STLB_HIT", + "PublicDescription": "Counts the number of first level TLB misses but second level hits due to a demand load that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of page walks completed due to store DTLB misses to any page size.", "Counter": "0,1,2,3,4,5,6,7", @@ -17,6 +59,31 @@ "SampleAfterValue": "1000003", "UMask": "0xe" }, + { + "BriefDescription": "Counts the number of page walks completed due to store DTLB misses to a 4K page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x49", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts the number of page walks completed due to stores whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages. Includes page walks that page fault.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of page walks initiated by a instruction fetch that missed the first and second level TLBs.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x85", + "EventName": "ITLB_MISSES.MISS_CAUSED_WALK", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of first level TLB misses but second level hits due to an instruction fetch that did not start a page walk. Account for all pages sizes. Will result in an ITLB write from STLB.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x85", + "EventName": "ITLB_MISSES.STLB_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of page walks completed due to instruction fetch misses to any page size.", "Counter": "0,1,2,3,4,5,6,7", @@ -25,5 +92,32 @@ "PublicDescription": "Counts the number of page walks completed due to instruction fetches whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to any page size. Includes page walks that page fault.", "SampleAfterValue": "1000003", "UMask": "0xe" + }, + { + "BriefDescription": "Counts the number of page walks completed due to instruction fetch misses to a 4K page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x85", + "EventName": "ITLB_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts the number of page walks completed due to instruction fetches whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages. Includes page walks that page fault.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of page walks outstanding for iside in PMH every cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x85", + "EventName": "ITLB_MISSES.WALK_PENDING", + "PublicDescription": "Counts the number of page walks outstanding for iside in PMH every cycle. A PMH page walk is outstanding from page walk start till PMH becomes idle again (ready to serve next walk). Includes EPT-walk intervals. Walks could be counted by edge detecting on this event, but would count restarted suspended walks.", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of retired loads that are blocked due to a first level TLB miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.DTLB_MISS", + "PublicDescription": "Counts the number of retired loads that are blocked due to a first level TLB miss. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8" } ] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json index b2f8947f6741..a44e1f027c1d 100644 --- a/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json @@ -181,6 +181,15 @@ "SampleAfterValue": "200003", "UMask": "0xff" }, + { + "BriefDescription": "All requests that hit L2 cache [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf" + }, { "BriefDescription": "Read requests with true-miss in L2 cache. [This event is alias to L2_RQSTS.MISS]", "Counter": "0,1,2,3", @@ -279,6 +288,15 @@ "SampleAfterValue": "200003", "UMask": "0x21" }, + { + "BriefDescription": "All requests that hit L2 cache [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf" + }, { "BriefDescription": "L2_RQSTS.HWPF_MISS", "Counter": "0,1,2,3", @@ -350,6 +368,15 @@ "SampleAfterValue": "200003", "UMask": "0x40" }, + { + "BriefDescription": "Cycles when L1D is locked", + "Counter": "0,1,2,3", + "EventCode": "0x42", + "EventName": "LOCK_CYCLES.CACHE_LOCK_DURATION", + "PublicDescription": "This event counts the number of cycles when the L1D is locked. It is a superset of the 0x1 mask (BUS_LOCK_CLOCKS.BUS_LOCK_DURATION).", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, { "BriefDescription": "Core-originated cacheable requests that missed L3 (Except hardware prefetches to the L3)", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/grandridge/cache.json b/tools/perf/pmu-events/arch/x86/grandridge/cache.json index 0aa921ba89b4..393ce9421c12 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/cache.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/cache.json @@ -121,6 +121,14 @@ "SampleAfterValue": "1000003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which missed in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x35", + "EventName": "MEM_BOUND_STALLS_IFETCH.L2_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x7e" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to an ICACHE or ITLB miss which hit in the LLC. If the core has access to an L3 cache, an LLC hit refers to an L3 cache hit, otherwise it counts zeros.", "Counter": "0,1,2,3,4,5,6,7", @@ -129,6 +137,14 @@ "SampleAfterValue": "1000003", "UMask": "0x6" }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to an ICACHE or ITLB miss which hit in the LLC, no snoop was required. LLC provides the data. If the core has access to an L3 cache, an LLC hit refers to an L3 cache hit, otherwise it counts zeros.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x35", + "EventName": "MEM_BOUND_STALLS_IFETCH.LLC_HIT_NOSNOOP", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to an ICACHE or ITLB miss which missed all the caches. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.", "Counter": "0,1,2,3,4,5,6,7", @@ -137,6 +153,14 @@ "SampleAfterValue": "1000003", "UMask": "0x78" }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to an ICACHE or ITLB miss which missed all the caches. DRAM, MMIO or other LOCAL memory type provides the data. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x35", + "EventName": "MEM_BOUND_STALLS_IFETCH.LLC_MISS_LOCALMEM", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to an L1 demand load miss.", "Counter": "0,1,2,3,4,5,6,7", @@ -154,6 +178,14 @@ "SampleAfterValue": "1000003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which missed in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.L2_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x7e" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC. If the core has access to an L3 cache, an LLC hit refers to an L3 cache hit, otherwise it counts zeros.", "Counter": "0,1,2,3,4,5,6,7", @@ -162,6 +194,14 @@ "SampleAfterValue": "1000003", "UMask": "0x6" }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC, a snoop was required, the snoop misses or the snoop hits but NO_FWD. LLC provides the data. If the core has access to an L3 cache, an LLC hit refers to an L3 cache hit, otherwise it counts zeros.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_HIT_SNOOP", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which missed all the local caches. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.", "Counter": "0,1,2,3,4,5,6,7", @@ -170,6 +210,14 @@ "SampleAfterValue": "1000003", "UMask": "0x78" }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled to a demand load miss and the data was provided from an unknown source. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_MISS_LOCALMEM", + "SampleAfterValue": "1000003", + "UMask": "0x50" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled to a store buffer full condition", "Counter": "0,1,2,3,4,5,6,7", @@ -178,6 +226,14 @@ "SampleAfterValue": "1000003", "UMask": "0x80" }, + { + "BriefDescription": "Counts the total number of load ops retired that miss the L3 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_UOPS_L3_MISS_RETIRED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xff" + }, { "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in DRAM", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json b/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json index 5266eed969be..c567f073713c 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json @@ -90,6 +90,14 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.STD", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of floating point operations retired that required microcode assist.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/grandridge/frontend.json b/tools/perf/pmu-events/arch/x86/grandridge/frontend.json index fef5cba533bb..8a591e31d331 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/frontend.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/frontend.json @@ -8,6 +8,54 @@ "SampleAfterValue": "200003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged with having preceded with frontend bound behavior", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ALL", + "SampleAfterValue": "1000003" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a baclear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.BRANCH_DETECT", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles /empty issue slots due to a btclear", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.BRANCH_RESTEER", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged following an ms flow due to the bubble/wasted issue slot from exiting long ms flow", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.CISC", + "PublicDescription": "Counts the number of instructions retired that were tagged following an ms flow due to the bubble/wasted issue slot from exiting long ms flow", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged every cycle the decoder is unable to send 3 uops per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.DECODE", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ICACHE", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss", "Counter": "0,1,2,3,4,5,6,7", @@ -16,6 +64,22 @@ "SampleAfterValue": "1000003", "UMask": "0x10" }, + { + "BriefDescription": "Counts the number of instruction retired tagged after a wasted issue slot if none of the previous events occurred", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.OTHER", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Counts the number of instruction retired that are tagged after a branch instruction causes bubbles/empty issue slots due to a predecode wrong.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.PREDECODE", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json b/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json index 20986b987e18..0a8f7d327150 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json @@ -260,6 +260,13 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the total number of machine clears for any reason including, but not limited to, memory ordering, memory disambiguation, SMC, and FP assist.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY", + "SampleAfterValue": "20003" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering in which an internal load passes an older store within the same CPU.", "Counter": "0,1,2,3,4,5,6,7", @@ -268,6 +275,14 @@ "SampleAfterValue": "20003", "UMask": "0x8" }, + { + "BriefDescription": "Counts the number of machines clears due to memory renaming.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, { "BriefDescription": "Counts the number of machine clears due to a page fault. Counts both I-Side and D-Side (Loads/Stores) page faults. A page fault occurs when either the page is not present, or an access violation occurs.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/grandridge/uncore-io.json b/tools/perf/pmu-events/arch/x86/grandridge/uncore-io.json index 764cf2f0b4a8..1a495a89ed57 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/uncore-io.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/uncore-io.json @@ -824,7 +824,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.ABORT", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.ABORT", @@ -836,7 +836,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.CONFINED_P2P", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.CONFINED_P2P", @@ -848,7 +848,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.LOC_P2P", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.LOC_P2P", @@ -860,7 +860,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MCAST", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MCAST", @@ -872,7 +872,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MEM", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MEM", @@ -884,7 +884,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MSGB", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MSGB", @@ -896,7 +896,7 @@ "Unit": "IIO" }, { - "BriefDescription": "-", + "BriefDescription": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX", "Counter": "0,1,2,3", "EventCode": "0x8e", "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX", @@ -915,7 +915,6 @@ "FCMask": "0x01", "PerPkg": "1", "PortMask": "0x0FF", - "PublicDescription": "-", "UMask": "0x4", "Unit": "IIO" }, diff --git a/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json b/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json index 6a11e5505957..8413391d752c 100644 --- a/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json @@ -195,7 +195,7 @@ "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM0", "UMask": "0x1", "Unit": "IMC" }, @@ -206,7 +206,7 @@ "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH0_DIMM1", "UMask": "0x2", "Unit": "IMC" }, @@ -217,7 +217,7 @@ "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM0", "UMask": "0x4", "Unit": "IMC" }, @@ -228,7 +228,7 @@ "EventName": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_MR4_2XREF_CYCLES.SCH1_DIMM1", "UMask": "0x8", "Unit": "IMC" }, @@ -239,7 +239,7 @@ "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM0", "UMask": "0x1", "Unit": "IMC" }, @@ -250,7 +250,7 @@ "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH0_DIMM1", "UMask": "0x2", "Unit": "IMC" }, @@ -261,7 +261,7 @@ "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM0", "UMask": "0x4", "Unit": "IMC" }, @@ -272,7 +272,7 @@ "EventName": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_PDC_MR4ACTIVE_CYCLES.SCH1_DIMM1", "UMask": "0x8", "Unit": "IMC" }, @@ -617,7 +617,7 @@ "EventName": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT0", "UMask": "0x1", "Unit": "IMC" }, @@ -628,7 +628,7 @@ "EventName": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_CRIT_CYCLES.SLOT1", "UMask": "0x2", "Unit": "IMC" }, @@ -639,7 +639,7 @@ "EventName": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT0", "UMask": "0x1", "Unit": "IMC" }, @@ -650,7 +650,7 @@ "EventName": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_HIGH_CYCLES.SLOT1", "UMask": "0x2", "Unit": "IMC" }, @@ -661,7 +661,7 @@ "EventName": "UNC_M_THROTTLE_LOW_CYCLES.SLOT0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_LOW_CYCLES.SLOT0", "UMask": "0x1", "Unit": "IMC" }, @@ -672,7 +672,7 @@ "EventName": "UNC_M_THROTTLE_LOW_CYCLES.SLOT1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_LOW_CYCLES.SLOT1", "UMask": "0x2", "Unit": "IMC" }, @@ -683,7 +683,7 @@ "EventName": "UNC_M_THROTTLE_MID_CYCLES.SLOT0", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_MID_CYCLES.SLOT0", "UMask": "0x1", "Unit": "IMC" }, @@ -694,7 +694,7 @@ "EventName": "UNC_M_THROTTLE_MID_CYCLES.SLOT1", "Experimental": "1", "PerPkg": "1", - "PublicDescription": "-", + "PublicDescription": "UNC_M_THROTTLE_MID_CYCLES.SLOT1", "UMask": "0x2", "Unit": "IMC" }, diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/cache.json b/tools/perf/pmu-events/arch/x86/graniterapids/cache.json index db28866444b6..95d6b46f3445 100644 --- a/tools/perf/pmu-events/arch/x86/graniterapids/cache.json +++ b/tools/perf/pmu-events/arch/x86/graniterapids/cache.json @@ -296,6 +296,15 @@ "SampleAfterValue": "200003", "UMask": "0x40" }, + { + "BriefDescription": "Cycles when L1D is locked", + "Counter": "0,1,2,3", + "EventCode": "0x42", + "EventName": "LOCK_CYCLES.CACHE_LOCK_DURATION", + "PublicDescription": "This event counts the number of cycles when the L1D is locked. It is a superset of the 0x1 mask (BUS_LOCK_CLOCKS.BUS_LOCK_DURATION).", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, { "BriefDescription": "Core-originated cacheable requests that missed L3 (Except hardware prefetches to the L3)", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json index 5eb1145f204f..9f0c4c7198b0 100644 --- a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json @@ -808,6 +808,16 @@ "PerPkg": "1", "Unit": "IRP" }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Write Transactions as Secondary", + "Counter": "0,1,2,3", + "EventCode": "0x1E", + "EventName": "UNC_I_MISC0.2ND_WR_INSERT", + "Experimental": "1", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "IRP" + }, { "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects", "Counter": "0,1,2,3", diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json index f559e27e2815..9cd2905726fd 100644 --- a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json @@ -539,7 +539,7 @@ "Unit": "IMC" }, { - "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.", + "BriefDescription": "DRAM Precharge commands. : Precharge due to page table : Counts the number of DRAM Precharge commands sent on this channel.", "Counter": "0,1,2,3", "EventCode": "0x03", "EventName": "UNC_M_PRE_COUNT.PGT", diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/cache.json b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json index 2db3e8a51fbd..5b350233a5e1 100644 --- a/tools/perf/pmu-events/arch/x86/lunarlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json @@ -1,6 +1,6 @@ [ { - "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.", + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0x31", "EventName": "CORE_REJECT_L2Q.ANY", @@ -289,6 +289,16 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts the number of total L2 Cache Accesses that resulted in a Miss from a front door request only (does not include rejects or recycles), per core event", "Counter": "0,1,2,3,4,5,6,7", @@ -387,6 +397,16 @@ "UMask": "0x21", "Unit": "cpu_core" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Read requests with true-miss in L2 cache [This event is alias to L2_REQUEST.MISS]", "Counter": "0,1,2,3,4,5,6,7,8,9", diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json index d98723b3cd78..a7467b2f291d 100644 --- a/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json @@ -190,7 +190,6 @@ "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD", "PublicDescription": "Counts taken forward conditional branch instructions retired. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x102", "Unit": "cpu_core" }, { @@ -315,6 +314,24 @@ "UMask": "0xfd", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of relative JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.REL_JMP", + "SampleAfterValue": "200003", + "UMask": "0xdf", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of taken branch instructions retired", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -437,7 +454,7 @@ "EventName": "BR_MISP_RETIRED.COND_TAKEN_BWD_COST", "PublicDescription": "number of branch instructions retired that were mispredicted and taken backward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x8001", + "UMask": "0x41", "Unit": "cpu_core" }, { @@ -466,7 +483,7 @@ "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD_COST", "PublicDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1", "SampleAfterValue": "400009", - "UMask": "0x8002", + "UMask": "0x140", "Unit": "cpu_core" }, { @@ -566,11 +583,11 @@ "Unit": "cpu_core" }, { - "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS", + "BriefDescription": "This event counts the number of mispredicted ret instructions retired.", "Counter": "0,1,2,3,4,5,6,7,8,9", "EventCode": "0xc5", "EventName": "BR_MISP_RETIRED.RET", - "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. Available PDIST counters: 0,1", + "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0,1", "SampleAfterValue": "100007", "UMask": "0x8", "Unit": "cpu_core" @@ -1364,6 +1381,15 @@ "UMask": "0x88", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts number of virtual trap actually taken (e.g. highest priority event during retirement). It can count virtual trap from FPC port 0 or port 1 (x87/SSE) equally in a single counter.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of nukes due to memory renaming", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json index 63c4aa2791e4..a1e79f06645a 100644 --- a/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json @@ -1,6 +1,30 @@ [ { - "BriefDescription": "Read CAS command sent to DRAM", + "BriefDescription": "ACT command for a read request sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x24", + "EventName": "UNC_M_ACT_COUNT_RD", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "ACT command sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x26", + "EventName": "UNC_M_ACT_COUNT_TOTAL", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "ACT command for a write request sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x25", + "EventName": "UNC_M_ACT_COUNT_WR", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read CAS command sent to DRAM.", "Counter": "0,1,2,3,4", "EventCode": "0x22", "EventName": "UNC_M_CAS_COUNT_RD", @@ -8,7 +32,7 @@ "Unit": "iMC" }, { - "BriefDescription": "Write CAS command sent to DRAM", + "BriefDescription": "Write CAS command sent to DRAM.", "Counter": "0,1,2,3,4", "EventCode": "0x23", "EventName": "UNC_M_CAS_COUNT_WR", @@ -16,7 +40,94 @@ "Unit": "iMC" }, { - "BriefDescription": "Any Rank at Hot state", + "BriefDescription": "Counting the number of clocks.", + "Counter": "0,1,2,3,4", + "EventCode": "0x01", + "EventName": "UNC_M_CLOCKTICKS", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE in DRAM is low.", + "Counter": "0,1,2,3,4", + "EventCode": "0x29", + "EventName": "UNC_M_DRAM_CKE_OFF_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Empty.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1D", + "EventName": "UNC_M_DRAM_PAGE_EMPTY_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Empty", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Empty.", + "Counter": "0,1,2,3,4", + "EventCode": "0x20", + "EventName": "UNC_M_DRAM_PAGE_EMPTY_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Empty", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Hit.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1C", + "EventName": "UNC_M_DRAM_PAGE_HIT_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Hit", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Hit.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1F", + "EventName": "UNC_M_DRAM_PAGE_HIT_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Hit", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Miss.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1E", + "EventName": "UNC_M_DRAM_PAGE_MISS_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Miss", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Miss.", + "Counter": "0,1,2,3,4", + "EventCode": "0x21", + "EventName": "UNC_M_DRAM_PAGE_MISS_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Miss", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM in Self-refresh (all channels).", + "Counter": "0,1,2,3,4", + "EventCode": "0x12", + "EventName": "UNC_M_DRAM_SELF_REFRESH", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Any Rank at Hot state.", "Counter": "0,1,2,3,4", "EventCode": "0x19", "EventName": "UNC_M_DRAM_THERMAL_HOT", @@ -25,7 +136,7 @@ "Unit": "iMC" }, { - "BriefDescription": "Any Rank at Warm state", + "BriefDescription": "Any Rank at Warm state.", "Counter": "0,1,2,3,4", "EventCode": "0x1A", "EventName": "UNC_M_DRAM_THERMAL_WARM", @@ -33,6 +144,42 @@ "PerPkg": "1", "Unit": "iMC" }, + { + "BriefDescription": "PRE command sent to DRAM for a read/write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x27", + "EventName": "UNC_M_PRE_COUNT_PAGE_MISS", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.", + "Counter": "0,1,2,3,4", + "EventCode": "0x3A", + "EventName": "UNC_M_RD_DATA", + "PerPkg": "1", + "PublicDescription": "This counter counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of VC0 read in channel0 - this event can increment by more than 1 (per channel/sub-ch).", + "Counter": "0,1,2,3,4", + "EventCode": "0x13", + "EventName": "UNC_M_RD_OCCUPANCY_CH0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Number of VC0 read in channel0 - this event can increment by more than 1 (per channel/sub-ch)", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of VC0 read in channel1 - this event can increment by more than 1 (per channel/sub-ch).", + "Counter": "0,1,2,3,4", + "EventCode": "0x14", + "EventName": "UNC_M_RD_OCCUPANCY_CH1", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, { "BriefDescription": "Total number of read and write byte transfers to/from DRAM, in 32B chunk, per DDR channel. Counter increments by 1 after sending or receiving 32B chunk data.", "Counter": "0,1,2,3,4", @@ -40,5 +187,58 @@ "EventName": "UNC_M_TOTAL_DATA", "PerPkg": "1", "Unit": "iMC" + }, + { + "BriefDescription": "Total number of requests entering MC, this is the sum of all RD + WR requests for all VCs.", + "Counter": "0,1,2,3,4", + "EventCode": "0x39", + "EventName": "UNC_M_TOTAL_REQUESTS", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC0 read request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x02", + "EventName": "UNC_M_VC0_REQUESTS_RD", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC0 write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x03", + "EventName": "UNC_M_VC0_REQUESTS_WR", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC1 read request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x04", + "EventName": "UNC_M_VC1_REQUESTS_RD", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC1 write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x05", + "EventName": "UNC_M_VC1_REQUESTS_WR", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts number of bytes written, in 32B chunk, per DDR channel. Counter increments by 1 after sending 32B chunk data.", + "Counter": "0,1,2,3,4", + "EventCode": "0x3B", + "EventName": "UNC_M_WR_DATA", + "PerPkg": "1", + "Unit": "iMC" } ] diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 8a9e1735e21e..a7f870669827 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -1,19 +1,19 @@ Family-model,Version,Filename,EventType -GenuineIntel-6-(97|9A|B7|BA|BF),v1.37,alderlake,core -GenuineIntel-6-BE,v1.37,alderlaken,core -GenuineIntel-6-C[56],v1.16,arrowlake,core +GenuineIntel-6-(97|9A|B7|BA|BF),v1.39,alderlake,core +GenuineIntel-6-BE,v1.39,alderlaken,core +GenuineIntel-6-C[56],v1.19,arrowlake,core GenuineIntel-6-(1C|26|27|35|36),v5,bonnell,core GenuineIntel-6-(3D|47),v30,broadwell,core GenuineIntel-6-56,v12,broadwellde,core GenuineIntel-6-4F,v23,broadwellx,core GenuineIntel-6-55-[56789ABCDEF],v1.25,cascadelakex,core -GenuineIntel-6-DD,v1.00,clearwaterforest,core +GenuineIntel-6-DD,v1.02,clearwaterforest,core GenuineIntel-6-9[6C],v1.05,elkhartlake,core -GenuineIntel-6-CF,v1.21,emeraldrapids,core +GenuineIntel-6-CF,v1.24,emeraldrapids,core GenuineIntel-6-5[CF],v13,goldmont,core GenuineIntel-6-7A,v1.01,goldmontplus,core -GenuineIntel-6-B6,v1.11,grandridge,core -GenuineIntel-6-A[DE],v1.17,graniterapids,core +GenuineIntel-6-B6,v1.12,grandridge,core +GenuineIntel-6-A[DE],v1.19,graniterapids,core GenuineIntel-6-(3C|45|46),v36,haswell,core GenuineIntel-6-3F,v29,haswellx,core GenuineIntel-6-7[DE],v1.24,icelake,core @@ -22,20 +22,20 @@ GenuineIntel-6-3A,v24,ivybridge,core GenuineIntel-6-3E,v24,ivytown,core GenuineIntel-6-2D,v24,jaketown,core GenuineIntel-6-(57|85),v16,knightslanding,core -GenuineIntel-6-BD,v1.21,lunarlake,core -GenuineIntel-6-(AA|AC|B5),v1.20,meteorlake,core +GenuineIntel-6-BD,v1.25,lunarlake,core +GenuineIntel-6-(AA|AC|B5),v1.21,meteorlake,core GenuineIntel-6-1[AEF],v4,nehalemep,core GenuineIntel-6-2E,v4,nehalemex,core -GenuineIntel-6-CC,v1.04,pantherlake,core +GenuineIntel-6-(CC|D5),v1.06,pantherlake,core GenuineIntel-6-A7,v1.04,rocketlake,core GenuineIntel-6-2A,v19,sandybridge,core -GenuineIntel-6-8F,v1.36,sapphirerapids,core -GenuineIntel-6-AF,v1.15,sierraforest,core +GenuineIntel-6-8F,v1.39,sapphirerapids,core +GenuineIntel-6-AF,v1.17,sierraforest,core GenuineIntel-6-(37|4A|4C|4D|5A),v15,silvermont,core GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v59,skylake,core GenuineIntel-6-55-[01234],v1.37,skylakex,core GenuineIntel-6-86,v1.23,snowridgex,core -GenuineIntel-6-8[CD],v1.18,tigerlake,core +GenuineIntel-6-8[CD],v1.19,tigerlake,core GenuineIntel-6-2C,v5,westmereep-dp,core GenuineIntel-6-25,v4,westmereep-sp,core GenuineIntel-6-2F,v4,westmereex,core diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json index 4c1220c19456..6419bc36f249 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json @@ -1,4 +1,13 @@ [ + { + "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x31", + "EventName": "CORE_REJECT_L2Q.ANY", + "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q. It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link. The L2Q may also reject transactions from a core to insure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops. (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.", + "SampleAfterValue": "200003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L1D cacheline (dirty) evictions caused by load misses, stores, and prefetches.", "Counter": "0,1,2,3,4,5,6,7", @@ -181,6 +190,15 @@ "UMask": "0x4", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x30", + "EventName": "L2_REJECT_XQ.ANY", + "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link. The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).", + "SampleAfterValue": "200003", + "Unit": "cpu_atom" + }, { "BriefDescription": "All accesses to L2 cache [This event is alias to L2_RQSTS.REFERENCES]", "Counter": "0,1,2,3", @@ -885,6 +903,33 @@ "UMask": "0x20", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and modified data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_HITM", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and no data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_NO_FWD", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and non-modified data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_WITH_FWD", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of load ops retired that miss the L3 cache and hit in DRAM", "Counter": "0,1,2,3,4,5,6,7", @@ -993,6 +1038,16 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "SampleAfterValue": "200003", + "UMask": "0x83", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of load ops retired.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/floating-point.json b/tools/perf/pmu-events/arch/x86/meteorlake/floating-point.json index 28dc5e06ee31..1ccbd54904c5 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/floating-point.json @@ -19,6 +19,24 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of active floating point dividers per cycle in the loop stage.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of floating point divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts all microcode FP assists.", "Counter": "0,1,2,3,4,5,6,7", @@ -274,6 +292,51 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of uops executed on all floating point ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xf", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0xe", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/memory.json b/tools/perf/pmu-events/arch/x86/meteorlake/memory.json index f0cbeda4d5ca..7cdd5cb39009 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/memory.json @@ -19,6 +19,15 @@ "UMask": "0x6", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.ANY", + "SampleAfterValue": "1000003", + "UMask": "0x7f", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.", "Counter": "0,1,2,3,4,5,6,7", @@ -37,6 +46,15 @@ "UMask": "0xf4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.", "Counter": "0,1,2,3,4,5,6,7", @@ -46,6 +64,16 @@ "UMask": "0x81", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", "Counter": "0,1,2,3,4,5,6,7", @@ -56,6 +84,15 @@ "UMask": "0xc0", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.PGWALK", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.", "Counter": "0,1,2,3,4,5,6,7", @@ -65,6 +102,15 @@ "UMask": "0xa0", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store address match.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.ST_ADDR", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.", "Counter": "0,1,2,3,4,5,6,7", @@ -74,6 +120,24 @@ "UMask": "0x84", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json index 7662846745bd..09e1147c4733 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json @@ -20,6 +20,16 @@ "UMask": "0x9", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of cycles when any of the integer dividers are active.", + "Counter": "0,1,2,3,4,5,6,7", + "CounterMask": "1", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event counts the cycles the integer divider is busy.", "Counter": "0,1,2,3,4,5,6,7", @@ -30,6 +40,24 @@ "UMask": "0x8", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of active integer dividers per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of integer divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, { "BriefDescription": "Number of occurrences where a microcode assist is invoked by hardware.", "Counter": "0,1,2,3,4,5,6,7", @@ -255,6 +283,16 @@ "UMask": "0xdf", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of taken branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "Errata": "MTL013", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -929,6 +967,89 @@ "UMask": "0x10", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of uops executed on all Integer ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xff", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on a load port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.LD", + "PublicDescription": "Counts the number of uops executed on a load port. This event counts for integer uops even if the destination is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P3", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0,1, 2, 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0x78", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on a Store address port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STA", + "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STD_JMP", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, { "BriefDescription": "INT_VEC_RETIRED.128BIT", "Counter": "0,1,2,3,4,5,6,7", @@ -1131,6 +1252,24 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FAST", + "SampleAfterValue": "20003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of machines clears due to memory renaming.", "Counter": "0,1,2,3,4,5,6,7", @@ -1303,6 +1442,25 @@ "UMask": "0x4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "200003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.NON_C01_MS_SCB", + "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.", + "SampleAfterValue": "200003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/virtual-memory.json b/tools/perf/pmu-events/arch/x86/meteorlake/virtual-memory.json index 305b96b26a4e..04396c7b3e08 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/virtual-memory.json @@ -356,6 +356,15 @@ "UMask": "0x10", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.DTLB_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/cache.json b/tools/perf/pmu-events/arch/x86/pantherlake/cache.json index e5323093eec0..165e955b870c 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/cache.json @@ -1,4 +1,13 @@ [ + { + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x31", + "EventName": "CORE_REJECT_L2Q.ANY", + "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full condition which likely indicates back pressure from L2Q. It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link. The L2Q may also reject transactions from a core to ensure fairness between cores, or to delay a core's dirty eviction when the address conflicts with incoming external snoops. Note that L2 prefetcher requests that are dropped are not counted by this event. Counts on a per core basis.", + "SampleAfterValue": "1000003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cache lines replaced in L0 data cache.", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -130,6 +139,15 @@ "UMask": "0x4", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x30", + "EventName": "L2_REJECT_XQ.ANY", + "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link. The IDI link is a central on-die protocol for memory and coherence traffic between the core and the uncore. It is highly optimized for efficiency, bandwidth, and coherency, using multiple channels and flow control mechanisms to manage high-throughput, low-latency data and protocol communication within the chip. The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).", + "SampleAfterValue": "1000003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door requests for Code Read, Data Read, RFO, ITOM, and L2 Prefetches. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -158,6 +176,15 @@ "UMask": "0xc4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Code Read requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_CODE_RD_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x84", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Code Read requests that resulted in a Miss. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -176,6 +203,15 @@ "UMask": "0xc1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Data Read requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_DATA_RD_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x81", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door Demand Data Read requests that resulted in a Miss. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -194,6 +230,15 @@ "UMask": "0xc2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Demand RFO requests that resulted in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.DEMAND_RFO_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x82", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door Demand RFO requests that resulted in a Miss. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -212,6 +257,16 @@ "UMask": "0x1bf", "Unit": "cpu_atom" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -221,6 +276,24 @@ "UMask": "0xc8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests that result in a hit. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HWPF_HIT", + "SampleAfterValue": "1000003", + "UMask": "0x88", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of L2 cache accesses from front door Hardware Prefetch requests that result in a miss. Does not include rejects or recycles, per core event.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HWPF_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x48", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of L2 cache accesses from front door requests that resulted in a Miss. Does not include rejects or recycles, per core event.", "Counter": "0,1,2,3,4,5,6,7", @@ -310,6 +383,16 @@ "UMask": "0x21", "Unit": "cpu_core" }, + { + "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3,4,5,6,7,8,9", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0x5f", + "Unit": "cpu_core" + }, { "BriefDescription": "Read requests with true-miss in L2 cache [This event is alias to L2_REQUEST.MISS]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -1019,6 +1102,19 @@ "UMask": "0x82", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of tagged load uops retired.", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY", + "MSRIndex": "0x3F6", + "MSRValue": "0x1", + "PublicDescription": "Counts the number of tagged load uops retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 1024.", "Counter": "0,1,2,3,4,5,6,7", @@ -1033,27 +1129,27 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_128", "MSRIndex": "0x3F6", "MSRValue": "0x80", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 128. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 16.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_16", "MSRIndex": "0x3F6", "MSRValue": "0x10", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 16. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" @@ -1072,79 +1168,79 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_256", "MSRIndex": "0x3F6", "MSRValue": "0x100", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 256. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 32.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_32", "MSRIndex": "0x3F6", "MSRValue": "0x20", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 32. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 4.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_4", "MSRIndex": "0x3F6", "MSRValue": "0x4", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 4. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 512.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_512", "MSRIndex": "0x3F6", "MSRValue": "0x200", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 512. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 64.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_64", "MSRIndex": "0x3F6", "MSRValue": "0x40", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 64. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled.", + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 8.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_8", "MSRIndex": "0x3F6", "MSRValue": "0x8", - "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled. Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of tagged load uops retired that exceed the latency threshold of 8. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x5", "Unit": "cpu_atom" @@ -1159,6 +1255,26 @@ "UMask": "0x21", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of load uops retired that were correctly predicated by the memory renaming (MRN) feature.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.MRN_LOADS", + "PublicDescription": "Counts the number of load uops retired that were correctly predicated by the memory renaming (MRN) feature. MRN loads are a part of the memory renaming process that optimizes data forwarding between stores and loads, reducing latency and improving performance. This involves tagging stores that forward to loads and copying the physical source (the actual data) directly from the store to the load. The MRN'd load can complete without accessing the memory, thus reducing load-to-use latency. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x9", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of store uops retired that were correctly tagged by the memory renaming (MRN) feature.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.MRN_STORES", + "PublicDescription": "Counts the number of store uops retired that were correctly tagged by the memory renaming (MRN) feature. MRN stores are a part of the memory renaming process that optimizes data forwarding between stores and loads, reducing latency and improving performance. This involves tagging stores that forward to loads and copying the physical source (the actual data) directly from the store to the load. The MRN store ensures that it's data is accurately forwarded to matching subsequent loads. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xa", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of memory uops retired that were splits.", "Counter": "0,1,2,3,4,5,6,7", @@ -1220,12 +1336,12 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number of stores uops retired same as MEM_UOPS_RETIRED.ALL_STORES", + "BriefDescription": "Counts the number of stores uops retired.", "Counter": "0,1,2,3,4,5,6,7", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.STORE_LATENCY", - "PublicDescription": "Counts the number of stores uops retired same as MEM_UOPS_RETIRED.ALL_STORES Available PDIST counters: 0,1", + "PublicDescription": "Counts the number of stores uops retired. Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x6", "Unit": "cpu_atom" diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/counter.json b/tools/perf/pmu-events/arch/x86/pantherlake/counter.json index 432b6946ccbc..9794b435f650 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/counter.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/counter.json @@ -13,5 +13,10 @@ "Unit": "iMC", "CountersNumFixed": "0", "CountersNumGeneric": "5" + }, + { + "Unit": "SANTA", + "CountersNumFixed": 1, + "CountersNumGeneric": "0" } ] \ No newline at end of file diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/frontend.json b/tools/perf/pmu-events/arch/x86/pantherlake/frontend.json index 5e69b81742f5..ef490c38569d 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/frontend.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/frontend.json @@ -120,6 +120,16 @@ "UMask": "0x3", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ICACHE", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss", "Counter": "0,1,2,3,4,5,6,7", @@ -344,6 +354,56 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L2_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xe", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x6", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and did not have to snoop another core.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT_NO_SNOOP", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and did not have to snoop another core. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and required a snoop (that resulted in a snoop miss, snoop hitm, snoop hit with fwd, or snoop hit with no fwd).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_HIT_WITH_SNOOP", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that hit in the L3 cache, and required a snoop (that resulted in a snoop miss, snoop hitm, snoop hit with fwd, or snoop hit with no fwd). Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 and L3 caches.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc9", + "EventName": "FRONTEND_RETIRED_SOURCE.ICACHE_L3_MISS", + "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to Instruction L1 cache miss, that also missed in the L2 and L3 caches. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss that hit in the second level TLB.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/memory.json b/tools/perf/pmu-events/arch/x86/pantherlake/memory.json index 4248cc101391..ac0446359de3 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/memory.json @@ -26,6 +26,26 @@ "UMask": "0x81", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER_AT_RET", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0xc0", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", "Counter": "0,1,2,3,4,5,6,7", @@ -63,6 +83,15 @@ "UMask": "0x2", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MEMORY_ORDERING_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8002", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 1024 cycles.", "Counter": "2,3,4,5,6,7,8,9", diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/other.json b/tools/perf/pmu-events/arch/x86/pantherlake/other.json index 915c52f5abd1..91dd2d25ebaf 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/other.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/other.json @@ -18,6 +18,16 @@ "UMask": "0x8", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xe4", + "EventName": "LBR_INSERTS.ANY", + "PublicDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts streaming stores that have any type of response.", "Counter": "0,1,2,3,4,5,6,7,8,9", diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json b/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json index 86009237df2f..5d5303c02954 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json @@ -258,6 +258,16 @@ "UMask": "0x30", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near direct CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIRECT_CALL", + "PublicDescription": "Counts the number of near direct CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, { "BriefDescription": "near relative call instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_REL_CALL]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -268,6 +278,16 @@ "UMask": "0x20", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near direct JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIRECT_JMP", + "PublicDescription": "Counts the number of near direct JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_DIR_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "near relative jump instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_REL_JMP]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -278,6 +298,28 @@ "UMask": "0x80", "Unit": "cpu_core" }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIR_CALL", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_DIR_JMP", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_DIRECT_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of near indirect JMP and near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.ALL_NEAR_IND]", "Counter": "0,1,2,3,4,5,6,7", @@ -298,6 +340,16 @@ "UMask": "0x50", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_CALL", + "PublicDescription": "Counts the number of near indirect CALL branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "Indirect near call instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_CALL]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -308,6 +360,16 @@ "UMask": "0x10", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near indirect JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_JMP", + "PublicDescription": "Counts the number of near indirect JMP branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Indirect near jump instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_IND_JMP]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -328,6 +390,17 @@ "UMask": "0x58", "Unit": "cpu_atom" }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_CALL]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_IND_CALL", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_CALL] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_CALL]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -339,6 +412,17 @@ "UMask": "0x10", "Unit": "cpu_core" }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_JMP]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_IND_JMP", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_JMP] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_INDIRECT_JMP]", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -350,6 +434,16 @@ "UMask": "0x40", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near JMP branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_JMP", + "PublicDescription": "Counts the number of near JMP branch instructions retired. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0xc0", + "Unit": "cpu_atom" + }, { "BriefDescription": "Indirect and Direct Relative near jump instructions retired.", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -382,6 +476,27 @@ "UMask": "0x80", "Unit": "cpu_core" }, + { + "BriefDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_RETURN]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_RET", + "PublicDescription": "This event is deprecated. [This event is alias to BR_INST_RETIRED.NEAR_RETURN] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of near RET branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_RET]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_RETURN", + "PublicDescription": "Counts the number of near RET branch instructions retired. [This event is alias to BR_INST_RETIRED.NEAR_RET] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Return instructions retired.", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -772,11 +887,32 @@ "Unit": "cpu_core" }, { - "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS [This event is alias to BR_MISP_RETIRED.RET]", + "BriefDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN]", + "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_RET", + "PublicDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET]", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_RETURN", + "PublicDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET] Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "This event counts the number of mispredicted ret instructions retired [This event is alias to BR_MISP_RETIRED.RET]", "Counter": "0,1,2,3,4,5,6,7,8,9", "EventCode": "0xc5", "EventName": "BR_MISP_RETIRED.NEAR_RETURN", - "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. [This event is alias to BR_MISP_RETIRED.RET] Available PDIST counters: 0,1", + "PublicDescription": "This event counts the number of mispredicted ret instructions retired [This event is alias to BR_MISP_RETIRED.RET] Available PDIST counters: 0,1", "SampleAfterValue": "100007", "UMask": "0x8", "Unit": "cpu_core" @@ -966,7 +1102,7 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles.", + "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.CORE]", "Counter": "Fixed counter 1", "EventName": "CPU_CLK_UNHALTED.THREAD", "SampleAfterValue": "2000003", @@ -1554,6 +1690,23 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the total number of machine clears for any reason including, but not limited to, memory ordering, memory disambiguation, SMC, and FP assist.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY", + "SampleAfterValue": "1000003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x80ff", + "Unit": "cpu_atom" + }, { "BriefDescription": "Number of machine clears (nukes) of any type.", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -1575,6 +1728,42 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.DISAMBIGUATION_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8008", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears due to memory renaming.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE_FAST", + "SampleAfterValue": "1000003", + "UMask": "0x8010", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of machine clears due to a page fault. Counts both I-Side and D-Side (Loads/Stores) page faults. A page fault occurs when either the page is not present, or an access violation occurs.", "Counter": "0,1,2,3,4,5,6,7", @@ -1603,6 +1792,16 @@ "UMask": "0x4", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of machine clears due to program modifying data (cross-core modifying code) within 1K of a recently fetched code page.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.XMC", + "PublicDescription": "Counts the number of machine clears due to program modifying data (self modifying code) within 1K of a recently fetched code page.", + "SampleAfterValue": "1000003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts cycles where no execution is happening due to loads waiting for L1 cache (that is: no execution & load in flight & no load missed L1 cache)", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -1650,11 +1849,12 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL.", + "BriefDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY]", "Counter": "0,1,2,3,4,5,6,7", + "Deprecated": "1", "EventCode": "0xe4", "EventName": "MISC_RETIRED.LBR_INSERTS", - "PublicDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. Available PDIST counters: 0,1", + "PublicDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY] Available PDIST counters: 0,1", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_atom" @@ -1751,7 +1951,7 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the number issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0x75", "EventName": "SERIALIZATION.COLOR_STALLS", @@ -1943,6 +2143,25 @@ "UMask": "0x2", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to IEC and FPC RAT stalls - which can be due to the FIQ and IEC reservation station stall (integer, FP and SIMD scheduler not being able to accept another uop).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.NON_MEM_SCHEDULER", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to mrbl stall. A marble refers to a physical register file entry, also known as the physical destination (PDST).", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.REGISTER", + "PublicDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to mrbl stall. A 'marble' refers to a physical register file entry, also known as the physical destination (PDST).", + "SampleAfterValue": "1000003", + "UMask": "0x20", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to ROB full", "Counter": "0,1,2,3,4,5,6,7", @@ -2014,6 +2233,15 @@ "UMask": "0x8", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.FRONTEND_BANDWIDTH", + "SampleAfterValue": "1000003", + "UMask": "0x8d", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to latency related stalls including BACLEARs, BTCLEARs, ITLB misses, and ICache misses.", "Counter": "0,1,2,3,4,5,6,7", @@ -2262,6 +2490,14 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the total number of uops retired.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.ALL", + "SampleAfterValue": "1000003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Cycles with retired uop(s).", "Counter": "0,1,2,3,4,5,6,7,8,9", @@ -2273,6 +2509,16 @@ "UMask": "0x2", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of uops retired that were dual destination.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.DUAL_DEST", + "PublicDescription": "Counts the number of uops retired that were dual destination. Dual destination (dual dest) micro-operations require writing to two separate destination registers or memory addresses upon execution. These uops consume two entries in the ROB and tracked separately because they involve more complex operations that necessitate multiple results. Available PDIST counters: 0,1", + "SampleAfterValue": "1000003", + "UMask": "0x10", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of uops retired that are the last uop of a macro-instruction.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json new file mode 100644 index 000000000000..69ef928d57f6 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json @@ -0,0 +1,10 @@ +[ + { + "BriefDescription": "This 48-bit fixed counter counts the UCLK cycles.", + "Counter": "FIXED", + "EventCode": "0xff", + "EventName": "UNC_CLOCK.SOCKET", + "PerPkg": "1", + "Unit": "SANTA" + } +] diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json index a881b99be5f3..8faa03e1c6d0 100644 --- a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json @@ -1,6 +1,30 @@ [ { - "BriefDescription": "Read CAS command sent to DRAM", + "BriefDescription": "ACT command for a read request sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x24", + "EventName": "UNC_M_ACT_COUNT_RD", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "ACT command sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x26", + "EventName": "UNC_M_ACT_COUNT_TOTAL", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "ACT command for a write request sent to DRAM.", + "Counter": "0,1,2,3,4", + "EventCode": "0x25", + "EventName": "UNC_M_ACT_COUNT_WR", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read CAS command sent to DRAM.", "Counter": "0,1,2,3,4", "EventCode": "0x22", "EventName": "UNC_M_CAS_COUNT_RD", @@ -8,13 +32,153 @@ "Unit": "iMC" }, { - "BriefDescription": "Write CAS command sent to DRAM", + "BriefDescription": "Write CAS command sent to DRAM.", "Counter": "0,1,2,3,4", "EventCode": "0x23", "EventName": "UNC_M_CAS_COUNT_WR", "PerPkg": "1", "Unit": "iMC" }, + { + "BriefDescription": "Counting the number of clocks.", + "Counter": "0,1,2,3,4", + "EventCode": "0x01", + "EventName": "UNC_M_CLOCKTICKS", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE in DRAM is low.", + "Counter": "0,1,2,3,4", + "EventCode": "0x29", + "EventName": "UNC_M_DRAM_CKE_OFF_CYCLES", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Empty.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1D", + "EventName": "UNC_M_DRAM_PAGE_EMPTY_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Empty", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Empty.", + "Counter": "0,1,2,3,4", + "EventCode": "0x20", + "EventName": "UNC_M_DRAM_PAGE_EMPTY_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Empty", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Hit.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1C", + "EventName": "UNC_M_DRAM_PAGE_HIT_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Hit", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Hit.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1F", + "EventName": "UNC_M_DRAM_PAGE_HIT_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Hit", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming read request page status is Page Miss.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1E", + "EventName": "UNC_M_DRAM_PAGE_MISS_RD", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming read request page status is Page Miss", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming write request page status is Page Miss.", + "Counter": "0,1,2,3,4", + "EventCode": "0x21", + "EventName": "UNC_M_DRAM_PAGE_MISS_WR", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "incoming write request page status is Page Miss", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM in Self-refresh (all channels).", + "Counter": "0,1,2,3,4", + "EventCode": "0x12", + "EventName": "UNC_M_DRAM_SELF_REFRESH", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Any Rank at Hot state.", + "Counter": "0,1,2,3,4", + "EventCode": "0x19", + "EventName": "UNC_M_DRAM_THERMAL_HOT", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Any Rank at Warm state.", + "Counter": "0,1,2,3,4", + "EventCode": "0x1A", + "EventName": "UNC_M_DRAM_THERMAL_WARM", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "PRE command sent to DRAM for a read/write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x27", + "EventName": "UNC_M_PRE_COUNT_PAGE_MISS", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.", + "Counter": "0,1,2,3,4", + "EventCode": "0x3A", + "EventName": "UNC_M_RD_DATA", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of VC0 read in channel0 - this event can increment by more than 1 (per channel/sub-ch).", + "Counter": "0,1,2,3,4", + "EventCode": "0x13", + "EventName": "UNC_M_RD_OCCUPANCY_CH0", + "Experimental": "1", + "PerPkg": "1", + "PublicDescription": "Number of VC0 read in channel0 - this event can increment by more than 1 (per channel/sub-ch)", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of VC0 read in channel1 - this event can increment by more than 1 (per channel/sub-ch).", + "Counter": "0,1,2,3,4", + "EventCode": "0x14", + "EventName": "UNC_M_RD_OCCUPANCY_CH1", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, { "BriefDescription": "Total number of read and write byte transfers to/from DRAM, in 32B chunk, per DDR channel. Counter increments by 1 after sending or receiving 32B chunk data.", "Counter": "0,1,2,3,4", @@ -22,5 +186,58 @@ "EventName": "UNC_M_TOTAL_DATA", "PerPkg": "1", "Unit": "iMC" + }, + { + "BriefDescription": "Total number of requests entering MC, this is the sum of all RD + WR requests for all VCs.", + "Counter": "0,1,2,3,4", + "EventCode": "0x39", + "EventName": "UNC_M_TOTAL_REQUESTS", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC0 read request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x02", + "EventName": "UNC_M_VC0_REQUESTS_RD", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC0 write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x03", + "EventName": "UNC_M_VC0_REQUESTS_WR", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC1 read request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x04", + "EventName": "UNC_M_VC1_REQUESTS_RD", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Incoming VC1 write request.", + "Counter": "0,1,2,3,4", + "EventCode": "0x05", + "EventName": "UNC_M_VC1_REQUESTS_WR", + "Experimental": "1", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts number of bytes written, in 32B chunk, per DDR channel. Counter increments by 1 after sending 32B chunk data.", + "Counter": "0,1,2,3,4", + "EventCode": "0x3B", + "EventName": "UNC_M_WR_DATA", + "PerPkg": "1", + "Unit": "iMC" } ] diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json index 373b26c84448..4c096b5e6766 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json @@ -181,6 +181,15 @@ "SampleAfterValue": "200003", "UMask": "0xff" }, + { + "BriefDescription": "All requests that hit L2 cache [This event is alias to L2_RQSTS.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_REQUEST.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf" + }, { "BriefDescription": "Read requests with true-miss in L2 cache. [This event is alias to L2_RQSTS.MISS]", "Counter": "0,1,2,3", @@ -279,6 +288,15 @@ "SampleAfterValue": "200003", "UMask": "0x21" }, + { + "BriefDescription": "All requests that hit L2 cache [This event is alias to L2_REQUEST.HIT]", + "Counter": "0,1,2,3", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HIT", + "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]", + "SampleAfterValue": "200003", + "UMask": "0xdf" + }, { "BriefDescription": "L2_RQSTS.HWPF_MISS", "Counter": "0,1,2,3", @@ -350,6 +368,15 @@ "SampleAfterValue": "200003", "UMask": "0x40" }, + { + "BriefDescription": "Cycles when L1D is locked", + "Counter": "0,1,2,3", + "EventCode": "0x42", + "EventName": "LOCK_CYCLES.CACHE_LOCK_DURATION", + "PublicDescription": "This event counts the number of cycles when the L1D is locked. It is a superset of the 0x1 mask (BUS_LOCK_CLOCKS.BUS_LOCK_DURATION).", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, { "BriefDescription": "Core-originated cacheable requests that missed L3 (Except hardware prefetches to the L3)", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/cache.json b/tools/perf/pmu-events/arch/x86/sierraforest/cache.json index 168f43557a0e..71acb49b2ab9 100644 --- a/tools/perf/pmu-events/arch/x86/sierraforest/cache.json +++ b/tools/perf/pmu-events/arch/x86/sierraforest/cache.json @@ -1,4 +1,12 @@ [ + { + "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x31", + "EventName": "CORE_REJECT_L2Q.ANY", + "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q. It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link. The L2Q may also reject transactions from a core to insure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops. (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.", + "SampleAfterValue": "200003" + }, { "BriefDescription": "Counts the number of L1D cacheline (dirty) evictions caused by load misses, stores, and prefetches.", "Counter": "0,1,2,3,4,5,6,7", @@ -62,6 +70,14 @@ "SampleAfterValue": "1000003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x30", + "EventName": "L2_REJECT_XQ.ANY", + "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link. The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).", + "SampleAfterValue": "200003" + }, { "BriefDescription": "Counts the number of L2 Cache Accesses that resulted in a Hit from a front door request only (does not include rejects or recycles), per core event", "Counter": "0,1,2,3,4,5,6,7", @@ -154,6 +170,14 @@ "SampleAfterValue": "1000003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which missed in the L2 cache.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.L2_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x7e" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which hit in the LLC. If the core has access to an L3 cache, an LLC hit refers to an L3 cache hit, otherwise it counts zeros.", "Counter": "0,1,2,3,4,5,6,7", @@ -178,6 +202,14 @@ "SampleAfterValue": "1000003", "UMask": "0x78" }, + { + "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which missed all the caches, a snoop was required, and hits in other core or module on the same die. Another core provides the data with a FWD, NO_FWD, or HITM. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x34", + "EventName": "MEM_BOUND_STALLS_LOAD.LLC_MISS_OTHERMOD", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, { "BriefDescription": "Counts the number of unhalted cycles when the core is stalled to a store buffer full condition", "Counter": "0,1,2,3,4,5,6,7", @@ -227,6 +259,14 @@ "SampleAfterValue": "1000003", "UMask": "0x4" }, + { + "BriefDescription": "Counts the number of load ops retired that hit in the L3 cache in which a snoop was required and modified data was forwarded.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_UOPS_MISC_RETIRED.L3_HIT_SNOOP_HITM", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, { "BriefDescription": "Counts the number of load ops retired that hit the L1 data cache.", "Counter": "0,1,2,3,4,5,6,7", @@ -307,6 +347,15 @@ "SampleAfterValue": "20003", "UMask": "0x1" }, + { + "BriefDescription": "Counts the number of memory uops retired. A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)", + "Counter": "0,1,2,3,4,5,6,7", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL", + "SampleAfterValue": "200003", + "UMask": "0x83" + }, { "BriefDescription": "Counts the number of load ops retired.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/floating-point.json b/tools/perf/pmu-events/arch/x86/sierraforest/floating-point.json index 5266eed969be..67d52477aad6 100644 --- a/tools/perf/pmu-events/arch/x86/sierraforest/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/sierraforest/floating-point.json @@ -8,6 +8,22 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the number of active floating point dividers per cycle in the loop stage.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of floating point divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.FPDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, { "BriefDescription": "Counts the number of all types of floating point operations per uop with all default weighting", "Counter": "0,1,2,3,4,5,6,7", @@ -90,6 +106,54 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the number of uops executed on all floating point ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xf" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0xe" + }, + { + "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb2", + "EventName": "FP_VINT_UOPS_EXECUTED.STD", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of floating point operations retired that required microcode assist.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/memory.json b/tools/perf/pmu-events/arch/x86/sierraforest/memory.json index dc850a179517..01e07f90e76b 100644 --- a/tools/perf/pmu-events/arch/x86/sierraforest/memory.json +++ b/tools/perf/pmu-events/arch/x86/sierraforest/memory.json @@ -1,4 +1,12 @@ [ + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.ANY", + "SampleAfterValue": "1000003", + "UMask": "0x7f" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.", "Counter": "0,1,2,3,4,5,6,7", @@ -15,6 +23,14 @@ "SampleAfterValue": "1000003", "UMask": "0xf4" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.L1_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.", "Counter": "0,1,2,3,4,5,6,7", @@ -23,6 +39,15 @@ "SampleAfterValue": "1000003", "UMask": "0x81" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.OTHER", + "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.", "Counter": "0,1,2,3,4,5,6,7", @@ -32,6 +57,14 @@ "SampleAfterValue": "1000003", "UMask": "0xc0" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.PGWALK", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.", "Counter": "0,1,2,3,4,5,6,7", @@ -40,6 +73,14 @@ "SampleAfterValue": "1000003", "UMask": "0xa0" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store address match.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.ST_ADDR", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.", "Counter": "0,1,2,3,4,5,6,7", @@ -48,6 +89,22 @@ "SampleAfterValue": "1000003", "UMask": "0x84" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.WCB_FULL_AT_RET", + "SampleAfterValue": "1000003", + "UMask": "0x82" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/pipeline.json b/tools/perf/pmu-events/arch/x86/sierraforest/pipeline.json index cf67ff6135e0..e7ffb0042477 100644 --- a/tools/perf/pmu-events/arch/x86/sierraforest/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/sierraforest/pipeline.json @@ -8,6 +8,31 @@ "SampleAfterValue": "1000003", "UMask": "0x3" }, + { + "BriefDescription": "Counts the number of cycles when any of the integer dividers are active.", + "Counter": "0,1,2,3,4,5,6,7", + "CounterMask": "1", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of active integer dividers per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_OCCUPANCY", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of integer divider uops executed per cycle.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xcd", + "EventName": "ARITH.IDIV_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the total number of branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -121,6 +146,15 @@ "SampleAfterValue": "200003", "UMask": "0xdf" }, + { + "BriefDescription": "Counts the number of taken branch instructions retired.", + "Counter": "0,1,2,3,4,5,6,7", + "Errata": "SRF7", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.TAKEN", + "SampleAfterValue": "200003", + "UMask": "0x80" + }, { "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", "Counter": "0,1,2,3,4,5,6,7", @@ -244,6 +278,80 @@ "EventName": "INST_RETIRED.ANY_P", "SampleAfterValue": "2000003" }, + { + "BriefDescription": "Counts the number of uops executed on all Integer ports.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.ALL", + "SampleAfterValue": "1000003", + "UMask": "0xff" + }, + { + "BriefDescription": "Counts the number of uops executed on a load port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.LD", + "PublicDescription": "Counts the number of uops executed on a load port. This event counts for integer uops even if the destination is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P0", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 1.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P1", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 2.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P2", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.P3", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of uops executed on integer port 0,1, 2, 3.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.PRIMARY", + "SampleAfterValue": "1000003", + "UMask": "0x78" + }, + { + "BriefDescription": "Counts the number of uops executed on a Store address port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STA", + "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xb3", + "EventName": "INT_UOPS_EXECUTED.STD_JMP", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, { "BriefDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check.", "Counter": "0,1,2,3,4,5,6,7", @@ -268,6 +376,13 @@ "SampleAfterValue": "1000003", "UMask": "0x2" }, + { + "BriefDescription": "Counts the total number of machine clears for any reason including, but not limited to, memory ordering, memory disambiguation, SMC, and FP assist.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.ANY", + "SampleAfterValue": "20003" + }, { "BriefDescription": "Counts the number of machine clears due to memory ordering in which an internal load passes an older store within the same CPU.", "Counter": "0,1,2,3,4,5,6,7", @@ -276,6 +391,30 @@ "SampleAfterValue": "20003", "UMask": "0x8" }, + { + "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FAST", + "SampleAfterValue": "20003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts the number of virtual traps taken.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP", + "SampleAfterValue": "20003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts the number of machines clears due to memory renaming.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MRN_NUKE", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, { "BriefDescription": "Counts the number of machine clears due to a page fault. Counts both I-Side and D-Side (Loads/Stores) page faults. A page fault occurs when either the page is not present, or an access violation occurs.", "Counter": "0,1,2,3,4,5,6,7", @@ -349,6 +488,23 @@ "SampleAfterValue": "200003", "UMask": "0x4" }, + { + "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.COLOR_STALLS", + "SampleAfterValue": "200003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x75", + "EventName": "SERIALIZATION.NON_C01_MS_SCB", + "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.", + "SampleAfterValue": "200003", + "UMask": "0x2" + }, { "BriefDescription": "Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. [This event is alias to TOPDOWN_BAD_SPECULATION.ALL_P]", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/virtual-memory.json b/tools/perf/pmu-events/arch/x86/sierraforest/virtual-memory.json index 35cc5b6d41f2..88996fdac15f 100644 --- a/tools/perf/pmu-events/arch/x86/sierraforest/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/sierraforest/virtual-memory.json @@ -137,6 +137,14 @@ "SampleAfterValue": "200003", "UMask": "0x10" }, + { + "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x05", + "EventName": "LD_HEAD.DTLB_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, { "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/empty-pmu-events.c b/tools/perf/pmu-events/empty-pmu-events.c index a92dd0424f79..2af4865713be 100644 --- a/tools/perf/pmu-events/empty-pmu-events.c +++ b/tools/perf/pmu-events/empty-pmu-events.c @@ -1,6 +1,5 @@ - /* SPDX-License-Identifier: GPL-2.0 */ -/* THIS FILE WAS AUTOGENERATED BY jevents.py arch=none model=none ! */ +/* THIS FILE WAS AUTOGENERATED BY `jevents.py arch=none model=none` ! */ #include #include "util/header.h" @@ -9,2777 +8,5404 @@ #include struct compact_pmu_event { - int offset; + int offset; }; struct pmu_table_entry { - const struct compact_pmu_event *entries; - uint32_t num_entries; - struct compact_pmu_event pmu_name; + const struct compact_pmu_event *entries; + uint32_t num_entries; + struct compact_pmu_event pmu_name; }; +/* clang-format off */ static const char *const big_c_string = -/* offset=0 */ "default_core\000" -/* offset=13 */ "l1-dcache\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=99 */ "l1-dcache-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=190 */ "l1-dcache-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=286 */ "l1-dcache-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=387 */ "l1-dcache-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=482 */ "l1-dcache-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=580 */ "l1-dcache-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00000\000\000\000\000\000" -/* offset=682 */ "l1-dcache-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=782 */ "l1-dcache-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00000\000\000\000\000\000" -/* offset=874 */ "l1-dcache-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=971 */ "l1-dcache-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1073 */ "l1-dcache-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1169 */ "l1-dcache-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1268 */ "l1-dcache-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=1371 */ "l1-dcache-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=1472 */ "l1-dcache-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1563 */ "l1-dcache-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1659 */ "l1-dcache-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1760 */ "l1-dcache-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1855 */ "l1-dcache-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=1953 */ "l1-dcache-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=2055 */ "l1-dcache-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=2155 */ "l1-dcache-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=2252 */ "l1-dcache-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=2354 */ "l1-dcache-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=2461 */ "l1-dcache-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=2562 */ "l1-dcache-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=2666 */ "l1-dcache-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00000\000\000\000\000\000" -/* offset=2770 */ "l1-dcache-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=2872 */ "l1-dcache-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00000\000\000\000\000\000" -/* offset=2970 */ "l1-dcache-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3073 */ "l1-dcache-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3181 */ "l1-dcache-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3283 */ "l1-dcache-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3388 */ "l1-dcache-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=3493 */ "l1-dcache-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=3596 */ "l1-dcache-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3693 */ "l1-dcache-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3795 */ "l1-dcache-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=3902 */ "l1-dcache-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=4003 */ "l1-dcache-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=4107 */ "l1-dcache-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=4211 */ "l1-dcache-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=4313 */ "l1-dcache-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=4416 */ "l1-dcache-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=4524 */ "l1-dcache-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=4637 */ "l1-dcache-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=4744 */ "l1-dcache-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=4854 */ "l1-dcache-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00000\000\000\000\000\000" -/* offset=4964 */ "l1-dcache-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=5072 */ "l1-dcache-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00000\000\000\000\000\000" -/* offset=5177 */ "l1-dcache-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=5287 */ "l1-dcache-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=5402 */ "l1-dcache-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=5511 */ "l1-dcache-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=5623 */ "l1-dcache-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=5735 */ "l1-dcache-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=5845 */ "l1-dcache-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=5956 */ "l1-dcache-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6072 */ "l1-dcache-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6193 */ "l1-dcache-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6308 */ "l1-dcache-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6426 */ "l1-dcache-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=6544 */ "l1-dcache-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=6660 */ "l1-dcache-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6771 */ "l1-dcache-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=6887 */ "l1-dcache-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=7008 */ "l1-dcache-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=7123 */ "l1-dcache-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=7241 */ "l1-dcache-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=7359 */ "l1-dcache-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=7475 */ "l1-dcache-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=7566 */ "l1-dcache-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=7662 */ "l1-dcache-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=7752 */ "l1-dcache-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=7845 */ "l1-dcache-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=7942 */ "l1-dcache-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=8037 */ "l1-d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8118 */ "l1-d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8204 */ "l1-d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8295 */ "l1-d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8391 */ "l1-d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8481 */ "l1-d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8574 */ "l1-d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=8671 */ "l1-d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=8766 */ "l1-d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8853 */ "l1-d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=8945 */ "l1-d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9042 */ "l1-d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9133 */ "l1-d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9227 */ "l1-d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=9325 */ "l1-d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=9421 */ "l1-d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9507 */ "l1-d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9598 */ "l1-d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9694 */ "l1-d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9784 */ "l1-d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=9877 */ "l1-d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=9974 */ "l1-d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=10069 */ "l1-d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10161 */ "l1-d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10258 */ "l1-d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10360 */ "l1-d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10456 */ "l1-d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10555 */ "l1-d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=10654 */ "l1-d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=10751 */ "l1-d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10844 */ "l1-d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=10942 */ "l1-d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11045 */ "l1-d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11142 */ "l1-d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11242 */ "l1-d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=11342 */ "l1-d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=11440 */ "l1-d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11532 */ "l1-d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11629 */ "l1-d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11731 */ "l1-d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11827 */ "l1-d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=11926 */ "l1-d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=12025 */ "l1-d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=12122 */ "l1-d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12220 */ "l1-d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12323 */ "l1-d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12431 */ "l1-d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12533 */ "l1-d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12638 */ "l1-d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=12743 */ "l1-d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=12846 */ "l1-d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=12946 */ "l1-d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13051 */ "l1-d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13161 */ "l1-d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13265 */ "l1-d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13372 */ "l1-d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=13479 */ "l1-d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=13584 */ "l1-d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13690 */ "l1-d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13801 */ "l1-d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=13917 */ "l1-d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14027 */ "l1-d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14140 */ "l1-d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=14253 */ "l1-d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=14364 */ "l1-d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14470 */ "l1-d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14581 */ "l1-d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14697 */ "l1-d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14807 */ "l1-d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=14920 */ "l1-d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=15033 */ "l1-d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=15144 */ "l1-d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15230 */ "l1-d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15321 */ "l1-d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15406 */ "l1-d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15494 */ "l1-d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=15586 */ "l1-d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=15676 */ "l1d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15756 */ "l1d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15841 */ "l1d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=15931 */ "l1d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16026 */ "l1d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16115 */ "l1d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16207 */ "l1d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=16303 */ "l1d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=16397 */ "l1d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16483 */ "l1d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16574 */ "l1d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16670 */ "l1d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16760 */ "l1d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=16853 */ "l1d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=16950 */ "l1d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=17045 */ "l1d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=17130 */ "l1d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=17220 */ "l1d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=17315 */ "l1d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=17404 */ "l1d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=17496 */ "l1d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=17592 */ "l1d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=17686 */ "l1d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=17777 */ "l1d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=17873 */ "l1d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=17974 */ "l1d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18069 */ "l1d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18167 */ "l1d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=18265 */ "l1d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=18361 */ "l1d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18453 */ "l1d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18550 */ "l1d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18652 */ "l1d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18748 */ "l1d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=18847 */ "l1d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=18946 */ "l1d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=19043 */ "l1d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=19134 */ "l1d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=19230 */ "l1d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=19331 */ "l1d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=19426 */ "l1d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=19524 */ "l1d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=19622 */ "l1d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=19718 */ "l1d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=19815 */ "l1d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=19917 */ "l1d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20024 */ "l1d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20125 */ "l1d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20229 */ "l1d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=20333 */ "l1d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=20435 */ "l1d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20534 */ "l1d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20638 */ "l1d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20747 */ "l1d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20850 */ "l1d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=20956 */ "l1d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=21062 */ "l1d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=21166 */ "l1d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=21271 */ "l1d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=21381 */ "l1d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=21496 */ "l1d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=21605 */ "l1d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=21717 */ "l1d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=21829 */ "l1d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=21939 */ "l1d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=22044 */ "l1d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=22154 */ "l1d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=22269 */ "l1d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=22378 */ "l1d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=22490 */ "l1d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=22602 */ "l1d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=22712 */ "l1d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=22797 */ "l1d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=22887 */ "l1d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=22971 */ "l1d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23058 */ "l1d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=23149 */ "l1d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=23238 */ "l1-data\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23322 */ "l1-data-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23411 */ "l1-data-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23505 */ "l1-data-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23604 */ "l1-data-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23697 */ "l1-data-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=23793 */ "l1-data-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=23893 */ "l1-data-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=23991 */ "l1-data-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24081 */ "l1-data-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24176 */ "l1-data-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24276 */ "l1-data-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24370 */ "l1-data-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24467 */ "l1-data-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=24568 */ "l1-data-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=24667 */ "l1-data-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24756 */ "l1-data-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24850 */ "l1-data-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=24949 */ "l1-data-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=25042 */ "l1-data-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=25138 */ "l1-data-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=25238 */ "l1-data-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=25336 */ "l1-data-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=25431 */ "l1-data-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=25531 */ "l1-data-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=25636 */ "l1-data-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=25735 */ "l1-data-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=25837 */ "l1-data-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=25939 */ "l1-data-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=26039 */ "l1-data-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26135 */ "l1-data-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26236 */ "l1-data-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26342 */ "l1-data-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26442 */ "l1-data-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26545 */ "l1-data-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=26648 */ "l1-data-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=26749 */ "l1-data-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26844 */ "l1-data-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=26944 */ "l1-data-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=27049 */ "l1-data-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=27148 */ "l1-data-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" -/* offset=27250 */ "l1-data-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=27352 */ "l1-data-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" -/* offset=27452 */ "l1-data-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=27553 */ "l1-data-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=27659 */ "l1-data-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=27770 */ "l1-data-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=27875 */ "l1-data-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=27983 */ "l1-data-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=28091 */ "l1-data-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=28197 */ "l1-data-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=28300 */ "l1-data-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=28408 */ "l1-data-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=28521 */ "l1-data-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=28628 */ "l1-data-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=28738 */ "l1-data-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=28848 */ "l1-data-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=28956 */ "l1-data-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29065 */ "l1-data-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29179 */ "l1-data-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29298 */ "l1-data-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29411 */ "l1-data-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29527 */ "l1-data-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=29643 */ "l1-data-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=29757 */ "l1-data-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29866 */ "l1-data-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=29980 */ "l1-data-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=30099 */ "l1-data-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=30212 */ "l1-data-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" -/* offset=30328 */ "l1-data-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=30444 */ "l1-data-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" -/* offset=30558 */ "l1-data-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=30647 */ "l1-data-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=30741 */ "l1-data-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=30829 */ "l1-data-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" -/* offset=30920 */ "l1-data-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=31015 */ "l1-data-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" -/* offset=31108 */ "l1-icache\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31201 */ "l1-icache-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31299 */ "l1-icache-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31402 */ "l1-icache-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31510 */ "l1-icache-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31612 */ "l1-icache-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=31717 */ "l1-icache-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00000\000\000\000\000\000" -/* offset=31826 */ "l1-icache-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=31933 */ "l1-icache-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00000\000\000\000\000\000" -/* offset=32032 */ "l1-icache-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32136 */ "l1-icache-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32245 */ "l1-icache-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32348 */ "l1-icache-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32454 */ "l1-icache-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=32564 */ "l1-icache-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=32672 */ "l1-icache-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32770 */ "l1-icache-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32873 */ "l1-icache-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=32981 */ "l1-icache-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=33083 */ "l1-icache-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=33188 */ "l1-icache-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=33297 */ "l1-icache-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=33404 */ "l1-icache-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=33514 */ "l1-icache-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=33629 */ "l1-icache-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=33749 */ "l1-icache-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=33863 */ "l1-icache-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=33980 */ "l1-icache-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00000\000\000\000\000\000" -/* offset=34097 */ "l1-icache-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=34212 */ "l1-icache-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00000\000\000\000\000\000" -/* offset=34324 */ "l1-icache-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=34441 */ "l1-icache-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=34563 */ "l1-icache-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=34679 */ "l1-icache-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=34798 */ "l1-icache-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=34917 */ "l1-icache-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=35034 */ "l1-icache-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=35152 */ "l1-icache-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=35275 */ "l1-icache-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=35403 */ "l1-icache-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=35525 */ "l1-icache-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=35650 */ "l1-icache-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=35775 */ "l1-icache-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=35898 */ "l1-icache-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=36016 */ "l1-icache-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=36139 */ "l1-icache-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=36267 */ "l1-icache-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=36389 */ "l1-icache-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=36514 */ "l1-icache-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=36639 */ "l1-icache-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=36762 */ "l1-icache-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=36860 */ "l1-icache-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=36963 */ "l1-icache-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37060 */ "l1-icache-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37160 */ "l1-icache-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=37264 */ "l1-icache-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=37366 */ "l1-i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37454 */ "l1-i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37547 */ "l1-i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37645 */ "l1-i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37748 */ "l1-i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37845 */ "l1-i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=37945 */ "l1-i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=38049 */ "l1-i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=38151 */ "l1-i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38245 */ "l1-i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38344 */ "l1-i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38448 */ "l1-i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38546 */ "l1-i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38647 */ "l1-i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=38752 */ "l1-i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=38855 */ "l1-i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=38948 */ "l1-i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=39046 */ "l1-i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=39149 */ "l1-i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=39246 */ "l1-i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=39346 */ "l1-i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=39450 */ "l1-i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=39552 */ "l1-i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=39657 */ "l1-i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=39767 */ "l1-i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=39882 */ "l1-i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=39991 */ "l1-i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40103 */ "l1-i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=40215 */ "l1-i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=40325 */ "l1-i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40432 */ "l1-i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40544 */ "l1-i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40661 */ "l1-i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40772 */ "l1-i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=40886 */ "l1-i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=41000 */ "l1-i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=41112 */ "l1-i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=41225 */ "l1-i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=41343 */ "l1-i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=41466 */ "l1-i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=41583 */ "l1-i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=41703 */ "l1-i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=41823 */ "l1-i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=41941 */ "l1-i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=42054 */ "l1-i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=42172 */ "l1-i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=42295 */ "l1-i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=42412 */ "l1-i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=42532 */ "l1-i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=42652 */ "l1-i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=42770 */ "l1-i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=42863 */ "l1-i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=42961 */ "l1-i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43053 */ "l1-i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43148 */ "l1-i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=43247 */ "l1-i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=43344 */ "l1i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43431 */ "l1i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43523 */ "l1i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43620 */ "l1i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43722 */ "l1i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43818 */ "l1i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=43917 */ "l1i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=44020 */ "l1i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=44121 */ "l1i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44214 */ "l1i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44312 */ "l1i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44415 */ "l1i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44512 */ "l1i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44612 */ "l1i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=44716 */ "l1i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=44818 */ "l1i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=44910 */ "l1i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=45007 */ "l1i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=45109 */ "l1i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=45205 */ "l1i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=45304 */ "l1i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=45407 */ "l1i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=45508 */ "l1i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=45612 */ "l1i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=45721 */ "l1i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=45835 */ "l1i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=45943 */ "l1i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46054 */ "l1i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=46165 */ "l1i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=46274 */ "l1i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46380 */ "l1i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46491 */ "l1i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46607 */ "l1i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46717 */ "l1i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=46830 */ "l1i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=46943 */ "l1i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=47054 */ "l1i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47166 */ "l1i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47283 */ "l1i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47405 */ "l1i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47521 */ "l1i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47640 */ "l1i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=47759 */ "l1i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=47876 */ "l1i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=47988 */ "l1i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=48105 */ "l1i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=48227 */ "l1i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=48343 */ "l1i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=48462 */ "l1i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=48581 */ "l1i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=48698 */ "l1i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=48790 */ "l1i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=48887 */ "l1i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=48978 */ "l1i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49072 */ "l1i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=49170 */ "l1i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=49266 */ "l1-instruction\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49364 */ "l1-instruction-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49467 */ "l1-instruction-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49575 */ "l1-instruction-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49688 */ "l1-instruction-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49795 */ "l1-instruction-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=49905 */ "l1-instruction-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=50019 */ "l1-instruction-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=50131 */ "l1-instruction-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=50235 */ "l1-instruction-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=50344 */ "l1-instruction-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=50458 */ "l1-instruction-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=50566 */ "l1-instruction-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=50677 */ "l1-instruction-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=50792 */ "l1-instruction-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=50905 */ "l1-instruction-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=51008 */ "l1-instruction-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=51116 */ "l1-instruction-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=51229 */ "l1-instruction-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=51336 */ "l1-instruction-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=51446 */ "l1-instruction-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=51560 */ "l1-instruction-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=51672 */ "l1-instruction-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=51787 */ "l1-instruction-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=51907 */ "l1-instruction-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52032 */ "l1-instruction-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52151 */ "l1-instruction-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52273 */ "l1-instruction-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=52395 */ "l1-instruction-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=52515 */ "l1-instruction-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52632 */ "l1-instruction-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52754 */ "l1-instruction-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=52881 */ "l1-instruction-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53002 */ "l1-instruction-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53126 */ "l1-instruction-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=53250 */ "l1-instruction-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=53372 */ "l1-instruction-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53495 */ "l1-instruction-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53623 */ "l1-instruction-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53756 */ "l1-instruction-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=53883 */ "l1-instruction-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54013 */ "l1-instruction-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=54143 */ "l1-instruction-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=54271 */ "l1-instruction-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54394 */ "l1-instruction-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54522 */ "l1-instruction-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54655 */ "l1-instruction-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54782 */ "l1-instruction-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" -/* offset=54912 */ "l1-instruction-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=55042 */ "l1-instruction-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" -/* offset=55170 */ "l1-instruction-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=55273 */ "l1-instruction-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=55381 */ "l1-instruction-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=55483 */ "l1-instruction-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" -/* offset=55588 */ "l1-instruction-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=55697 */ "l1-instruction-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" -/* offset=55804 */ "llc\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=55882 */ "llc-load\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=55965 */ "llc-load-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56053 */ "llc-load-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56146 */ "llc-load-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56233 */ "llc-load-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56323 */ "llc-load-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00000\000\000\000\000\000" -/* offset=56417 */ "llc-load-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=56509 */ "llc-loads\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00000\000\000\000\000\000" -/* offset=56593 */ "llc-loads-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56682 */ "llc-loads-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56776 */ "llc-loads-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56864 */ "llc-loads-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=56955 */ "llc-loads-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=57050 */ "llc-loads-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=57143 */ "llc-read\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=57226 */ "llc-read-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=57314 */ "llc-read-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=57407 */ "llc-read-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=57494 */ "llc-read-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=57584 */ "llc-read-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=57678 */ "llc-read-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=57770 */ "llc-store\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=57859 */ "llc-store-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=57953 */ "llc-store-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58052 */ "llc-store-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58145 */ "llc-store-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58241 */ "llc-store-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00000\000\000\000\000\000" -/* offset=58337 */ "llc-store-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=58431 */ "llc-stores\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00000\000\000\000\000\000" -/* offset=58521 */ "llc-stores-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58616 */ "llc-stores-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58716 */ "llc-stores-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58810 */ "llc-stores-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=58907 */ "llc-stores-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=59004 */ "llc-stores-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=59099 */ "llc-write\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=59188 */ "llc-write-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=59282 */ "llc-write-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=59381 */ "llc-write-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=59474 */ "llc-write-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=59570 */ "llc-write-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=59666 */ "llc-write-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=59760 */ "llc-prefetch\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=59855 */ "llc-prefetch-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=59955 */ "llc-prefetch-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60060 */ "llc-prefetch-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60159 */ "llc-prefetch-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60261 */ "llc-prefetch-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00000\000\000\000\000\000" -/* offset=60363 */ "llc-prefetch-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=60463 */ "llc-prefetches\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00000\000\000\000\000\000" -/* offset=60560 */ "llc-prefetches-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60662 */ "llc-prefetches-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60769 */ "llc-prefetches-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60870 */ "llc-prefetches-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=60974 */ "llc-prefetches-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=61078 */ "llc-prefetches-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=61180 */ "llc-speculative-read\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=61283 */ "llc-speculative-read-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=61391 */ "llc-speculative-read-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=61504 */ "llc-speculative-read-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=61611 */ "llc-speculative-read-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=61721 */ "llc-speculative-read-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=61831 */ "llc-speculative-read-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=61939 */ "llc-speculative-load\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=62042 */ "llc-speculative-load-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=62150 */ "llc-speculative-load-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=62263 */ "llc-speculative-load-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=62370 */ "llc-speculative-load-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=62480 */ "llc-speculative-load-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=62590 */ "llc-speculative-load-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=62698 */ "llc-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=62781 */ "llc-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=62869 */ "llc-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=62951 */ "llc-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63036 */ "llc-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=63125 */ "llc-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=63212 */ "l2\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63309 */ "l2-load\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63411 */ "l2-load-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63518 */ "l2-load-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63630 */ "l2-load-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63736 */ "l2-load-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=63845 */ "l2-load-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=63958 */ "l2-load-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=64069 */ "l2-loads\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64172 */ "l2-loads-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64280 */ "l2-loads-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64393 */ "l2-loads-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64500 */ "l2-loads-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64610 */ "l2-loads-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=64724 */ "l2-loads-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=64836 */ "l2-read\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=64938 */ "l2-read-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=65045 */ "l2-read-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=65157 */ "l2-read-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=65263 */ "l2-read-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=65372 */ "l2-read-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=65485 */ "l2-read-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=65596 */ "l2-store\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=65704 */ "l2-store-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=65817 */ "l2-store-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=65935 */ "l2-store-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66047 */ "l2-store-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66162 */ "l2-store-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=66277 */ "l2-store-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=66390 */ "l2-stores\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66499 */ "l2-stores-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66613 */ "l2-stores-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66732 */ "l2-stores-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66845 */ "l2-stores-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=66961 */ "l2-stores-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=67077 */ "l2-stores-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=67191 */ "l2-write\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=67299 */ "l2-write-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=67412 */ "l2-write-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=67530 */ "l2-write-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=67642 */ "l2-write-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" -/* offset=67757 */ "l2-write-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=67872 */ "l2-write-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" -/* offset=67985 */ "l2-prefetch\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68099 */ "l2-prefetch-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68218 */ "l2-prefetch-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68342 */ "l2-prefetch-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68460 */ "l2-prefetch-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68581 */ "l2-prefetch-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=68702 */ "l2-prefetch-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=68821 */ "l2-prefetches\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=68937 */ "l2-prefetches-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69058 */ "l2-prefetches-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69184 */ "l2-prefetches-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69304 */ "l2-prefetches-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69427 */ "l2-prefetches-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=69550 */ "l2-prefetches-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=69671 */ "l2-speculative-read\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69793 */ "l2-speculative-read-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=69920 */ "l2-speculative-read-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70052 */ "l2-speculative-read-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70178 */ "l2-speculative-read-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70307 */ "l2-speculative-read-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=70436 */ "l2-speculative-read-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=70563 */ "l2-speculative-load\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70685 */ "l2-speculative-load-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70812 */ "l2-speculative-load-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=70944 */ "l2-speculative-load-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=71070 */ "l2-speculative-load-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" -/* offset=71199 */ "l2-speculative-load-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=71328 */ "l2-speculative-load-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" -/* offset=71455 */ "l2-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=71557 */ "l2-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=71664 */ "l2-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=71765 */ "l2-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" -/* offset=71869 */ "l2-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=71977 */ "l2-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" -/* offset=72083 */ "dtlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72154 */ "dtlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72230 */ "dtlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72311 */ "dtlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72397 */ "dtlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72477 */ "dtlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72560 */ "dtlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00000\000\000\000\000\000" -/* offset=72647 */ "dtlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=72732 */ "dtlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00000\000\000\000\000\000" -/* offset=72809 */ "dtlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72891 */ "dtlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=72978 */ "dtlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73059 */ "dtlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73143 */ "dtlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=73231 */ "dtlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=73317 */ "dtlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73393 */ "dtlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73474 */ "dtlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73560 */ "dtlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73640 */ "dtlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=73723 */ "dtlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=73810 */ "dtlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=73895 */ "dtlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=73977 */ "dtlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74064 */ "dtlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74156 */ "dtlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74242 */ "dtlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74331 */ "dtlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00000\000\000\000\000\000" -/* offset=74420 */ "dtlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=74507 */ "dtlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00000\000\000\000\000\000" -/* offset=74590 */ "dtlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74678 */ "dtlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74771 */ "dtlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74858 */ "dtlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=74948 */ "dtlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=75038 */ "dtlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=75126 */ "dtlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=75208 */ "dtlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=75295 */ "dtlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=75387 */ "dtlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=75473 */ "dtlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=75562 */ "dtlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=75651 */ "dtlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=75738 */ "dtlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=75826 */ "dtlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=75919 */ "dtlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76017 */ "dtlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76109 */ "dtlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76204 */ "dtlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00000\000\000\000\000\000" -/* offset=76299 */ "dtlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=76392 */ "dtlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00000\000\000\000\000\000" -/* offset=76482 */ "dtlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76577 */ "dtlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76677 */ "dtlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76771 */ "dtlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=76868 */ "dtlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=76965 */ "dtlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=77060 */ "dtlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77156 */ "dtlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77257 */ "dtlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77363 */ "dtlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77463 */ "dtlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77566 */ "dtlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=77669 */ "dtlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=77770 */ "dtlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77866 */ "dtlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=77967 */ "dtlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=78073 */ "dtlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=78173 */ "dtlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=78276 */ "dtlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=78379 */ "dtlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=78480 */ "dtlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=78556 */ "dtlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=78637 */ "dtlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=78712 */ "dtlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=78790 */ "dtlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=78872 */ "dtlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=78952 */ "d-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79024 */ "d-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79101 */ "d-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79183 */ "d-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79270 */ "d-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79351 */ "d-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79435 */ "d-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=79523 */ "d-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=79609 */ "d-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79687 */ "d-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79770 */ "d-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79858 */ "d-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=79940 */ "d-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80025 */ "d-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=80114 */ "d-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=80201 */ "d-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80278 */ "d-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80360 */ "d-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80447 */ "d-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80528 */ "d-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=80612 */ "d-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=80700 */ "d-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=80786 */ "d-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=80869 */ "d-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=80957 */ "d-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81050 */ "d-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81137 */ "d-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81227 */ "d-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=81317 */ "d-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=81405 */ "d-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81489 */ "d-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81578 */ "d-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81672 */ "d-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81760 */ "d-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=81851 */ "d-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=81942 */ "d-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=82031 */ "d-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=82114 */ "d-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=82202 */ "d-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=82295 */ "d-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=82382 */ "d-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=82472 */ "d-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=82562 */ "d-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=82650 */ "d-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=82739 */ "d-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=82833 */ "d-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=82932 */ "d-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83025 */ "d-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83121 */ "d-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=83217 */ "d-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=83311 */ "d-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83402 */ "d-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83498 */ "d-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83599 */ "d-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83694 */ "d-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=83792 */ "d-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=83890 */ "d-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=83986 */ "d-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84083 */ "d-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84185 */ "d-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84292 */ "d-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84393 */ "d-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84497 */ "d-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=84601 */ "d-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=84703 */ "d-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84800 */ "d-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=84902 */ "d-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=85009 */ "d-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=85110 */ "d-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=85214 */ "d-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=85318 */ "d-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=85420 */ "d-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=85497 */ "d-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=85579 */ "d-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=85655 */ "d-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=85734 */ "d-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=85817 */ "d-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=85898 */ "data-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=85973 */ "data-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86053 */ "data-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86138 */ "data-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86228 */ "data-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86312 */ "data-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86399 */ "data-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=86490 */ "data-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=86579 */ "data-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86660 */ "data-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86746 */ "data-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86837 */ "data-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=86922 */ "data-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87010 */ "data-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=87102 */ "data-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=87192 */ "data-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87272 */ "data-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87357 */ "data-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87447 */ "data-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87531 */ "data-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=87618 */ "data-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=87709 */ "data-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=87798 */ "data-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=87884 */ "data-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=87975 */ "data-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88071 */ "data-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88161 */ "data-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88254 */ "data-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=88347 */ "data-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=88438 */ "data-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88525 */ "data-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88617 */ "data-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88714 */ "data-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88805 */ "data-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=88899 */ "data-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=88993 */ "data-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=89085 */ "data-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=89171 */ "data-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=89262 */ "data-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=89358 */ "data-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=89448 */ "data-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" -/* offset=89541 */ "data-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=89634 */ "data-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" -/* offset=89725 */ "data-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=89817 */ "data-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=89914 */ "data-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90016 */ "data-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90112 */ "data-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90211 */ "data-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=90310 */ "data-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=90407 */ "data-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90501 */ "data-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90600 */ "data-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90704 */ "data-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90802 */ "data-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=90903 */ "data-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=91004 */ "data-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=91103 */ "data-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91203 */ "data-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91308 */ "data-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91418 */ "data-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91522 */ "data-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91629 */ "data-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=91736 */ "data-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=91841 */ "data-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=91941 */ "data-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=92046 */ "data-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=92156 */ "data-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=92260 */ "data-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" -/* offset=92367 */ "data-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=92474 */ "data-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" -/* offset=92579 */ "data-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=92659 */ "data-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=92744 */ "data-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=92823 */ "data-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" -/* offset=92905 */ "data-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=92991 */ "data-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" -/* offset=93075 */ "itlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93153 */ "itlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93236 */ "itlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93324 */ "itlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93417 */ "itlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93504 */ "itlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93594 */ "itlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00000\000\000\000\000\000" -/* offset=93688 */ "itlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=93780 */ "itlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00000\000\000\000\000\000" -/* offset=93864 */ "itlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=93953 */ "itlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94047 */ "itlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94135 */ "itlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94226 */ "itlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=94321 */ "itlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=94414 */ "itlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94497 */ "itlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94585 */ "itlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94678 */ "itlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94765 */ "itlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=94855 */ "itlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=94949 */ "itlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=95041 */ "itlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95124 */ "itlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95212 */ "itlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95294 */ "itlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95379 */ "itlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=95468 */ "itlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=95555 */ "i-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95634 */ "i-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95718 */ "i-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95807 */ "i-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95901 */ "i-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=95989 */ "i-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96080 */ "i-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=96175 */ "i-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=96268 */ "i-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96353 */ "i-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96443 */ "i-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96538 */ "i-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96627 */ "i-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96719 */ "i-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=96815 */ "i-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=96909 */ "i-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=96993 */ "i-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97082 */ "i-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97176 */ "i-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97264 */ "i-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97355 */ "i-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=97450 */ "i-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=97543 */ "i-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97627 */ "i-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97716 */ "i-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97799 */ "i-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=97885 */ "i-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=97975 */ "i-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=98063 */ "instruction-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98152 */ "instruction-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98246 */ "instruction-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98345 */ "instruction-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98449 */ "instruction-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98547 */ "instruction-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98648 */ "instruction-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=98753 */ "instruction-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=98856 */ "instruction-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=98951 */ "instruction-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99051 */ "instruction-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99156 */ "instruction-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99255 */ "instruction-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99357 */ "instruction-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=99463 */ "instruction-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=99567 */ "instruction-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99661 */ "instruction-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99760 */ "instruction-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99864 */ "instruction-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=99962 */ "instruction-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=100063 */ "instruction-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=100168 */ "instruction-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=100271 */ "instruction-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=100365 */ "instruction-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=100464 */ "instruction-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=100557 */ "instruction-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" -/* offset=100653 */ "instruction-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=100753 */ "instruction-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" -/* offset=100851 */ "branch\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=100938 */ "branch-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101030 */ "branch-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101127 */ "branch-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101229 */ "branch-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101325 */ "branch-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101424 */ "branch-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00000\000\000\000\000\000" -/* offset=101527 */ "branch-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=101628 */ "branch-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00000\000\000\000\000\000" -/* offset=101721 */ "branch-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101819 */ "branch-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=101922 */ "branch-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102019 */ "branch-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102119 */ "branch-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=102223 */ "branch-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=102325 */ "branch-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102417 */ "branch-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102514 */ "branch-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102616 */ "branch-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102712 */ "branch-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=102811 */ "branch-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=102914 */ "branch-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=103015 */ "branch-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103107 */ "branch-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103204 */ "branch-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103295 */ "branch-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103389 */ "branch-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=103485 */ "branches-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103579 */ "branches-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103678 */ "branches-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103782 */ "branches-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103880 */ "branches-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=103981 */ "branches-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=104086 */ "branches-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=104189 */ "branches-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104284 */ "branches-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104384 */ "branches-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104489 */ "branches-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104588 */ "branches-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104690 */ "branches-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=104796 */ "branches-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=104900 */ "branches-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=104994 */ "branches-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105093 */ "branches-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105197 */ "branches-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105295 */ "branches-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105396 */ "branches-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=105501 */ "branches-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=105604 */ "branches-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105698 */ "branches-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105797 */ "branches-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105890 */ "branches-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=105986 */ "branches-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=106086 */ "branches-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=106184 */ "bpu\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106268 */ "bpu-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106357 */ "bpu-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106451 */ "bpu-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106550 */ "bpu-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106643 */ "bpu-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=106739 */ "bpu-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=106839 */ "bpu-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=106937 */ "bpu-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107027 */ "bpu-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107122 */ "bpu-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107222 */ "bpu-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107316 */ "bpu-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107413 */ "bpu-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=107514 */ "bpu-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=107613 */ "bpu-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107702 */ "bpu-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107796 */ "bpu-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107895 */ "bpu-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=107988 */ "bpu-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108084 */ "bpu-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=108184 */ "bpu-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=108282 */ "bpu-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108371 */ "bpu-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108465 */ "bpu-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108553 */ "bpu-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108644 */ "bpu-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=108739 */ "bpu-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=108832 */ "btb\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=108916 */ "btb-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109005 */ "btb-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109099 */ "btb-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109198 */ "btb-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109291 */ "btb-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109387 */ "btb-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=109487 */ "btb-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=109585 */ "btb-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109675 */ "btb-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109770 */ "btb-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109870 */ "btb-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=109964 */ "btb-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110061 */ "btb-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=110162 */ "btb-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=110261 */ "btb-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110350 */ "btb-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110444 */ "btb-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110543 */ "btb-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110636 */ "btb-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=110732 */ "btb-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=110832 */ "btb-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=110930 */ "btb-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111019 */ "btb-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111113 */ "btb-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111201 */ "btb-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111292 */ "btb-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=111387 */ "btb-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=111480 */ "bpc\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111564 */ "bpc-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111653 */ "bpc-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111747 */ "bpc-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111846 */ "bpc-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=111939 */ "bpc-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112035 */ "bpc-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=112135 */ "bpc-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=112233 */ "bpc-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112323 */ "bpc-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112418 */ "bpc-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112518 */ "bpc-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112612 */ "bpc-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112709 */ "bpc-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=112810 */ "bpc-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=112909 */ "bpc-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=112998 */ "bpc-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113092 */ "bpc-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113191 */ "bpc-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113284 */ "bpc-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113380 */ "bpc-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=113480 */ "bpc-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=113578 */ "bpc-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113667 */ "bpc-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113761 */ "bpc-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113849 */ "bpc-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" -/* offset=113940 */ "bpc-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=114035 */ "bpc-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" -/* offset=114128 */ "node\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114203 */ "node-load\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114283 */ "node-load-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114368 */ "node-load-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114458 */ "node-load-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114542 */ "node-load-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114629 */ "node-load-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00000\000\000\000\000\000" -/* offset=114720 */ "node-load-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=114809 */ "node-loads\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00000\000\000\000\000\000" -/* offset=114890 */ "node-loads-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=114976 */ "node-loads-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115067 */ "node-loads-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115152 */ "node-loads-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115240 */ "node-loads-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=115332 */ "node-loads-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=115422 */ "node-read\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115502 */ "node-read-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115587 */ "node-read-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115677 */ "node-read-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115761 */ "node-read-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=115848 */ "node-read-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=115939 */ "node-read-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=116028 */ "node-store\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116114 */ "node-store-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116205 */ "node-store-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116301 */ "node-store-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116391 */ "node-store-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116484 */ "node-store-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00000\000\000\000\000\000" -/* offset=116577 */ "node-store-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" -/* offset=116668 */ "node-stores\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00000\000\000\000\000\000" -/* offset=116755 */ "node-stores-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116847 */ "node-stores-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=116944 */ "node-stores-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117035 */ "node-stores-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117129 */ "node-stores-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" -/* offset=117223 */ "node-stores-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" -/* offset=117315 */ "node-write\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117401 */ "node-write-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117492 */ "node-write-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117588 */ "node-write-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117678 */ "node-write-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" -/* offset=117771 */ "node-write-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" -/* offset=117864 */ "node-write-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" -/* offset=117955 */ "node-prefetch\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118047 */ "node-prefetch-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118144 */ "node-prefetch-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118246 */ "node-prefetch-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118342 */ "node-prefetch-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118441 */ "node-prefetch-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00000\000\000\000\000\000" -/* offset=118540 */ "node-prefetch-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=118637 */ "node-prefetches\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00000\000\000\000\000\000" -/* offset=118731 */ "node-prefetches-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118830 */ "node-prefetches-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=118934 */ "node-prefetches-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119032 */ "node-prefetches-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119133 */ "node-prefetches-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=119234 */ "node-prefetches-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=119333 */ "node-speculative-read\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119433 */ "node-speculative-read-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119538 */ "node-speculative-read-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119648 */ "node-speculative-read-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119752 */ "node-speculative-read-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=119859 */ "node-speculative-read-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=119966 */ "node-speculative-read-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=120071 */ "node-speculative-load\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=120171 */ "node-speculative-load-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=120276 */ "node-speculative-load-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=120386 */ "node-speculative-load-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=120490 */ "node-speculative-load-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" -/* offset=120597 */ "node-speculative-load-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=120704 */ "node-speculative-load-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" -/* offset=120809 */ "node-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=120889 */ "node-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=120974 */ "node-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=121053 */ "node-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" -/* offset=121135 */ "node-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=121221 */ "node-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" -/* offset=121305 */ "cpu-cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000" -/* offset=121467 */ "cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cpu-cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000" -/* offset=121629 */ "instructions\000legacy hardware\000Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts\000legacy-hardware-config=1\000\00000\000\000\000\000\000" -/* offset=121805 */ "cache-references\000legacy hardware\000Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU\000legacy-hardware-config=2\000\00000\000\000\000\000\000" -/* offset=122075 */ "cache-misses\000legacy hardware\000Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in conjunction with the PERF_COUNT_HW_CACHE_REFERENCES event to calculate cache miss rates\000legacy-hardware-config=3\000\00000\000\000\000\000\000" -/* offset=122318 */ "branches\000legacy hardware\000Retired branch instructions [This event is an alias of branch-instructions]\000legacy-hardware-config=4\000\00000\000\000\000\000\000" -/* offset=122452 */ "branch-instructions\000legacy hardware\000Retired branch instructions [This event is an alias of branches]\000legacy-hardware-config=4\000\00000\000\000\000\000\000" -/* offset=122586 */ "branch-misses\000legacy hardware\000Mispredicted branch instructions\000legacy-hardware-config=5\000\00000\000\000\000\000\000" -/* offset=122682 */ "bus-cycles\000legacy hardware\000Bus cycles, which can be different from total cycles\000legacy-hardware-config=6\000\00000\000\000\000\000\000" -/* offset=122795 */ "stalled-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of idle-cycles-frontend]\000legacy-hardware-config=7\000\00000\000\000\000\000\000" -/* offset=122945 */ "idle-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of stalled-cycles-fronted]\000legacy-hardware-config=7\000\00000\000\000\000\000\000" -/* offset=123094 */ "stalled-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of idle-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000" -/* offset=123247 */ "idle-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of stalled-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000" -/* offset=123400 */ "ref-cycles\000legacy hardware\000Total cycles; not affected by CPU frequency scaling\000legacy-hardware-config=9\000\00000\000\000\000\000\000" -/* offset=123512 */ "software\000" -/* offset=123521 */ "cpu-clock\000software\000Per-CPU high-resolution timer based event\000config=0\000\000001e-6msec\000\000\000\000\000" -/* offset=123607 */ "task-clock\000software\000Per-task high-resolution timer based event\000config=1\000\000001e-6msec\000\000\000\000\000" -/* offset=123695 */ "faults\000software\000Number of page faults [This event is an alias of page-faults]\000config=2\000\00000\000\000\000\000\000" -/* offset=123790 */ "page-faults\000software\000Number of page faults [This event is an alias of faults]\000config=2\000\00000\000\000\000\000\000" -/* offset=123885 */ "context-switches\000software\000Number of context switches [This event is an alias of cs]\000config=3\000\00000\000\000\000\000\000" -/* offset=123986 */ "cs\000software\000Number of context switches [This event is an alias of context-switches]\000config=3\000\00000\000\000\000\000\000" -/* offset=124087 */ "cpu-migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of migrations]\000config=4\000\00000\000\000\000\000\000" -/* offset=124219 */ "migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of cpu-migrations]\000config=4\000\00000\000\000\000\000\000" -/* offset=124351 */ "minor-faults\000software\000Number of minor page faults. Minor faults don't require I/O to handle\000config=5\000\00000\000\000\000\000\000" -/* offset=124460 */ "major-faults\000software\000Number of major page faults. Major faults require I/O to handle\000config=6\000\00000\000\000\000\000\000" -/* offset=124563 */ "alignment-faults\000software\000Number of kernel handled memory alignment faults\000config=7\000\00000\000\000\000\000\000" -/* offset=124655 */ "emulation-faults\000software\000Number of kernel handled unimplemented instruction faults handled through emulation\000config=8\000\00000\000\000\000\000\000" -/* offset=124782 */ "dummy\000software\000A placeholder event that doesn't count anything\000config=9\000\00000\000\000\000\000\000" -/* offset=124862 */ "bpf-output\000software\000An event used by BPF programs to write to the perf ring buffer\000config=0xa\000\00000\000\000\000\000\000" -/* offset=124964 */ "cgroup-switches\000software\000Number of context switches to a task in a different cgroup\000config=0xb\000\00000\000\000\000\000\000" -/* offset=125067 */ "tool\000" -/* offset=125072 */ "duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000\00000\000\000\000\000\000" -/* offset=125148 */ "user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000\000\000\000\000\000" -/* offset=125218 */ "system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\000\000\000\000\000" -/* offset=125286 */ "has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000\00000\000\000\000\000\000" -/* offset=125362 */ "num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU\000config=5\000\00000\000\000\000\000\000" -/* offset=125507 */ "num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPUs on a core\000config=6\000\00000\000\000\000\000\000" -/* offset=125610 */ "num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be multiple such CPUs on a core\000config=7\000\00000\000\000\000\000\000" -/* offset=125727 */ "num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000\00000\000\000\000\000\000" -/* offset=125803 */ "num_packages\000tool\000Number of packages. Each package has 1 or more die\000config=9\000\00000\000\000\000\000\000" -/* offset=125889 */ "slots\000tool\000Number of functional units that in parallel can execute parts of an instruction\000config=0xa\000\00000\000\000\000\000\000" -/* offset=125999 */ "smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0\000config=0xb\000\00000\000\000\000\000\000" -/* offset=126106 */ "system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per second\000config=0xc\000\00000\000\000\000\000\000" -/* offset=126205 */ "core_wide\000tool\0001 if not SMT, if SMT are events being gathered on all SMT threads 1 otherwise 0\000config=0xd\000\00000\000\000\000\000\000" -/* offset=126319 */ "target_cpu\000tool\0001 if CPUs being analyzed, 0 if threads/processes\000config=0xe\000\00000\000\000\000\000\000" -/* offset=126403 */ "bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000\000\000\000" -/* offset=126465 */ "bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000\000\000\000" -/* offset=126527 */ "l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000\000\000\000Attributable Level 3 cache access, read\000" -/* offset=126625 */ "segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000\000\000\000" -/* offset=126727 */ "dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000\000\000\000" -/* offset=126860 */ "eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000\000\000\000" -/* offset=126978 */ "hisi_sccl,ddrc\000" -/* offset=126993 */ "uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000\000\000\000\000" -/* offset=127063 */ "uncore_cbox\000" -/* offset=127075 */ "unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000\000\000\000\000" -/* offset=127229 */ "event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000\000\000\000\000" -/* offset=127283 */ "event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000\000\000\000\000" -/* offset=127341 */ "hisi_sccl,l3c\000" -/* offset=127355 */ "uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000\000\000\000\000" -/* offset=127423 */ "uncore_imc_free_running\000" -/* offset=127447 */ "uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000\000\000\000\000" -/* offset=127527 */ "uncore_imc\000" -/* offset=127538 */ "uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000\000\000\000\000" -/* offset=127603 */ "uncore_sys_ddr_pmu\000" -/* offset=127622 */ "sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000\000\000\000" -/* offset=127698 */ "uncore_sys_ccn_pmu\000" -/* offset=127717 */ "sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000\000\000\000" -/* offset=127794 */ "uncore_sys_cmn_pmu\000" -/* offset=127813 */ "sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000\000\000\000" -/* offset=127956 */ "CPUs_utilized\000Default\000(software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@) / (duration_time * 1e9)\000\000Average CPU utilization\000\0001CPUs\000\000\000\000011" -/* offset=128142 */ "cs_per_second\000Default\000software@context\\-switches\\,name\\=context\\-switches@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Context switches per CPU second\000\0001cs/sec\000\000\000\000011" -/* offset=128375 */ "migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-migrations@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Process migrations to a new CPU per CPU second\000\0001migrations/sec\000\000\000\000011" -/* offset=128635 */ "page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faults@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Page faults per CPU second\000\0001faults/sec\000\000\000\000011" -/* offset=128866 */ "insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\000Instructions Per Cycle\000\0001instructions\000\000\000\000001" -/* offset=128979 */ "stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions if has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend) else (stalled\\-cycles\\-frontend / instructions if has_event(stalled\\-cycles\\-frontend) else (stalled\\-cycles\\-backend / instructions if has_event(stalled\\-cycles\\-backend) else 0)))\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001" -/* offset=129404 */ "frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles if has_event(stalled\\-cycles\\-frontend) else 0)\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001" -/* offset=129583 */ "backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if has_event(stalled\\-cycles\\-backend) else 0)\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001" -/* offset=129757 */ "cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011" -/* offset=129933 */ "branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011" -/* offset=130113 */ "branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001" -/* offset=130217 */ "l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001" -/* offset=130333 */ "llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001" -/* offset=130434 */ "l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001" -/* offset=130549 */ "dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001" -/* offset=130655 */ "itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001" -/* offset=130761 */ "l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001" -/* offset=130909 */ "CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000" -/* offset=130932 */ "IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000" -/* offset=130996 */ "Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000" -/* offset=131163 */ "dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" -/* offset=131228 */ "icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" -/* offset=131296 */ "cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000" -/* offset=131368 */ "DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000" -/* offset=131463 */ "DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000" -/* offset=131598 */ "DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000" -/* offset=131663 */ "DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000" -/* offset=131732 */ "DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000" -/* offset=131803 */ "M1\000\000ipc + M2\000\000\000\000\000\000\000\000000" -/* offset=131826 */ "M2\000\000ipc + M1\000\000\000\000\000\000\000\000000" -/* offset=131849 */ "M3\000\0001 / M3\000\000\000\000\000\000\000\000000" -/* offset=131870 */ "L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000" +/* offset=0 */ +"default_core\000" +/* offset=13 */ +"l1-dcache\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=99 */ +"l1-dcache-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=190 */ +"l1-dcache-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=286 */ +"l1-dcache-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=387 */ +"l1-dcache-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=482 */ +"l1-dcache-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=580 */ +"l1-dcache-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00000\000\000\000\000\000" +/* offset=682 */ +"l1-dcache-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=782 */ +"l1-dcache-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00000\000\000\000\000\000" +/* offset=874 */ +"l1-dcache-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=971 */ +"l1-dcache-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1073 */ +"l1-dcache-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1169 */ +"l1-dcache-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1268 */ +"l1-dcache-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=1371 */ +"l1-dcache-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=1472 */ +"l1-dcache-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1563 */ +"l1-dcache-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1659 */ +"l1-dcache-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1760 */ +"l1-dcache-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1855 */ +"l1-dcache-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=1953 */ +"l1-dcache-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=2055 */ +"l1-dcache-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=2155 */ +"l1-dcache-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=2252 */ +"l1-dcache-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=2354 */ +"l1-dcache-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=2461 */ +"l1-dcache-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=2562 */ +"l1-dcache-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=2666 */ +"l1-dcache-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00000\000\000\000\000\000" +/* offset=2770 */ +"l1-dcache-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=2872 */ +"l1-dcache-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00000\000\000\000\000\000" +/* offset=2970 */ +"l1-dcache-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3073 */ +"l1-dcache-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3181 */ +"l1-dcache-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3283 */ +"l1-dcache-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3388 */ +"l1-dcache-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=3493 */ +"l1-dcache-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=3596 */ +"l1-dcache-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3693 */ +"l1-dcache-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3795 */ +"l1-dcache-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=3902 */ +"l1-dcache-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=4003 */ +"l1-dcache-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=4107 */ +"l1-dcache-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=4211 */ +"l1-dcache-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=4313 */ +"l1-dcache-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=4416 */ +"l1-dcache-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=4524 */ +"l1-dcache-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=4637 */ +"l1-dcache-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=4744 */ +"l1-dcache-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=4854 */ +"l1-dcache-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00000\000\000\000\000\000" +/* offset=4964 */ +"l1-dcache-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=5072 */ +"l1-dcache-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00000\000\000\000\000\000" +/* offset=5177 */ +"l1-dcache-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=5287 */ +"l1-dcache-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=5402 */ +"l1-dcache-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=5511 */ +"l1-dcache-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=5623 */ +"l1-dcache-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=5735 */ +"l1-dcache-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=5845 */ +"l1-dcache-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=5956 */ +"l1-dcache-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6072 */ +"l1-dcache-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6193 */ +"l1-dcache-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6308 */ +"l1-dcache-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6426 */ +"l1-dcache-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=6544 */ +"l1-dcache-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=6660 */ +"l1-dcache-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6771 */ +"l1-dcache-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=6887 */ +"l1-dcache-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=7008 */ +"l1-dcache-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=7123 */ +"l1-dcache-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=7241 */ +"l1-dcache-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=7359 */ +"l1-dcache-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=7475 */ +"l1-dcache-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=7566 */ +"l1-dcache-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=7662 */ +"l1-dcache-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=7752 */ +"l1-dcache-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=7845 */ +"l1-dcache-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=7942 */ +"l1-dcache-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=8037 */ +"l1-d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8118 */ +"l1-d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8204 */ +"l1-d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8295 */ +"l1-d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8391 */ +"l1-d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8481 */ +"l1-d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8574 */ +"l1-d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=8671 */ +"l1-d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=8766 */ +"l1-d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8853 */ +"l1-d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=8945 */ +"l1-d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9042 */ +"l1-d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9133 */ +"l1-d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9227 */ +"l1-d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=9325 */ +"l1-d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=9421 */ +"l1-d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9507 */ +"l1-d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9598 */ +"l1-d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9694 */ +"l1-d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9784 */ +"l1-d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=9877 */ +"l1-d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=9974 */ +"l1-d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=10069 */ +"l1-d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10161 */ +"l1-d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10258 */ +"l1-d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10360 */ +"l1-d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10456 */ +"l1-d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10555 */ +"l1-d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=10654 */ +"l1-d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=10751 */ +"l1-d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10844 */ +"l1-d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=10942 */ +"l1-d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11045 */ +"l1-d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11142 */ +"l1-d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11242 */ +"l1-d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=11342 */ +"l1-d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=11440 */ +"l1-d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11532 */ +"l1-d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11629 */ +"l1-d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11731 */ +"l1-d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11827 */ +"l1-d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=11926 */ +"l1-d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=12025 */ +"l1-d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=12122 */ +"l1-d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12220 */ +"l1-d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12323 */ +"l1-d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12431 */ +"l1-d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12533 */ +"l1-d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12638 */ +"l1-d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=12743 */ +"l1-d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=12846 */ +"l1-d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=12946 */ +"l1-d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13051 */ +"l1-d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13161 */ +"l1-d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13265 */ +"l1-d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13372 */ +"l1-d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=13479 */ +"l1-d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=13584 */ +"l1-d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13690 */ +"l1-d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13801 */ +"l1-d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=13917 */ +"l1-d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14027 */ +"l1-d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14140 */ +"l1-d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=14253 */ +"l1-d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=14364 */ +"l1-d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14470 */ +"l1-d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14581 */ +"l1-d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14697 */ +"l1-d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14807 */ +"l1-d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=14920 */ +"l1-d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=15033 */ +"l1-d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=15144 */ +"l1-d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15230 */ +"l1-d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15321 */ +"l1-d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15406 */ +"l1-d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15494 */ +"l1-d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=15586 */ +"l1-d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=15676 */ +"l1d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15756 */ +"l1d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15841 */ +"l1d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=15931 */ +"l1d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16026 */ +"l1d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16115 */ +"l1d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16207 */ +"l1d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=16303 */ +"l1d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=16397 */ +"l1d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16483 */ +"l1d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16574 */ +"l1d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16670 */ +"l1d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16760 */ +"l1d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=16853 */ +"l1d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=16950 */ +"l1d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=17045 */ +"l1d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=17130 */ +"l1d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=17220 */ +"l1d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=17315 */ +"l1d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=17404 */ +"l1d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=17496 */ +"l1d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=17592 */ +"l1d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=17686 */ +"l1d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=17777 */ +"l1d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=17873 */ +"l1d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=17974 */ +"l1d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18069 */ +"l1d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18167 */ +"l1d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=18265 */ +"l1d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=18361 */ +"l1d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18453 */ +"l1d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18550 */ +"l1d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18652 */ +"l1d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18748 */ +"l1d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=18847 */ +"l1d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=18946 */ +"l1d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=19043 */ +"l1d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=19134 */ +"l1d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=19230 */ +"l1d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=19331 */ +"l1d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=19426 */ +"l1d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=19524 */ +"l1d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=19622 */ +"l1d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=19718 */ +"l1d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=19815 */ +"l1d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=19917 */ +"l1d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20024 */ +"l1d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20125 */ +"l1d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20229 */ +"l1d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=20333 */ +"l1d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=20435 */ +"l1d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20534 */ +"l1d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20638 */ +"l1d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20747 */ +"l1d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20850 */ +"l1d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=20956 */ +"l1d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=21062 */ +"l1d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=21166 */ +"l1d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=21271 */ +"l1d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=21381 */ +"l1d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=21496 */ +"l1d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=21605 */ +"l1d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=21717 */ +"l1d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=21829 */ +"l1d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=21939 */ +"l1d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=22044 */ +"l1d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=22154 */ +"l1d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=22269 */ +"l1d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=22378 */ +"l1d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=22490 */ +"l1d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=22602 */ +"l1d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=22712 */ +"l1d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=22797 */ +"l1d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=22887 */ +"l1d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=22971 */ +"l1d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23058 */ +"l1d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=23149 */ +"l1d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=23238 */ +"l1-data\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23322 */ +"l1-data-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23411 */ +"l1-data-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23505 */ +"l1-data-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23604 */ +"l1-data-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23697 */ +"l1-data-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=23793 */ +"l1-data-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=23893 */ +"l1-data-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=23991 */ +"l1-data-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24081 */ +"l1-data-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24176 */ +"l1-data-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24276 */ +"l1-data-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24370 */ +"l1-data-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24467 */ +"l1-data-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=24568 */ +"l1-data-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=24667 */ +"l1-data-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24756 */ +"l1-data-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24850 */ +"l1-data-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=24949 */ +"l1-data-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=25042 */ +"l1-data-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=25138 */ +"l1-data-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=25238 */ +"l1-data-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=25336 */ +"l1-data-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=25431 */ +"l1-data-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=25531 */ +"l1-data-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=25636 */ +"l1-data-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=25735 */ +"l1-data-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=25837 */ +"l1-data-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=25939 */ +"l1-data-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=26039 */ +"l1-data-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26135 */ +"l1-data-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26236 */ +"l1-data-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26342 */ +"l1-data-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26442 */ +"l1-data-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26545 */ +"l1-data-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=26648 */ +"l1-data-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=26749 */ +"l1-data-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26844 */ +"l1-data-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=26944 */ +"l1-data-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=27049 */ +"l1-data-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=27148 */ +"l1-data-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000" +/* offset=27250 */ +"l1-data-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=27352 */ +"l1-data-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000" +/* offset=27452 */ +"l1-data-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=27553 */ +"l1-data-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=27659 */ +"l1-data-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=27770 */ +"l1-data-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=27875 */ +"l1-data-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=27983 */ +"l1-data-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=28091 */ +"l1-data-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=28197 */ +"l1-data-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=28300 */ +"l1-data-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=28408 */ +"l1-data-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=28521 */ +"l1-data-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=28628 */ +"l1-data-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=28738 */ +"l1-data-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=28848 */ +"l1-data-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=28956 */ +"l1-data-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29065 */ +"l1-data-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29179 */ +"l1-data-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29298 */ +"l1-data-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29411 */ +"l1-data-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29527 */ +"l1-data-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=29643 */ +"l1-data-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=29757 */ +"l1-data-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29866 */ +"l1-data-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=29980 */ +"l1-data-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=30099 */ +"l1-data-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=30212 */ +"l1-data-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000" +/* offset=30328 */ +"l1-data-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=30444 */ +"l1-data-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000" +/* offset=30558 */ +"l1-data-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=30647 */ +"l1-data-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=30741 */ +"l1-data-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=30829 */ +"l1-data-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000" +/* offset=30920 */ +"l1-data-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=31015 */ +"l1-data-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000" +/* offset=31108 */ +"l1-icache\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31201 */ +"l1-icache-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31299 */ +"l1-icache-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31402 */ +"l1-icache-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31510 */ +"l1-icache-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31612 */ +"l1-icache-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=31717 */ +"l1-icache-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00000\000\000\000\000\000" +/* offset=31826 */ +"l1-icache-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=31933 */ +"l1-icache-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00000\000\000\000\000\000" +/* offset=32032 */ +"l1-icache-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32136 */ +"l1-icache-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32245 */ +"l1-icache-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32348 */ +"l1-icache-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32454 */ +"l1-icache-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=32564 */ +"l1-icache-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=32672 */ +"l1-icache-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32770 */ +"l1-icache-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32873 */ +"l1-icache-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=32981 */ +"l1-icache-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=33083 */ +"l1-icache-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=33188 */ +"l1-icache-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=33297 */ +"l1-icache-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=33404 */ +"l1-icache-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=33514 */ +"l1-icache-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=33629 */ +"l1-icache-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=33749 */ +"l1-icache-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=33863 */ +"l1-icache-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=33980 */ +"l1-icache-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00000\000\000\000\000\000" +/* offset=34097 */ +"l1-icache-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=34212 */ +"l1-icache-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00000\000\000\000\000\000" +/* offset=34324 */ +"l1-icache-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=34441 */ +"l1-icache-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=34563 */ +"l1-icache-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=34679 */ +"l1-icache-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=34798 */ +"l1-icache-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=34917 */ +"l1-icache-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=35034 */ +"l1-icache-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=35152 */ +"l1-icache-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=35275 */ +"l1-icache-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=35403 */ +"l1-icache-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=35525 */ +"l1-icache-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=35650 */ +"l1-icache-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=35775 */ +"l1-icache-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=35898 */ +"l1-icache-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=36016 */ +"l1-icache-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=36139 */ +"l1-icache-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=36267 */ +"l1-icache-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=36389 */ +"l1-icache-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=36514 */ +"l1-icache-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=36639 */ +"l1-icache-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=36762 */ +"l1-icache-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=36860 */ +"l1-icache-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=36963 */ +"l1-icache-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37060 */ +"l1-icache-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37160 */ +"l1-icache-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=37264 */ +"l1-icache-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=37366 */ +"l1-i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37454 */ +"l1-i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37547 */ +"l1-i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37645 */ +"l1-i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37748 */ +"l1-i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37845 */ +"l1-i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=37945 */ +"l1-i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=38049 */ +"l1-i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=38151 */ +"l1-i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38245 */ +"l1-i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38344 */ +"l1-i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38448 */ +"l1-i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38546 */ +"l1-i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38647 */ +"l1-i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=38752 */ +"l1-i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=38855 */ +"l1-i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=38948 */ +"l1-i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=39046 */ +"l1-i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=39149 */ +"l1-i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=39246 */ +"l1-i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=39346 */ +"l1-i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=39450 */ +"l1-i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=39552 */ +"l1-i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=39657 */ +"l1-i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=39767 */ +"l1-i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=39882 */ +"l1-i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=39991 */ +"l1-i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40103 */ +"l1-i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=40215 */ +"l1-i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=40325 */ +"l1-i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40432 */ +"l1-i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40544 */ +"l1-i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40661 */ +"l1-i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40772 */ +"l1-i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=40886 */ +"l1-i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=41000 */ +"l1-i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=41112 */ +"l1-i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=41225 */ +"l1-i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=41343 */ +"l1-i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=41466 */ +"l1-i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=41583 */ +"l1-i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=41703 */ +"l1-i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=41823 */ +"l1-i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=41941 */ +"l1-i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=42054 */ +"l1-i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=42172 */ +"l1-i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=42295 */ +"l1-i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=42412 */ +"l1-i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=42532 */ +"l1-i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=42652 */ +"l1-i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=42770 */ +"l1-i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=42863 */ +"l1-i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=42961 */ +"l1-i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43053 */ +"l1-i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43148 */ +"l1-i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=43247 */ +"l1-i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=43344 */ +"l1i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43431 */ +"l1i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43523 */ +"l1i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43620 */ +"l1i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43722 */ +"l1i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43818 */ +"l1i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=43917 */ +"l1i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=44020 */ +"l1i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=44121 */ +"l1i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44214 */ +"l1i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44312 */ +"l1i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44415 */ +"l1i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44512 */ +"l1i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44612 */ +"l1i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=44716 */ +"l1i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=44818 */ +"l1i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=44910 */ +"l1i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=45007 */ +"l1i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=45109 */ +"l1i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=45205 */ +"l1i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=45304 */ +"l1i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=45407 */ +"l1i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=45508 */ +"l1i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=45612 */ +"l1i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=45721 */ +"l1i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=45835 */ +"l1i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=45943 */ +"l1i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46054 */ +"l1i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=46165 */ +"l1i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=46274 */ +"l1i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46380 */ +"l1i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46491 */ +"l1i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46607 */ +"l1i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46717 */ +"l1i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=46830 */ +"l1i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=46943 */ +"l1i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=47054 */ +"l1i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47166 */ +"l1i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47283 */ +"l1i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47405 */ +"l1i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47521 */ +"l1i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47640 */ +"l1i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=47759 */ +"l1i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=47876 */ +"l1i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=47988 */ +"l1i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=48105 */ +"l1i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=48227 */ +"l1i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=48343 */ +"l1i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=48462 */ +"l1i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=48581 */ +"l1i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=48698 */ +"l1i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=48790 */ +"l1i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=48887 */ +"l1i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=48978 */ +"l1i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49072 */ +"l1i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=49170 */ +"l1i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=49266 */ +"l1-instruction\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49364 */ +"l1-instruction-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49467 */ +"l1-instruction-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49575 */ +"l1-instruction-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49688 */ +"l1-instruction-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49795 */ +"l1-instruction-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=49905 */ +"l1-instruction-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=50019 */ +"l1-instruction-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=50131 */ +"l1-instruction-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=50235 */ +"l1-instruction-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=50344 */ +"l1-instruction-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=50458 */ +"l1-instruction-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=50566 */ +"l1-instruction-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=50677 */ +"l1-instruction-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=50792 */ +"l1-instruction-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=50905 */ +"l1-instruction-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=51008 */ +"l1-instruction-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=51116 */ +"l1-instruction-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=51229 */ +"l1-instruction-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=51336 */ +"l1-instruction-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=51446 */ +"l1-instruction-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=51560 */ +"l1-instruction-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=51672 */ +"l1-instruction-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=51787 */ +"l1-instruction-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=51907 */ +"l1-instruction-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52032 */ +"l1-instruction-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52151 */ +"l1-instruction-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52273 */ +"l1-instruction-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=52395 */ +"l1-instruction-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=52515 */ +"l1-instruction-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52632 */ +"l1-instruction-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52754 */ +"l1-instruction-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=52881 */ +"l1-instruction-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53002 */ +"l1-instruction-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53126 */ +"l1-instruction-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=53250 */ +"l1-instruction-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=53372 */ +"l1-instruction-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53495 */ +"l1-instruction-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53623 */ +"l1-instruction-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53756 */ +"l1-instruction-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=53883 */ +"l1-instruction-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54013 */ +"l1-instruction-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=54143 */ +"l1-instruction-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=54271 */ +"l1-instruction-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54394 */ +"l1-instruction-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54522 */ +"l1-instruction-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54655 */ +"l1-instruction-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54782 */ +"l1-instruction-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000" +/* offset=54912 */ +"l1-instruction-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=55042 */ +"l1-instruction-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000" +/* offset=55170 */ +"l1-instruction-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=55273 */ +"l1-instruction-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=55381 */ +"l1-instruction-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=55483 */ +"l1-instruction-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000" +/* offset=55588 */ +"l1-instruction-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=55697 */ +"l1-instruction-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000" +/* offset=55804 */ +"llc\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=55882 */ +"llc-load\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=55965 */ +"llc-load-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56053 */ +"llc-load-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56146 */ +"llc-load-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56233 */ +"llc-load-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56323 */ +"llc-load-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00000\000\000\000\000\000" +/* offset=56417 */ +"llc-load-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=56509 */ +"llc-loads\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00000\000\000\000\000\000" +/* offset=56593 */ +"llc-loads-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56682 */ +"llc-loads-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56776 */ +"llc-loads-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56864 */ +"llc-loads-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=56955 */ +"llc-loads-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=57050 */ +"llc-loads-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=57143 */ +"llc-read\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=57226 */ +"llc-read-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=57314 */ +"llc-read-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=57407 */ +"llc-read-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=57494 */ +"llc-read-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=57584 */ +"llc-read-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=57678 */ +"llc-read-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=57770 */ +"llc-store\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=57859 */ +"llc-store-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=57953 */ +"llc-store-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58052 */ +"llc-store-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58145 */ +"llc-store-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58241 */ +"llc-store-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00000\000\000\000\000\000" +/* offset=58337 */ +"llc-store-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=58431 */ +"llc-stores\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00000\000\000\000\000\000" +/* offset=58521 */ +"llc-stores-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58616 */ +"llc-stores-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58716 */ +"llc-stores-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58810 */ +"llc-stores-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=58907 */ +"llc-stores-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=59004 */ +"llc-stores-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=59099 */ +"llc-write\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=59188 */ +"llc-write-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=59282 */ +"llc-write-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=59381 */ +"llc-write-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=59474 */ +"llc-write-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=59570 */ +"llc-write-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=59666 */ +"llc-write-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=59760 */ +"llc-prefetch\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=59855 */ +"llc-prefetch-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=59955 */ +"llc-prefetch-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60060 */ +"llc-prefetch-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60159 */ +"llc-prefetch-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60261 */ +"llc-prefetch-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00000\000\000\000\000\000" +/* offset=60363 */ +"llc-prefetch-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=60463 */ +"llc-prefetches\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00000\000\000\000\000\000" +/* offset=60560 */ +"llc-prefetches-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60662 */ +"llc-prefetches-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60769 */ +"llc-prefetches-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60870 */ +"llc-prefetches-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=60974 */ +"llc-prefetches-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=61078 */ +"llc-prefetches-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=61180 */ +"llc-speculative-read\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=61283 */ +"llc-speculative-read-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=61391 */ +"llc-speculative-read-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=61504 */ +"llc-speculative-read-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=61611 */ +"llc-speculative-read-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=61721 */ +"llc-speculative-read-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=61831 */ +"llc-speculative-read-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=61939 */ +"llc-speculative-load\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=62042 */ +"llc-speculative-load-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=62150 */ +"llc-speculative-load-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=62263 */ +"llc-speculative-load-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=62370 */ +"llc-speculative-load-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=62480 */ +"llc-speculative-load-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=62590 */ +"llc-speculative-load-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=62698 */ +"llc-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=62781 */ +"llc-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=62869 */ +"llc-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=62951 */ +"llc-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63036 */ +"llc-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=63125 */ +"llc-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=63212 */ +"l2\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63309 */ +"l2-load\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63411 */ +"l2-load-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63518 */ +"l2-load-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63630 */ +"l2-load-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63736 */ +"l2-load-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=63845 */ +"l2-load-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=63958 */ +"l2-load-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=64069 */ +"l2-loads\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64172 */ +"l2-loads-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64280 */ +"l2-loads-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64393 */ +"l2-loads-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64500 */ +"l2-loads-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64610 */ +"l2-loads-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=64724 */ +"l2-loads-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=64836 */ +"l2-read\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=64938 */ +"l2-read-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=65045 */ +"l2-read-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=65157 */ +"l2-read-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=65263 */ +"l2-read-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=65372 */ +"l2-read-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=65485 */ +"l2-read-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=65596 */ +"l2-store\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=65704 */ +"l2-store-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=65817 */ +"l2-store-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=65935 */ +"l2-store-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66047 */ +"l2-store-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66162 */ +"l2-store-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=66277 */ +"l2-store-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=66390 */ +"l2-stores\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66499 */ +"l2-stores-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66613 */ +"l2-stores-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66732 */ +"l2-stores-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66845 */ +"l2-stores-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=66961 */ +"l2-stores-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=67077 */ +"l2-stores-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=67191 */ +"l2-write\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=67299 */ +"l2-write-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=67412 */ +"l2-write-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=67530 */ +"l2-write-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=67642 */ +"l2-write-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000" +/* offset=67757 */ +"l2-write-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=67872 */ +"l2-write-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000" +/* offset=67985 */ +"l2-prefetch\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68099 */ +"l2-prefetch-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68218 */ +"l2-prefetch-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68342 */ +"l2-prefetch-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68460 */ +"l2-prefetch-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68581 */ +"l2-prefetch-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=68702 */ +"l2-prefetch-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=68821 */ +"l2-prefetches\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=68937 */ +"l2-prefetches-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69058 */ +"l2-prefetches-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69184 */ +"l2-prefetches-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69304 */ +"l2-prefetches-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69427 */ +"l2-prefetches-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=69550 */ +"l2-prefetches-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=69671 */ +"l2-speculative-read\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69793 */ +"l2-speculative-read-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=69920 */ +"l2-speculative-read-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70052 */ +"l2-speculative-read-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70178 */ +"l2-speculative-read-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70307 */ +"l2-speculative-read-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=70436 */ +"l2-speculative-read-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=70563 */ +"l2-speculative-load\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70685 */ +"l2-speculative-load-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70812 */ +"l2-speculative-load-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=70944 */ +"l2-speculative-load-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=71070 */ +"l2-speculative-load-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000" +/* offset=71199 */ +"l2-speculative-load-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=71328 */ +"l2-speculative-load-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000" +/* offset=71455 */ +"l2-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=71557 */ +"l2-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=71664 */ +"l2-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=71765 */ +"l2-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000" +/* offset=71869 */ +"l2-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=71977 */ +"l2-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000" +/* offset=72083 */ +"dtlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72154 */ +"dtlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72230 */ +"dtlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72311 */ +"dtlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72397 */ +"dtlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72477 */ +"dtlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72560 */ +"dtlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00000\000\000\000\000\000" +/* offset=72647 */ +"dtlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=72732 */ +"dtlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00000\000\000\000\000\000" +/* offset=72809 */ +"dtlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72891 */ +"dtlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=72978 */ +"dtlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73059 */ +"dtlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73143 */ +"dtlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=73231 */ +"dtlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=73317 */ +"dtlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73393 */ +"dtlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73474 */ +"dtlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73560 */ +"dtlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73640 */ +"dtlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=73723 */ +"dtlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=73810 */ +"dtlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=73895 */ +"dtlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=73977 */ +"dtlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74064 */ +"dtlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74156 */ +"dtlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74242 */ +"dtlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74331 */ +"dtlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00000\000\000\000\000\000" +/* offset=74420 */ +"dtlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=74507 */ +"dtlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00000\000\000\000\000\000" +/* offset=74590 */ +"dtlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74678 */ +"dtlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74771 */ +"dtlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74858 */ +"dtlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=74948 */ +"dtlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=75038 */ +"dtlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=75126 */ +"dtlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=75208 */ +"dtlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=75295 */ +"dtlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=75387 */ +"dtlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=75473 */ +"dtlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=75562 */ +"dtlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=75651 */ +"dtlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=75738 */ +"dtlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=75826 */ +"dtlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=75919 */ +"dtlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76017 */ +"dtlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76109 */ +"dtlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76204 */ +"dtlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00000\000\000\000\000\000" +/* offset=76299 */ +"dtlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=76392 */ +"dtlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00000\000\000\000\000\000" +/* offset=76482 */ +"dtlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76577 */ +"dtlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76677 */ +"dtlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76771 */ +"dtlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=76868 */ +"dtlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=76965 */ +"dtlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=77060 */ +"dtlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77156 */ +"dtlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77257 */ +"dtlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77363 */ +"dtlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77463 */ +"dtlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77566 */ +"dtlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=77669 */ +"dtlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=77770 */ +"dtlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77866 */ +"dtlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=77967 */ +"dtlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=78073 */ +"dtlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=78173 */ +"dtlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=78276 */ +"dtlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=78379 */ +"dtlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=78480 */ +"dtlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=78556 */ +"dtlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=78637 */ +"dtlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=78712 */ +"dtlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=78790 */ +"dtlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=78872 */ +"dtlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=78952 */ +"d-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79024 */ +"d-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79101 */ +"d-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79183 */ +"d-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79270 */ +"d-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79351 */ +"d-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79435 */ +"d-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=79523 */ +"d-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=79609 */ +"d-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79687 */ +"d-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79770 */ +"d-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79858 */ +"d-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=79940 */ +"d-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80025 */ +"d-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=80114 */ +"d-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=80201 */ +"d-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80278 */ +"d-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80360 */ +"d-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80447 */ +"d-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80528 */ +"d-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=80612 */ +"d-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=80700 */ +"d-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=80786 */ +"d-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=80869 */ +"d-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=80957 */ +"d-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81050 */ +"d-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81137 */ +"d-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81227 */ +"d-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=81317 */ +"d-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=81405 */ +"d-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81489 */ +"d-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81578 */ +"d-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81672 */ +"d-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81760 */ +"d-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=81851 */ +"d-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=81942 */ +"d-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=82031 */ +"d-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=82114 */ +"d-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=82202 */ +"d-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=82295 */ +"d-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=82382 */ +"d-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=82472 */ +"d-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=82562 */ +"d-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=82650 */ +"d-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=82739 */ +"d-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=82833 */ +"d-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=82932 */ +"d-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83025 */ +"d-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83121 */ +"d-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=83217 */ +"d-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=83311 */ +"d-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83402 */ +"d-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83498 */ +"d-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83599 */ +"d-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83694 */ +"d-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=83792 */ +"d-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=83890 */ +"d-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=83986 */ +"d-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84083 */ +"d-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84185 */ +"d-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84292 */ +"d-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84393 */ +"d-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84497 */ +"d-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=84601 */ +"d-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=84703 */ +"d-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84800 */ +"d-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=84902 */ +"d-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=85009 */ +"d-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=85110 */ +"d-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=85214 */ +"d-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=85318 */ +"d-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=85420 */ +"d-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=85497 */ +"d-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=85579 */ +"d-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=85655 */ +"d-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=85734 */ +"d-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=85817 */ +"d-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=85898 */ +"data-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=85973 */ +"data-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86053 */ +"data-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86138 */ +"data-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86228 */ +"data-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86312 */ +"data-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86399 */ +"data-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=86490 */ +"data-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=86579 */ +"data-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86660 */ +"data-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86746 */ +"data-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86837 */ +"data-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=86922 */ +"data-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87010 */ +"data-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=87102 */ +"data-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=87192 */ +"data-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87272 */ +"data-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87357 */ +"data-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87447 */ +"data-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87531 */ +"data-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=87618 */ +"data-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=87709 */ +"data-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=87798 */ +"data-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=87884 */ +"data-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=87975 */ +"data-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88071 */ +"data-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88161 */ +"data-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88254 */ +"data-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=88347 */ +"data-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=88438 */ +"data-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88525 */ +"data-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88617 */ +"data-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88714 */ +"data-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88805 */ +"data-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=88899 */ +"data-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=88993 */ +"data-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=89085 */ +"data-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=89171 */ +"data-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=89262 */ +"data-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=89358 */ +"data-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=89448 */ +"data-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000" +/* offset=89541 */ +"data-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=89634 */ +"data-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000" +/* offset=89725 */ +"data-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=89817 */ +"data-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=89914 */ +"data-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90016 */ +"data-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90112 */ +"data-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90211 */ +"data-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=90310 */ +"data-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=90407 */ +"data-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90501 */ +"data-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90600 */ +"data-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90704 */ +"data-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90802 */ +"data-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=90903 */ +"data-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=91004 */ +"data-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=91103 */ +"data-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91203 */ +"data-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91308 */ +"data-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91418 */ +"data-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91522 */ +"data-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91629 */ +"data-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=91736 */ +"data-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=91841 */ +"data-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=91941 */ +"data-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=92046 */ +"data-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=92156 */ +"data-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=92260 */ +"data-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000" +/* offset=92367 */ +"data-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=92474 */ +"data-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000" +/* offset=92579 */ +"data-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=92659 */ +"data-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=92744 */ +"data-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=92823 */ +"data-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000" +/* offset=92905 */ +"data-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=92991 */ +"data-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000" +/* offset=93075 */ +"itlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93153 */ +"itlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93236 */ +"itlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93324 */ +"itlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93417 */ +"itlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93504 */ +"itlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93594 */ +"itlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00000\000\000\000\000\000" +/* offset=93688 */ +"itlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=93780 */ +"itlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00000\000\000\000\000\000" +/* offset=93864 */ +"itlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=93953 */ +"itlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94047 */ +"itlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94135 */ +"itlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94226 */ +"itlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=94321 */ +"itlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=94414 */ +"itlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94497 */ +"itlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94585 */ +"itlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94678 */ +"itlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94765 */ +"itlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=94855 */ +"itlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=94949 */ +"itlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=95041 */ +"itlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95124 */ +"itlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95212 */ +"itlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95294 */ +"itlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95379 */ +"itlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=95468 */ +"itlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=95555 */ +"i-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95634 */ +"i-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95718 */ +"i-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95807 */ +"i-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95901 */ +"i-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=95989 */ +"i-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96080 */ +"i-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=96175 */ +"i-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=96268 */ +"i-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96353 */ +"i-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96443 */ +"i-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96538 */ +"i-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96627 */ +"i-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96719 */ +"i-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=96815 */ +"i-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=96909 */ +"i-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=96993 */ +"i-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97082 */ +"i-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97176 */ +"i-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97264 */ +"i-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97355 */ +"i-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=97450 */ +"i-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=97543 */ +"i-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97627 */ +"i-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97716 */ +"i-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97799 */ +"i-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=97885 */ +"i-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=97975 */ +"i-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=98063 */ +"instruction-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98152 */ +"instruction-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98246 */ +"instruction-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98345 */ +"instruction-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98449 */ +"instruction-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98547 */ +"instruction-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98648 */ +"instruction-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=98753 */ +"instruction-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=98856 */ +"instruction-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=98951 */ +"instruction-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99051 */ +"instruction-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99156 */ +"instruction-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99255 */ +"instruction-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99357 */ +"instruction-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=99463 */ +"instruction-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=99567 */ +"instruction-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99661 */ +"instruction-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99760 */ +"instruction-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99864 */ +"instruction-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=99962 */ +"instruction-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=100063 */ +"instruction-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=100168 */ +"instruction-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=100271 */ +"instruction-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=100365 */ +"instruction-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=100464 */ +"instruction-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=100557 */ +"instruction-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000" +/* offset=100653 */ +"instruction-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=100753 */ +"instruction-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000" +/* offset=100851 */ +"branch\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=100938 */ +"branch-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101030 */ +"branch-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101127 */ +"branch-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101229 */ +"branch-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101325 */ +"branch-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101424 */ +"branch-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00000\000\000\000\000\000" +/* offset=101527 */ +"branch-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=101628 */ +"branch-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00000\000\000\000\000\000" +/* offset=101721 */ +"branch-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101819 */ +"branch-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=101922 */ +"branch-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102019 */ +"branch-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102119 */ +"branch-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=102223 */ +"branch-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=102325 */ +"branch-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102417 */ +"branch-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102514 */ +"branch-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102616 */ +"branch-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102712 */ +"branch-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=102811 */ +"branch-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=102914 */ +"branch-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=103015 */ +"branch-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103107 */ +"branch-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103204 */ +"branch-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103295 */ +"branch-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103389 */ +"branch-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=103485 */ +"branches-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103579 */ +"branches-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103678 */ +"branches-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103782 */ +"branches-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103880 */ +"branches-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=103981 */ +"branches-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=104086 */ +"branches-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=104189 */ +"branches-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104284 */ +"branches-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104384 */ +"branches-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104489 */ +"branches-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104588 */ +"branches-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104690 */ +"branches-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=104796 */ +"branches-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=104900 */ +"branches-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=104994 */ +"branches-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105093 */ +"branches-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105197 */ +"branches-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105295 */ +"branches-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105396 */ +"branches-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=105501 */ +"branches-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=105604 */ +"branches-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105698 */ +"branches-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105797 */ +"branches-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105890 */ +"branches-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=105986 */ +"branches-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=106086 */ +"branches-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=106184 */ +"bpu\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106268 */ +"bpu-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106357 */ +"bpu-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106451 */ +"bpu-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106550 */ +"bpu-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106643 */ +"bpu-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=106739 */ +"bpu-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=106839 */ +"bpu-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=106937 */ +"bpu-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107027 */ +"bpu-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107122 */ +"bpu-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107222 */ +"bpu-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107316 */ +"bpu-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107413 */ +"bpu-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=107514 */ +"bpu-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=107613 */ +"bpu-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107702 */ +"bpu-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107796 */ +"bpu-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107895 */ +"bpu-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=107988 */ +"bpu-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108084 */ +"bpu-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=108184 */ +"bpu-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=108282 */ +"bpu-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108371 */ +"bpu-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108465 */ +"bpu-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108553 */ +"bpu-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108644 */ +"bpu-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=108739 */ +"bpu-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=108832 */ +"btb\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=108916 */ +"btb-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109005 */ +"btb-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109099 */ +"btb-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109198 */ +"btb-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109291 */ +"btb-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109387 */ +"btb-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=109487 */ +"btb-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=109585 */ +"btb-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109675 */ +"btb-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109770 */ +"btb-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109870 */ +"btb-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=109964 */ +"btb-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110061 */ +"btb-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=110162 */ +"btb-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=110261 */ +"btb-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110350 */ +"btb-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110444 */ +"btb-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110543 */ +"btb-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110636 */ +"btb-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=110732 */ +"btb-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=110832 */ +"btb-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=110930 */ +"btb-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111019 */ +"btb-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111113 */ +"btb-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111201 */ +"btb-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111292 */ +"btb-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=111387 */ +"btb-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=111480 */ +"bpc\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111564 */ +"bpc-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111653 */ +"bpc-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111747 */ +"bpc-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111846 */ +"bpc-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=111939 */ +"bpc-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112035 */ +"bpc-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=112135 */ +"bpc-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=112233 */ +"bpc-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112323 */ +"bpc-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112418 */ +"bpc-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112518 */ +"bpc-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112612 */ +"bpc-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112709 */ +"bpc-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=112810 */ +"bpc-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=112909 */ +"bpc-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=112998 */ +"bpc-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113092 */ +"bpc-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113191 */ +"bpc-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113284 */ +"bpc-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113380 */ +"bpc-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=113480 */ +"bpc-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=113578 */ +"bpc-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113667 */ +"bpc-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113761 */ +"bpc-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113849 */ +"bpc-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000" +/* offset=113940 */ +"bpc-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=114035 */ +"bpc-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000" +/* offset=114128 */ +"node\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114203 */ +"node-load\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114283 */ +"node-load-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114368 */ +"node-load-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114458 */ +"node-load-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114542 */ +"node-load-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114629 */ +"node-load-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00000\000\000\000\000\000" +/* offset=114720 */ +"node-load-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=114809 */ +"node-loads\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00000\000\000\000\000\000" +/* offset=114890 */ +"node-loads-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=114976 */ +"node-loads-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115067 */ +"node-loads-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115152 */ +"node-loads-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115240 */ +"node-loads-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=115332 */ +"node-loads-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=115422 */ +"node-read\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115502 */ +"node-read-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115587 */ +"node-read-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115677 */ +"node-read-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115761 */ +"node-read-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=115848 */ +"node-read-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=115939 */ +"node-read-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=116028 */ +"node-store\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116114 */ +"node-store-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116205 */ +"node-store-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116301 */ +"node-store-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116391 */ +"node-store-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116484 */ +"node-store-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00000\000\000\000\000\000" +/* offset=116577 */ +"node-store-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" +/* offset=116668 */ +"node-stores\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00000\000\000\000\000\000" +/* offset=116755 */ +"node-stores-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116847 */ +"node-stores-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=116944 */ +"node-stores-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117035 */ +"node-stores-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117129 */ +"node-stores-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" +/* offset=117223 */ +"node-stores-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" +/* offset=117315 */ +"node-write\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117401 */ +"node-write-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117492 */ +"node-write-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117588 */ +"node-write-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117678 */ +"node-write-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000" +/* offset=117771 */ +"node-write-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" +/* offset=117864 */ +"node-write-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000" +/* offset=117955 */ +"node-prefetch\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118047 */ +"node-prefetch-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118144 */ +"node-prefetch-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118246 */ +"node-prefetch-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118342 */ +"node-prefetch-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118441 */ +"node-prefetch-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00000\000\000\000\000\000" +/* offset=118540 */ +"node-prefetch-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=118637 */ +"node-prefetches\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00000\000\000\000\000\000" +/* offset=118731 */ +"node-prefetches-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118830 */ +"node-prefetches-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=118934 */ +"node-prefetches-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119032 */ +"node-prefetches-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119133 */ +"node-prefetches-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=119234 */ +"node-prefetches-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=119333 */ +"node-speculative-read\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119433 */ +"node-speculative-read-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119538 */ +"node-speculative-read-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119648 */ +"node-speculative-read-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119752 */ +"node-speculative-read-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=119859 */ +"node-speculative-read-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=119966 */ +"node-speculative-read-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=120071 */ +"node-speculative-load\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=120171 */ +"node-speculative-load-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=120276 */ +"node-speculative-load-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=120386 */ +"node-speculative-load-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=120490 */ +"node-speculative-load-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000" +/* offset=120597 */ +"node-speculative-load-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=120704 */ +"node-speculative-load-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000" +/* offset=120809 */ +"node-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=120889 */ +"node-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=120974 */ +"node-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=121053 */ +"node-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000" +/* offset=121135 */ +"node-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=121221 */ +"node-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000" +/* offset=121305 */ +"cpu-cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000" +/* offset=121467 */ +"cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cpu-cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000" +/* offset=121629 */ +"instructions\000legacy hardware\000Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts\000legacy-hardware-config=1\000\00000\000\000\000\000\000" +/* offset=121805 */ +"cache-references\000legacy hardware\000Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU\000legacy-hardware-config=2\000\00000\000\000\000\000\000" +/* offset=122075 */ +"cache-misses\000legacy hardware\000Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in conjunction with the PERF_COUNT_HW_CACHE_REFERENCES event to calculate cache miss rates\000legacy-hardware-config=3\000\00000\000\000\000\000\000" +/* offset=122318 */ +"branches\000legacy hardware\000Retired branch instructions [This event is an alias of branch-instructions]\000legacy-hardware-config=4\000\00000\000\000\000\000\000" +/* offset=122452 */ +"branch-instructions\000legacy hardware\000Retired branch instructions [This event is an alias of branches]\000legacy-hardware-config=4\000\00000\000\000\000\000\000" +/* offset=122586 */ +"branch-misses\000legacy hardware\000Mispredicted branch instructions\000legacy-hardware-config=5\000\00000\000\000\000\000\000" +/* offset=122682 */ +"bus-cycles\000legacy hardware\000Bus cycles, which can be different from total cycles\000legacy-hardware-config=6\000\00000\000\000\000\000\000" +/* offset=122795 */ +"stalled-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of idle-cycles-frontend]\000legacy-hardware-config=7\000\00000\000\000\000\000\000" +/* offset=122945 */ +"idle-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of stalled-cycles-fronted]\000legacy-hardware-config=7\000\00000\000\000\000\000\000" +/* offset=123094 */ +"stalled-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of idle-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000" +/* offset=123247 */ +"idle-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of stalled-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000" +/* offset=123400 */ +"ref-cycles\000legacy hardware\000Total cycles; not affected by CPU frequency scaling\000legacy-hardware-config=9\000\00000\000\000\000\000\000" +/* offset=123512 */ +"software\000" +/* offset=123521 */ +"cpu-clock\000software\000Per-CPU high-resolution timer based event\000config=0\000\000001e-6msec\000\000\000\000\000" +/* offset=123607 */ +"task-clock\000software\000Per-task high-resolution timer based event\000config=1\000\000001e-6msec\000\000\000\000\000" +/* offset=123695 */ +"faults\000software\000Number of page faults [This event is an alias of page-faults]\000config=2\000\00000\000\000\000\000\000" +/* offset=123790 */ +"page-faults\000software\000Number of page faults [This event is an alias of faults]\000config=2\000\00000\000\000\000\000\000" +/* offset=123885 */ +"context-switches\000software\000Number of context switches [This event is an alias of cs]\000config=3\000\00000\000\000\000\000\000" +/* offset=123986 */ +"cs\000software\000Number of context switches [This event is an alias of context-switches]\000config=3\000\00000\000\000\000\000\000" +/* offset=124087 */ +"cpu-migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of migrations]\000config=4\000\00000\000\000\000\000\000" +/* offset=124219 */ +"migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of cpu-migrations]\000config=4\000\00000\000\000\000\000\000" +/* offset=124351 */ +"minor-faults\000software\000Number of minor page faults. Minor faults don't require I/O to handle\000config=5\000\00000\000\000\000\000\000" +/* offset=124460 */ +"major-faults\000software\000Number of major page faults. Major faults require I/O to handle\000config=6\000\00000\000\000\000\000\000" +/* offset=124563 */ +"alignment-faults\000software\000Number of kernel handled memory alignment faults\000config=7\000\00000\000\000\000\000\000" +/* offset=124655 */ +"emulation-faults\000software\000Number of kernel handled unimplemented instruction faults handled through emulation\000config=8\000\00000\000\000\000\000\000" +/* offset=124782 */ +"dummy\000software\000A placeholder event that doesn't count anything\000config=9\000\00000\000\000\000\000\000" +/* offset=124862 */ +"bpf-output\000software\000An event used by BPF programs to write to the perf ring buffer\000config=0xa\000\00000\000\000\000\000\000" +/* offset=124964 */ +"cgroup-switches\000software\000Number of context switches to a task in a different cgroup\000config=0xb\000\00000\000\000\000\000\000" +/* offset=125067 */ +"tool\000" +/* offset=125072 */ +"duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000\00000\000\000\000\000\000" +/* offset=125148 */ +"user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000\000\000\000\000\000" +/* offset=125218 */ +"system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\000\000\000\000\000" +/* offset=125286 */ +"has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000\00000\000\000\000\000\000" +/* offset=125362 */ +"num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU\000config=5\000\00000\000\000\000\000\000" +/* offset=125507 */ +"num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPUs on a core\000config=6\000\00000\000\000\000\000\000" +/* offset=125610 */ +"num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be multiple such CPUs on a core\000config=7\000\00000\000\000\000\000\000" +/* offset=125727 */ +"num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000\00000\000\000\000\000\000" +/* offset=125803 */ +"num_packages\000tool\000Number of packages. Each package has 1 or more die\000config=9\000\00000\000\000\000\000\000" +/* offset=125889 */ +"slots\000tool\000Number of functional units that in parallel can execute parts of an instruction\000config=0xa\000\00000\000\000\000\000\000" +/* offset=125999 */ +"smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0\000config=0xb\000\00000\000\000\000\000\000" +/* offset=126106 */ +"system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per second\000config=0xc\000\00000\000\000\000\000\000" +/* offset=126205 */ +"core_wide\000tool\0001 if not SMT, if SMT are events being gathered on all SMT threads 1 otherwise 0\000config=0xd\000\00000\000\000\000\000\000" +/* offset=126319 */ +"target_cpu\000tool\0001 if CPUs being analyzed, 0 if threads/processes\000config=0xe\000\00000\000\000\000\000\000" +/* offset=126403 */ +"bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000\000\000\000" +/* offset=126465 */ +"bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000\000\000\000" +/* offset=126527 */ +"l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000\000\000\000Attributable Level 3 cache access, read\000" +/* offset=126625 */ +"segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000\000\000\000" +/* offset=126727 */ +"dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000\000\000\000" +/* offset=126860 */ +"eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000\000\000\000" +/* offset=126978 */ +"hisi_sccl,ddrc\000" +/* offset=126993 */ +"uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000\000\000\000\000" +/* offset=127063 */ +"uncore_cbox\000" +/* offset=127075 */ +"unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000\000\000\000\000" +/* offset=127229 */ +"event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000\000\000\000\000" +/* offset=127283 */ +"event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000\000\000\000\000" +/* offset=127341 */ +"hisi_sccl,l3c\000" +/* offset=127355 */ +"uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000\000\000\000\000" +/* offset=127423 */ +"uncore_imc_free_running\000" +/* offset=127447 */ +"uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000\000\000\000\000" +/* offset=127527 */ +"uncore_imc\000" +/* offset=127538 */ +"uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000\000\000\000\000" +/* offset=127603 */ +"uncore_sys_ddr_pmu\000" +/* offset=127622 */ +"sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000\000\000\000" +/* offset=127698 */ +"uncore_sys_ccn_pmu\000" +/* offset=127717 */ +"sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000\000\000\000" +/* offset=127794 */ +"uncore_sys_cmn_pmu\000" +/* offset=127813 */ +"sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000\000\000\000" +/* offset=127956 */ +"CPUs_utilized\000Default\000(software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@) / (duration_time * 1e9)\000\000Average CPU utilization\000\0001CPUs\000\000\000\000011" +/* offset=128142 */ +"cs_per_second\000Default\000software@context\\-switches\\,name\\=context\\-switches@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Context switches per CPU second\000\0001cs/sec\000\000\000\000011" +/* offset=128375 */ +"migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-migrations@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Process migrations to a new CPU per CPU second\000\0001migrations/sec\000\000\000\000011" +/* offset=128635 */ +"page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faults@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Page faults per CPU second\000\0001faults/sec\000\000\000\000011" +/* offset=128866 */ +"insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\000Instructions Per Cycle\000\0001instructions\000\000\000\000001" +/* offset=128979 */ +"stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions if has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend) else (stalled\\-cycles\\-frontend / instructions if has_event(stalled\\-cycles\\-frontend) else (stalled\\-cycles\\-backend / instructions if has_event(stalled\\-cycles\\-backend) else 0)))\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001" +/* offset=129404 */ +"frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles if has_event(stalled\\-cycles\\-frontend) else 0)\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001" +/* offset=129583 */ +"backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if has_event(stalled\\-cycles\\-backend) else 0)\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001" +/* offset=129757 */ +"cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011" +/* offset=129933 */ +"branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011" +/* offset=130113 */ +"branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001" +/* offset=130217 */ +"l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001" +/* offset=130333 */ +"llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001" +/* offset=130434 */ +"l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001" +/* offset=130549 */ +"dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001" +/* offset=130655 */ +"itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001" +/* offset=130761 */ +"l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001" +/* offset=130909 */ +"CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000" +/* offset=130932 */ +"IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000" +/* offset=130996 */ +"Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000" +/* offset=131163 */ +"dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" +/* offset=131228 */ +"icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" +/* offset=131296 */ +"cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000" +/* offset=131368 */ +"DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000" +/* offset=131463 */ +"DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000" +/* offset=131598 */ +"DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000" +/* offset=131663 */ +"DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000" +/* offset=131732 */ +"DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000" +/* offset=131803 */ +"M1\000\000ipc + M2\000\000\000\000\000\000\000\000000" +/* offset=131826 */ +"M2\000\000ipc + M1\000\000\000\000\000\000\000\000000" +/* offset=131849 */ +"M3\000\0001 / M3\000\000\000\000\000\000\000\000000" +/* offset=131870 */ +"L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000" ; static const struct compact_pmu_event pmu_events__common_default_core[] = { -{ 111480 }, /* bpc\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113849 }, /* bpc-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111564 }, /* bpc-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111939 }, /* bpc-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112135 }, /* bpc-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 112035 }, /* bpc-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 111846 }, /* bpc-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111747 }, /* bpc-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111653 }, /* bpc-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112233 }, /* bpc-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112612 }, /* bpc-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112810 }, /* bpc-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 112709 }, /* bpc-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 112518 }, /* bpc-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112418 }, /* bpc-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112323 }, /* bpc-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 114035 }, /* bpc-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 113940 }, /* bpc-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 113761 }, /* bpc-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112909 }, /* bpc-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113284 }, /* bpc-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113480 }, /* bpc-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 113380 }, /* bpc-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 113191 }, /* bpc-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113092 }, /* bpc-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 112998 }, /* bpc-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113667 }, /* bpc-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 113578 }, /* bpc-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106184 }, /* bpu\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108553 }, /* bpu-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106268 }, /* bpu-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106643 }, /* bpu-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106839 }, /* bpu-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 106739 }, /* bpu-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 106550 }, /* bpu-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106451 }, /* bpu-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106357 }, /* bpu-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106937 }, /* bpu-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107316 }, /* bpu-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107514 }, /* bpu-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 107413 }, /* bpu-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 107222 }, /* bpu-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107122 }, /* bpu-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107027 }, /* bpu-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108739 }, /* bpu-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 108644 }, /* bpu-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 108465 }, /* bpu-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107613 }, /* bpu-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107988 }, /* bpu-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108184 }, /* bpu-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 108084 }, /* bpu-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 107895 }, /* bpu-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107796 }, /* bpu-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 107702 }, /* bpu-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108371 }, /* bpu-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108282 }, /* bpu-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 100851 }, /* branch\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103295 }, /* branch-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 122452 }, /* branch-instructions\000legacy hardware\000Retired branch instructions [This event is an alias of branches]\000legacy-hardware-config=4\000\00000\000\000\000\000\000 */ -{ 100938 }, /* branch-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101325 }, /* branch-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101527 }, /* branch-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 101424 }, /* branch-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00000\000\000\000\000\000 */ -{ 101229 }, /* branch-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101127 }, /* branch-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101030 }, /* branch-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101628 }, /* branch-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00000\000\000\000\000\000 */ -{ 102019 }, /* branch-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102223 }, /* branch-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 102119 }, /* branch-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 101922 }, /* branch-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101819 }, /* branch-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 101721 }, /* branch-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103389 }, /* branch-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 122586 }, /* branch-misses\000legacy hardware\000Mispredicted branch instructions\000legacy-hardware-config=5\000\00000\000\000\000\000\000 */ -{ 103204 }, /* branch-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102325 }, /* branch-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102712 }, /* branch-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102914 }, /* branch-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 102811 }, /* branch-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 102616 }, /* branch-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102514 }, /* branch-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 102417 }, /* branch-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103107 }, /* branch-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103015 }, /* branch-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 122318 }, /* branches\000legacy hardware\000Retired branch instructions [This event is an alias of branch-instructions]\000legacy-hardware-config=4\000\00000\000\000\000\000\000 */ -{ 105890 }, /* branches-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103485 }, /* branches-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103880 }, /* branches-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104086 }, /* branches-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 103981 }, /* branches-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 103782 }, /* branches-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103678 }, /* branches-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 103579 }, /* branches-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104189 }, /* branches-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104588 }, /* branches-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104796 }, /* branches-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 104690 }, /* branches-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 104489 }, /* branches-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104384 }, /* branches-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104284 }, /* branches-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 106086 }, /* branches-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 105986 }, /* branches-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 105797 }, /* branches-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104900 }, /* branches-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 105295 }, /* branches-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 105501 }, /* branches-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 105396 }, /* branches-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 105197 }, /* branches-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 105093 }, /* branches-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 104994 }, /* branches-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 105698 }, /* branches-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 105604 }, /* branches-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108832 }, /* btb\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111201 }, /* btb-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 108916 }, /* btb-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109291 }, /* btb-load-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109487 }, /* btb-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 109387 }, /* btb-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 109198 }, /* btb-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109099 }, /* btb-load-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109005 }, /* btb-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109585 }, /* btb-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109964 }, /* btb-loads-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110162 }, /* btb-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 110061 }, /* btb-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 109870 }, /* btb-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109770 }, /* btb-loads-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 109675 }, /* btb-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111387 }, /* btb-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 111292 }, /* btb-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 111113 }, /* btb-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110261 }, /* btb-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110636 }, /* btb-read-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110832 }, /* btb-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 110732 }, /* btb-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache-config=0x10005\000\00010\000\000\000\000\000 */ -{ 110543 }, /* btb-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110444 }, /* btb-read-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110350 }, /* btb-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 111019 }, /* btb-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 110930 }, /* btb-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-config=5\000\00010\000\000\000\000\000 */ -{ 122682 }, /* bus-cycles\000legacy hardware\000Bus cycles, which can be different from total cycles\000legacy-hardware-config=6\000\00000\000\000\000\000\000 */ -{ 122075 }, /* cache-misses\000legacy hardware\000Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in conjunction with the PERF_COUNT_HW_CACHE_REFERENCES event to calculate cache miss rates\000legacy-hardware-config=3\000\00000\000\000\000\000\000 */ -{ 121805 }, /* cache-references\000legacy hardware\000Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU\000legacy-hardware-config=2\000\00000\000\000\000\000\000 */ -{ 121305 }, /* cpu-cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000 */ -{ 121467 }, /* cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU frequency scaling [This event is an alias of cpu-cycles]\000legacy-hardware-config=0\000\00000\000\000\000\000\000 */ -{ 78952 }, /* d-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 85655 }, /* d-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79024 }, /* d-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79351 }, /* d-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79523 }, /* d-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 79435 }, /* d-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 79270 }, /* d-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79183 }, /* d-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79101 }, /* d-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79609 }, /* d-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79940 }, /* d-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 80114 }, /* d-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 80025 }, /* d-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 79858 }, /* d-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79770 }, /* d-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 79687 }, /* d-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 85817 }, /* d-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 85734 }, /* d-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 85579 }, /* d-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 82650 }, /* d-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83025 }, /* d-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83217 }, /* d-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 83121 }, /* d-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 82932 }, /* d-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 82833 }, /* d-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 82739 }, /* d-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83311 }, /* d-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83694 }, /* d-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83890 }, /* d-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 83792 }, /* d-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 83599 }, /* d-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83498 }, /* d-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83402 }, /* d-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 80201 }, /* d-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 80528 }, /* d-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 80700 }, /* d-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 80612 }, /* d-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 80447 }, /* d-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 80360 }, /* d-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 80278 }, /* d-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 85497 }, /* d-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 85420 }, /* d-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 84703 }, /* d-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 85110 }, /* d-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 85318 }, /* d-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 85214 }, /* d-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 85009 }, /* d-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84902 }, /* d-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84800 }, /* d-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 83986 }, /* d-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84393 }, /* d-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84601 }, /* d-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 84497 }, /* d-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 84292 }, /* d-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84185 }, /* d-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 84083 }, /* d-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 80786 }, /* d-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81137 }, /* d-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81317 }, /* d-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 81227 }, /* d-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 81050 }, /* d-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 80957 }, /* d-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 80869 }, /* d-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81405 }, /* d-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81760 }, /* d-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81942 }, /* d-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 81851 }, /* d-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 81672 }, /* d-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81578 }, /* d-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 81489 }, /* d-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 82031 }, /* d-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 82382 }, /* d-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 82562 }, /* d-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 82472 }, /* d-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 82295 }, /* d-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 82202 }, /* d-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 82114 }, /* d-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 85898 }, /* data-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 92823 }, /* data-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 85973 }, /* data-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86312 }, /* data-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86490 }, /* data-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 86399 }, /* data-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 86228 }, /* data-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86138 }, /* data-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86053 }, /* data-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86579 }, /* data-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86922 }, /* data-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 87102 }, /* data-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 87010 }, /* data-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 86837 }, /* data-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86746 }, /* data-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 86660 }, /* data-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 92991 }, /* data-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 92905 }, /* data-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 92744 }, /* data-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 89725 }, /* data-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90112 }, /* data-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90310 }, /* data-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 90211 }, /* data-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 90016 }, /* data-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 89914 }, /* data-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 89817 }, /* data-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90407 }, /* data-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90802 }, /* data-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91004 }, /* data-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 90903 }, /* data-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 90704 }, /* data-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90600 }, /* data-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 90501 }, /* data-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 87192 }, /* data-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 87531 }, /* data-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 87709 }, /* data-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 87618 }, /* data-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 87447 }, /* data-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 87357 }, /* data-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 87272 }, /* data-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 92659 }, /* data-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 92579 }, /* data-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 91841 }, /* data-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 92260 }, /* data-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 92474 }, /* data-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 92367 }, /* data-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 92156 }, /* data-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 92046 }, /* data-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91941 }, /* data-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91103 }, /* data-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91522 }, /* data-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91736 }, /* data-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 91629 }, /* data-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 91418 }, /* data-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91308 }, /* data-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 91203 }, /* data-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 87798 }, /* data-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88161 }, /* data-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88347 }, /* data-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 88254 }, /* data-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 88071 }, /* data-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 87975 }, /* data-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 87884 }, /* data-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88438 }, /* data-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88805 }, /* data-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88993 }, /* data-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 88899 }, /* data-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 88714 }, /* data-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88617 }, /* data-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 88525 }, /* data-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 89085 }, /* data-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 89448 }, /* data-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 89634 }, /* data-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 89541 }, /* data-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 89358 }, /* data-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 89262 }, /* data-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 89171 }, /* data-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 72083 }, /* dtlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 78712 }, /* dtlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72154 }, /* dtlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72477 }, /* dtlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72647 }, /* dtlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 72560 }, /* dtlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00000\000\000\000\000\000 */ -{ 72397 }, /* dtlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72311 }, /* dtlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72230 }, /* dtlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72732 }, /* dtlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00000\000\000\000\000\000 */ -{ 73059 }, /* dtlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 73231 }, /* dtlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 73143 }, /* dtlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 72978 }, /* dtlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72891 }, /* dtlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 72809 }, /* dtlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 78872 }, /* dtlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 78790 }, /* dtlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 78637 }, /* dtlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 75738 }, /* dtlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76109 }, /* dtlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76299 }, /* dtlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 76204 }, /* dtlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00000\000\000\000\000\000 */ -{ 76017 }, /* dtlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 75919 }, /* dtlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 75826 }, /* dtlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76392 }, /* dtlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00000\000\000\000\000\000 */ -{ 76771 }, /* dtlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76965 }, /* dtlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 76868 }, /* dtlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 76677 }, /* dtlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76577 }, /* dtlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 76482 }, /* dtlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 73317 }, /* dtlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 73640 }, /* dtlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 73810 }, /* dtlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 73723 }, /* dtlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003\000\00010\000\000\000\000\000 */ -{ 73560 }, /* dtlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 73474 }, /* dtlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 73393 }, /* dtlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 78556 }, /* dtlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 78480 }, /* dtlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\00010\000\000\000\000\000 */ -{ 77770 }, /* dtlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 78173 }, /* dtlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 78379 }, /* dtlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 78276 }, /* dtlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 78073 }, /* dtlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77967 }, /* dtlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77866 }, /* dtlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77060 }, /* dtlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77463 }, /* dtlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77669 }, /* dtlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 77566 }, /* dtlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache-config=0x10203\000\00010\000\000\000\000\000 */ -{ 77363 }, /* dtlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77257 }, /* dtlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 77156 }, /* dtlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-config=0x203\000\00010\000\000\000\000\000 */ -{ 73895 }, /* dtlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74242 }, /* dtlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74420 }, /* dtlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 74331 }, /* dtlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00000\000\000\000\000\000 */ -{ 74156 }, /* dtlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74064 }, /* dtlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 73977 }, /* dtlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74507 }, /* dtlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00000\000\000\000\000\000 */ -{ 74858 }, /* dtlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75038 }, /* dtlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 74948 }, /* dtlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 74771 }, /* dtlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74678 }, /* dtlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 74590 }, /* dtlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75126 }, /* dtlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75473 }, /* dtlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75651 }, /* dtlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 75562 }, /* dtlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-config=0x10103\000\00010\000\000\000\000\000 */ -{ 75387 }, /* dtlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75295 }, /* dtlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 75208 }, /* dtlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x103\000\00010\000\000\000\000\000 */ -{ 95555 }, /* i-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97799 }, /* i-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95634 }, /* i-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95989 }, /* i-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96175 }, /* i-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 96080 }, /* i-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 95901 }, /* i-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95807 }, /* i-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95718 }, /* i-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96268 }, /* i-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96627 }, /* i-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96815 }, /* i-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 96719 }, /* i-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 96538 }, /* i-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96443 }, /* i-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96353 }, /* i-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97975 }, /* i-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 97885 }, /* i-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 97716 }, /* i-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96909 }, /* i-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97264 }, /* i-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97450 }, /* i-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 97355 }, /* i-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 97176 }, /* i-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97082 }, /* i-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 96993 }, /* i-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97627 }, /* i-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 97543 }, /* i-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 123247 }, /* idle-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of stalled-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000 */ -{ 122945 }, /* idle-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of stalled-cycles-fronted]\000legacy-hardware-config=7\000\00000\000\000\000\000\000 */ -{ 98063 }, /* instruction-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 100557 }, /* instruction-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98152 }, /* instruction-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98547 }, /* instruction-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98753 }, /* instruction-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 98648 }, /* instruction-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 98449 }, /* instruction-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98345 }, /* instruction-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98246 }, /* instruction-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98856 }, /* instruction-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99255 }, /* instruction-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99463 }, /* instruction-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 99357 }, /* instruction-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 99156 }, /* instruction-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99051 }, /* instruction-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 98951 }, /* instruction-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 100753 }, /* instruction-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 100653 }, /* instruction-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 100464 }, /* instruction-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99567 }, /* instruction-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99962 }, /* instruction-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 100168 }, /* instruction-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 100063 }, /* instruction-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 99864 }, /* instruction-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99760 }, /* instruction-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 99661 }, /* instruction-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 100365 }, /* instruction-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 100271 }, /* instruction-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 121629 }, /* instructions\000legacy hardware\000Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts\000legacy-hardware-config=1\000\00000\000\000\000\000\000 */ -{ 93075 }, /* itlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95294 }, /* itlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93153 }, /* itlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93504 }, /* itlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93688 }, /* itlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 93594 }, /* itlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00000\000\000\000\000\000 */ -{ 93417 }, /* itlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93324 }, /* itlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93236 }, /* itlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93780 }, /* itlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00000\000\000\000\000\000 */ -{ 94135 }, /* itlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94321 }, /* itlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 94226 }, /* itlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 94047 }, /* itlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93953 }, /* itlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 93864 }, /* itlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95468 }, /* itlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 95379 }, /* itlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 95212 }, /* itlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94414 }, /* itlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94765 }, /* itlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94949 }, /* itlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 94855 }, /* itlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=0x10004\000\00010\000\000\000\000\000 */ -{ 94678 }, /* itlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94585 }, /* itlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 94497 }, /* itlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95124 }, /* itlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 95041 }, /* itlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\000\00010\000\000\000\000\000 */ -{ 8037 }, /* l1-d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15406 }, /* l1-d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8118 }, /* l1-d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8481 }, /* l1-d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8671 }, /* l1-d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 8574 }, /* l1-d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 8391 }, /* l1-d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8295 }, /* l1-d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8204 }, /* l1-d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8766 }, /* l1-d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9133 }, /* l1-d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9325 }, /* l1-d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 9227 }, /* l1-d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 9042 }, /* l1-d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8945 }, /* l1-d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 8853 }, /* l1-d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15586 }, /* l1-d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 15494 }, /* l1-d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 15321 }, /* l1-d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 12122 }, /* l1-d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12533 }, /* l1-d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12743 }, /* l1-d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 12638 }, /* l1-d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 12431 }, /* l1-d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12323 }, /* l1-d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12220 }, /* l1-d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12846 }, /* l1-d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13265 }, /* l1-d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13479 }, /* l1-d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 13372 }, /* l1-d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 13161 }, /* l1-d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13051 }, /* l1-d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 12946 }, /* l1-d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 9421 }, /* l1-d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9784 }, /* l1-d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9974 }, /* l1-d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 9877 }, /* l1-d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 9694 }, /* l1-d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9598 }, /* l1-d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 9507 }, /* l1-d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15230 }, /* l1-d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15144 }, /* l1-d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 14364 }, /* l1-d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 14807 }, /* l1-d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 15033 }, /* l1-d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 14920 }, /* l1-d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 14697 }, /* l1-d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 14581 }, /* l1-d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 14470 }, /* l1-d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13584 }, /* l1-d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 14027 }, /* l1-d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 14253 }, /* l1-d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 14140 }, /* l1-d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 13917 }, /* l1-d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13801 }, /* l1-d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 13690 }, /* l1-d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 10069 }, /* l1-d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10456 }, /* l1-d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10654 }, /* l1-d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 10555 }, /* l1-d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 10360 }, /* l1-d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10258 }, /* l1-d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10161 }, /* l1-d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10751 }, /* l1-d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11142 }, /* l1-d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11342 }, /* l1-d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 11242 }, /* l1-d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 11045 }, /* l1-d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10942 }, /* l1-d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 10844 }, /* l1-d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11440 }, /* l1-d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11827 }, /* l1-d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 12025 }, /* l1-d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 11926 }, /* l1-d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 11731 }, /* l1-d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11629 }, /* l1-d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 11532 }, /* l1-d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 23238 }, /* l1-data\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 30829 }, /* l1-data-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23322 }, /* l1-data-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23697 }, /* l1-data-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23893 }, /* l1-data-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 23793 }, /* l1-data-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 23604 }, /* l1-data-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23505 }, /* l1-data-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23411 }, /* l1-data-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23991 }, /* l1-data-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24370 }, /* l1-data-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24568 }, /* l1-data-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 24467 }, /* l1-data-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 24276 }, /* l1-data-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24176 }, /* l1-data-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24081 }, /* l1-data-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 31015 }, /* l1-data-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 30920 }, /* l1-data-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 30741 }, /* l1-data-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 27452 }, /* l1-data-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 27875 }, /* l1-data-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28091 }, /* l1-data-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 27983 }, /* l1-data-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 27770 }, /* l1-data-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 27659 }, /* l1-data-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 27553 }, /* l1-data-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28197 }, /* l1-data-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28628 }, /* l1-data-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28848 }, /* l1-data-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 28738 }, /* l1-data-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 28521 }, /* l1-data-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28408 }, /* l1-data-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28300 }, /* l1-data-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 24667 }, /* l1-data-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 25042 }, /* l1-data-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 25238 }, /* l1-data-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 25138 }, /* l1-data-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 24949 }, /* l1-data-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24850 }, /* l1-data-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 24756 }, /* l1-data-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 30647 }, /* l1-data-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 30558 }, /* l1-data-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 29757 }, /* l1-data-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 30212 }, /* l1-data-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 30444 }, /* l1-data-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 30328 }, /* l1-data-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 30099 }, /* l1-data-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29980 }, /* l1-data-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29866 }, /* l1-data-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 28956 }, /* l1-data-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29411 }, /* l1-data-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29643 }, /* l1-data-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 29527 }, /* l1-data-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 29298 }, /* l1-data-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29179 }, /* l1-data-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 29065 }, /* l1-data-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 25336 }, /* l1-data-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 25735 }, /* l1-data-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 25939 }, /* l1-data-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 25837 }, /* l1-data-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 25636 }, /* l1-data-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 25531 }, /* l1-data-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 25431 }, /* l1-data-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26039 }, /* l1-data-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26442 }, /* l1-data-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26648 }, /* l1-data-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 26545 }, /* l1-data-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 26342 }, /* l1-data-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26236 }, /* l1-data-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26135 }, /* l1-data-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26749 }, /* l1-data-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 27148 }, /* l1-data-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 27352 }, /* l1-data-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 27250 }, /* l1-data-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 27049 }, /* l1-data-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26944 }, /* l1-data-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 26844 }, /* l1-data-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 13 }, /* l1-dcache\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 7752 }, /* l1-dcache-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 99 }, /* l1-dcache-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 482 }, /* l1-dcache-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 682 }, /* l1-dcache-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 580 }, /* l1-dcache-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00000\000\000\000\000\000 */ -{ 387 }, /* l1-dcache-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 286 }, /* l1-dcache-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 190 }, /* l1-dcache-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 782 }, /* l1-dcache-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00000\000\000\000\000\000 */ -{ 1169 }, /* l1-dcache-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 1371 }, /* l1-dcache-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 1268 }, /* l1-dcache-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 1073 }, /* l1-dcache-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 971 }, /* l1-dcache-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 874 }, /* l1-dcache-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 7942 }, /* l1-dcache-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 7845 }, /* l1-dcache-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 7662 }, /* l1-dcache-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 4313 }, /* l1-dcache-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 4744 }, /* l1-dcache-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 4964 }, /* l1-dcache-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 4854 }, /* l1-dcache-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00000\000\000\000\000\000 */ -{ 4637 }, /* l1-dcache-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 4524 }, /* l1-dcache-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 4416 }, /* l1-dcache-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5072 }, /* l1-dcache-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00000\000\000\000\000\000 */ -{ 5511 }, /* l1-dcache-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5735 }, /* l1-dcache-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 5623 }, /* l1-dcache-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 5402 }, /* l1-dcache-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5287 }, /* l1-dcache-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5177 }, /* l1-dcache-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 1472 }, /* l1-dcache-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 1855 }, /* l1-dcache-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 2055 }, /* l1-dcache-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 1953 }, /* l1-dcache-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 1760 }, /* l1-dcache-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 1659 }, /* l1-dcache-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 1563 }, /* l1-dcache-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 7566 }, /* l1-dcache-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 7475 }, /* l1-dcache-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 6660 }, /* l1-dcache-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 7123 }, /* l1-dcache-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 7359 }, /* l1-dcache-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 7241 }, /* l1-dcache-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 7008 }, /* l1-dcache-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 6887 }, /* l1-dcache-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 6771 }, /* l1-dcache-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5845 }, /* l1-dcache-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 6308 }, /* l1-dcache-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 6544 }, /* l1-dcache-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 6426 }, /* l1-dcache-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 6193 }, /* l1-dcache-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 6072 }, /* l1-dcache-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 5956 }, /* l1-dcache-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 2155 }, /* l1-dcache-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2562 }, /* l1-dcache-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2770 }, /* l1-dcache-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 2666 }, /* l1-dcache-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00000\000\000\000\000\000 */ -{ 2461 }, /* l1-dcache-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2354 }, /* l1-dcache-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2252 }, /* l1-dcache-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2872 }, /* l1-dcache-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00000\000\000\000\000\000 */ -{ 3283 }, /* l1-dcache-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 3493 }, /* l1-dcache-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 3388 }, /* l1-dcache-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 3181 }, /* l1-dcache-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 3073 }, /* l1-dcache-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 2970 }, /* l1-dcache-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 3596 }, /* l1-dcache-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 4003 }, /* l1-dcache-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 4211 }, /* l1-dcache-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 4107 }, /* l1-dcache-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 3902 }, /* l1-dcache-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 3795 }, /* l1-dcache-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 3693 }, /* l1-dcache-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 37366 }, /* l1-i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43053 }, /* l1-i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37454 }, /* l1-i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37845 }, /* l1-i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38049 }, /* l1-i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 37945 }, /* l1-i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 37748 }, /* l1-i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37645 }, /* l1-i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37547 }, /* l1-i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38151 }, /* l1-i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38546 }, /* l1-i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38752 }, /* l1-i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 38647 }, /* l1-i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 38448 }, /* l1-i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38344 }, /* l1-i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38245 }, /* l1-i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43247 }, /* l1-i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 43148 }, /* l1-i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 42961 }, /* l1-i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 39552 }, /* l1-i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 39991 }, /* l1-i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 40215 }, /* l1-i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 40103 }, /* l1-i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 39882 }, /* l1-i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 39767 }, /* l1-i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 39657 }, /* l1-i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 40325 }, /* l1-i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 40772 }, /* l1-i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41000 }, /* l1-i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 40886 }, /* l1-i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 40661 }, /* l1-i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 40544 }, /* l1-i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 40432 }, /* l1-i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 38855 }, /* l1-i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 39246 }, /* l1-i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 39450 }, /* l1-i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 39346 }, /* l1-i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 39149 }, /* l1-i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 39046 }, /* l1-i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 38948 }, /* l1-i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 42863 }, /* l1-i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 42770 }, /* l1-i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 41941 }, /* l1-i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 42412 }, /* l1-i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 42652 }, /* l1-i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 42532 }, /* l1-i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 42295 }, /* l1-i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 42172 }, /* l1-i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 42054 }, /* l1-i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41112 }, /* l1-i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41583 }, /* l1-i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41823 }, /* l1-i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 41703 }, /* l1-i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 41466 }, /* l1-i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41343 }, /* l1-i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 41225 }, /* l1-i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 31108 }, /* l1-icache\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37060 }, /* l1-icache-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31201 }, /* l1-icache-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31612 }, /* l1-icache-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31826 }, /* l1-icache-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 31717 }, /* l1-icache-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00000\000\000\000\000\000 */ -{ 31510 }, /* l1-icache-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31402 }, /* l1-icache-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31299 }, /* l1-icache-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 31933 }, /* l1-icache-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00000\000\000\000\000\000 */ -{ 32348 }, /* l1-icache-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 32564 }, /* l1-icache-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 32454 }, /* l1-icache-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 32245 }, /* l1-icache-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 32136 }, /* l1-icache-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 32032 }, /* l1-icache-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 37264 }, /* l1-icache-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 37160 }, /* l1-icache-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 36963 }, /* l1-icache-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 33404 }, /* l1-icache-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 33863 }, /* l1-icache-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 34097 }, /* l1-icache-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 33980 }, /* l1-icache-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00000\000\000\000\000\000 */ -{ 33749 }, /* l1-icache-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 33629 }, /* l1-icache-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 33514 }, /* l1-icache-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 34212 }, /* l1-icache-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00000\000\000\000\000\000 */ -{ 34679 }, /* l1-icache-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 34917 }, /* l1-icache-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 34798 }, /* l1-icache-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 34563 }, /* l1-icache-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 34441 }, /* l1-icache-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 34324 }, /* l1-icache-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 32672 }, /* l1-icache-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 33083 }, /* l1-icache-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 33297 }, /* l1-icache-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 33188 }, /* l1-icache-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 32981 }, /* l1-icache-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 32873 }, /* l1-icache-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 32770 }, /* l1-icache-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 36860 }, /* l1-icache-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 36762 }, /* l1-icache-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 35898 }, /* l1-icache-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 36389 }, /* l1-icache-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 36639 }, /* l1-icache-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 36514 }, /* l1-icache-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 36267 }, /* l1-icache-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 36139 }, /* l1-icache-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 36016 }, /* l1-icache-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 35034 }, /* l1-icache-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 35525 }, /* l1-icache-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 35775 }, /* l1-icache-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 35650 }, /* l1-icache-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 35403 }, /* l1-icache-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 35275 }, /* l1-icache-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 35152 }, /* l1-icache-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 49266 }, /* l1-instruction\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 55483 }, /* l1-instruction-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 49364 }, /* l1-instruction-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 49795 }, /* l1-instruction-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50019 }, /* l1-instruction-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 49905 }, /* l1-instruction-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 49688 }, /* l1-instruction-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 49575 }, /* l1-instruction-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 49467 }, /* l1-instruction-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50131 }, /* l1-instruction-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50566 }, /* l1-instruction-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50792 }, /* l1-instruction-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 50677 }, /* l1-instruction-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 50458 }, /* l1-instruction-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50344 }, /* l1-instruction-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 50235 }, /* l1-instruction-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 55697 }, /* l1-instruction-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 55588 }, /* l1-instruction-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 55381 }, /* l1-instruction-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 51672 }, /* l1-instruction-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 52151 }, /* l1-instruction-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 52395 }, /* l1-instruction-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 52273 }, /* l1-instruction-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 52032 }, /* l1-instruction-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 51907 }, /* l1-instruction-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 51787 }, /* l1-instruction-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 52515 }, /* l1-instruction-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53002 }, /* l1-instruction-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53250 }, /* l1-instruction-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 53126 }, /* l1-instruction-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 52881 }, /* l1-instruction-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 52754 }, /* l1-instruction-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 52632 }, /* l1-instruction-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 50905 }, /* l1-instruction-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 51336 }, /* l1-instruction-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 51560 }, /* l1-instruction-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 51446 }, /* l1-instruction-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 51229 }, /* l1-instruction-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 51116 }, /* l1-instruction-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 51008 }, /* l1-instruction-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 55273 }, /* l1-instruction-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 55170 }, /* l1-instruction-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 54271 }, /* l1-instruction-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 54782 }, /* l1-instruction-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 55042 }, /* l1-instruction-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 54912 }, /* l1-instruction-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 54655 }, /* l1-instruction-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 54522 }, /* l1-instruction-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 54394 }, /* l1-instruction-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53372 }, /* l1-instruction-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53883 }, /* l1-instruction-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 54143 }, /* l1-instruction-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 54013 }, /* l1-instruction-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 53756 }, /* l1-instruction-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53623 }, /* l1-instruction-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 53495 }, /* l1-instruction-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 15676 }, /* l1d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 22971 }, /* l1d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15756 }, /* l1d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16115 }, /* l1d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16303 }, /* l1d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 16207 }, /* l1d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 16026 }, /* l1d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15931 }, /* l1d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 15841 }, /* l1d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16397 }, /* l1d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16760 }, /* l1d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16950 }, /* l1d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 16853 }, /* l1d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 16670 }, /* l1d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16574 }, /* l1d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 16483 }, /* l1d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 23149 }, /* l1d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 23058 }, /* l1d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 22887 }, /* l1d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 19718 }, /* l1d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20125 }, /* l1d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20333 }, /* l1d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 20229 }, /* l1d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 20024 }, /* l1d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 19917 }, /* l1d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 19815 }, /* l1d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20435 }, /* l1d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20850 }, /* l1d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21062 }, /* l1d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 20956 }, /* l1d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 20747 }, /* l1d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20638 }, /* l1d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 20534 }, /* l1d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 17045 }, /* l1d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 17404 }, /* l1d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 17592 }, /* l1d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 17496 }, /* l1d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-config=0x10000\000\00010\000\000\000\000\000 */ -{ 17315 }, /* l1d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 17220 }, /* l1d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 17130 }, /* l1d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 22797 }, /* l1d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 22712 }, /* l1d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0\000\00010\000\000\000\000\000 */ -{ 21939 }, /* l1d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 22378 }, /* l1d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 22602 }, /* l1d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 22490 }, /* l1d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 22269 }, /* l1d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 22154 }, /* l1d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 22044 }, /* l1d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21166 }, /* l1d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21605 }, /* l1d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21829 }, /* l1d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 21717 }, /* l1d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misses\000legacy-cache-config=0x10200\000\00010\000\000\000\000\000 */ -{ 21496 }, /* l1d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21381 }, /* l1d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 21271 }, /* l1d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-cache-config=0x200\000\00010\000\000\000\000\000 */ -{ 17686 }, /* l1d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18069 }, /* l1d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18265 }, /* l1d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 18167 }, /* l1d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 17974 }, /* l1d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 17873 }, /* l1d-store-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 17777 }, /* l1d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18361 }, /* l1d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18748 }, /* l1d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18946 }, /* l1d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 18847 }, /* l1d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 18652 }, /* l1d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18550 }, /* l1d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 18453 }, /* l1d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 19043 }, /* l1d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 19426 }, /* l1d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 19622 }, /* l1d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 19524 }, /* l1d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-cache-config=0x10100\000\00010\000\000\000\000\000 */ -{ 19331 }, /* l1d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 19230 }, /* l1d-write-reference\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 19134 }, /* l1d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-config=0x100\000\00010\000\000\000\000\000 */ -{ 43344 }, /* l1i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 48978 }, /* l1i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43431 }, /* l1i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43818 }, /* l1i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44020 }, /* l1i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 43917 }, /* l1i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 43722 }, /* l1i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43620 }, /* l1i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 43523 }, /* l1i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44121 }, /* l1i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44512 }, /* l1i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44716 }, /* l1i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 44612 }, /* l1i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 44415 }, /* l1i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44312 }, /* l1i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44214 }, /* l1i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 49170 }, /* l1i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 49072 }, /* l1i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 48887 }, /* l1i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 45508 }, /* l1i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 45943 }, /* l1i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46165 }, /* l1i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 46054 }, /* l1i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 45835 }, /* l1i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 45721 }, /* l1i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 45612 }, /* l1i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46274 }, /* l1i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46717 }, /* l1i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46943 }, /* l1i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 46830 }, /* l1i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 46607 }, /* l1i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46491 }, /* l1i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 46380 }, /* l1i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 44818 }, /* l1i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 45205 }, /* l1i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 45407 }, /* l1i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 45304 }, /* l1i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-cache-config=0x10001\000\00010\000\000\000\000\000 */ -{ 45109 }, /* l1i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 45007 }, /* l1i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 44910 }, /* l1i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 48790 }, /* l1i-reference\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 48698 }, /* l1i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-config=1\000\00010\000\000\000\000\000 */ -{ 47876 }, /* l1i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 48343 }, /* l1i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 48581 }, /* l1i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 48462 }, /* l1i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 48227 }, /* l1i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 48105 }, /* l1i-speculative-load-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47988 }, /* l1i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47054 }, /* l1i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47521 }, /* l1i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47759 }, /* l1i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 47640 }, /* l1i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetch misses\000legacy-cache-config=0x10201\000\00010\000\000\000\000\000 */ -{ 47405 }, /* l1i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47283 }, /* l1i-speculative-read-reference\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 47166 }, /* l1i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch accesses\000legacy-cache-config=0x201\000\00010\000\000\000\000\000 */ -{ 63212 }, /* l2\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 71765 }, /* l2-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63309 }, /* l2-load\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63736 }, /* l2-load-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63958 }, /* l2-load-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 63845 }, /* l2-load-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 63630 }, /* l2-load-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63518 }, /* l2-load-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63411 }, /* l2-load-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64069 }, /* l2-loads\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64500 }, /* l2-loads-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64724 }, /* l2-loads-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 64610 }, /* l2-loads-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 64393 }, /* l2-loads-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64280 }, /* l2-loads-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64172 }, /* l2-loads-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 71977 }, /* l2-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 71869 }, /* l2-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 71664 }, /* l2-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 67985 }, /* l2-prefetch\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68460 }, /* l2-prefetch-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68702 }, /* l2-prefetch-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 68581 }, /* l2-prefetch-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 68342 }, /* l2-prefetch-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68218 }, /* l2-prefetch-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68099 }, /* l2-prefetch-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68821 }, /* l2-prefetches\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69304 }, /* l2-prefetches-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69550 }, /* l2-prefetches-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 69427 }, /* l2-prefetches-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 69184 }, /* l2-prefetches-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69058 }, /* l2-prefetches-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 68937 }, /* l2-prefetches-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 64836 }, /* l2-read\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 65263 }, /* l2-read-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 65485 }, /* l2-read-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 65372 }, /* l2-read-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 65157 }, /* l2-read-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 65045 }, /* l2-read-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 64938 }, /* l2-read-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 71557 }, /* l2-reference\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 71455 }, /* l2-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 70563 }, /* l2-speculative-load\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 71070 }, /* l2-speculative-load-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 71328 }, /* l2-speculative-load-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 71199 }, /* l2-speculative-load-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 70944 }, /* l2-speculative-load-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 70812 }, /* l2-speculative-load-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 70685 }, /* l2-speculative-load-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69671 }, /* l2-speculative-read\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 70178 }, /* l2-speculative-read-access\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 70436 }, /* l2-speculative-read-miss\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 70307 }, /* l2-speculative-read-misses\000legacy cache\000Level 2 (or higher) last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 70052 }, /* l2-speculative-read-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69920 }, /* l2-speculative-read-reference\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 69793 }, /* l2-speculative-read-refs\000legacy cache\000Level 2 (or higher) last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 65596 }, /* l2-store\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66047 }, /* l2-store-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66277 }, /* l2-store-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 66162 }, /* l2-store-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 65935 }, /* l2-store-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 65817 }, /* l2-store-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 65704 }, /* l2-store-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66390 }, /* l2-stores\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66845 }, /* l2-stores-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67077 }, /* l2-stores-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 66961 }, /* l2-stores-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 66732 }, /* l2-stores-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66613 }, /* l2-stores-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 66499 }, /* l2-stores-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67191 }, /* l2-write\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67642 }, /* l2-write-access\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67872 }, /* l2-write-miss\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 67757 }, /* l2-write-misses\000legacy cache\000Level 2 (or higher) last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 67530 }, /* l2-write-ops\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67412 }, /* l2-write-reference\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 67299 }, /* l2-write-refs\000legacy cache\000Level 2 (or higher) last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 55804 }, /* llc\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 62951 }, /* llc-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 55882 }, /* llc-load\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56233 }, /* llc-load-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56417 }, /* llc-load-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 56323 }, /* llc-load-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00000\000\000\000\000\000 */ -{ 56146 }, /* llc-load-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56053 }, /* llc-load-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 55965 }, /* llc-load-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56509 }, /* llc-loads\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00000\000\000\000\000\000 */ -{ 56864 }, /* llc-loads-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 57050 }, /* llc-loads-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 56955 }, /* llc-loads-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 56776 }, /* llc-loads-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56682 }, /* llc-loads-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 56593 }, /* llc-loads-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 63125 }, /* llc-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 63036 }, /* llc-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 62869 }, /* llc-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 59760 }, /* llc-prefetch\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 60159 }, /* llc-prefetch-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 60363 }, /* llc-prefetch-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 60261 }, /* llc-prefetch-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00000\000\000\000\000\000 */ -{ 60060 }, /* llc-prefetch-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 59955 }, /* llc-prefetch-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 59855 }, /* llc-prefetch-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 60463 }, /* llc-prefetches\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00000\000\000\000\000\000 */ -{ 60870 }, /* llc-prefetches-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61078 }, /* llc-prefetches-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 60974 }, /* llc-prefetches-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 60769 }, /* llc-prefetches-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 60662 }, /* llc-prefetches-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 60560 }, /* llc-prefetches-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 57143 }, /* llc-read\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 57494 }, /* llc-read-access\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 57678 }, /* llc-read-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 57584 }, /* llc-read-misses\000legacy cache\000Last level cache read misses\000legacy-cache-config=0x10002\000\00010\000\000\000\000\000 */ -{ 57407 }, /* llc-read-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 57314 }, /* llc-read-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 57226 }, /* llc-read-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 62781 }, /* llc-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 62698 }, /* llc-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\000\00010\000\000\000\000\000 */ -{ 61939 }, /* llc-speculative-load\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 62370 }, /* llc-speculative-load-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 62590 }, /* llc-speculative-load-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 62480 }, /* llc-speculative-load-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 62263 }, /* llc-speculative-load-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 62150 }, /* llc-speculative-load-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 62042 }, /* llc-speculative-load-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61180 }, /* llc-speculative-read\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61611 }, /* llc-speculative-read-access\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61831 }, /* llc-speculative-read-miss\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 61721 }, /* llc-speculative-read-misses\000legacy cache\000Last level cache prefetch misses\000legacy-cache-config=0x10202\000\00010\000\000\000\000\000 */ -{ 61504 }, /* llc-speculative-read-ops\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61391 }, /* llc-speculative-read-reference\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 61283 }, /* llc-speculative-read-refs\000legacy cache\000Last level cache prefetch accesses\000legacy-cache-config=0x202\000\00010\000\000\000\000\000 */ -{ 57770 }, /* llc-store\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 58145 }, /* llc-store-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 58337 }, /* llc-store-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 58241 }, /* llc-store-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00000\000\000\000\000\000 */ -{ 58052 }, /* llc-store-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 57953 }, /* llc-store-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 57859 }, /* llc-store-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 58431 }, /* llc-stores\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00000\000\000\000\000\000 */ -{ 58810 }, /* llc-stores-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59004 }, /* llc-stores-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 58907 }, /* llc-stores-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 58716 }, /* llc-stores-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 58616 }, /* llc-stores-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 58521 }, /* llc-stores-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59099 }, /* llc-write\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59474 }, /* llc-write-access\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59666 }, /* llc-write-miss\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 59570 }, /* llc-write-misses\000legacy cache\000Last level cache write misses\000legacy-cache-config=0x10102\000\00010\000\000\000\000\000 */ -{ 59381 }, /* llc-write-ops\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59282 }, /* llc-write-reference\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 59188 }, /* llc-write-refs\000legacy cache\000Last level cache write accesses\000legacy-cache-config=0x102\000\00010\000\000\000\000\000 */ -{ 114128 }, /* node\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 121053 }, /* node-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114203 }, /* node-load\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114542 }, /* node-load-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114720 }, /* node-load-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 114629 }, /* node-load-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00000\000\000\000\000\000 */ -{ 114458 }, /* node-load-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114368 }, /* node-load-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114283 }, /* node-load-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114809 }, /* node-loads\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00000\000\000\000\000\000 */ -{ 115152 }, /* node-loads-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 115332 }, /* node-loads-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 115240 }, /* node-loads-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 115067 }, /* node-loads-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114976 }, /* node-loads-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 114890 }, /* node-loads-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 121221 }, /* node-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 121135 }, /* node-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 120974 }, /* node-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 117955 }, /* node-prefetch\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118342 }, /* node-prefetch-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118540 }, /* node-prefetch-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 118441 }, /* node-prefetch-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00000\000\000\000\000\000 */ -{ 118246 }, /* node-prefetch-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118144 }, /* node-prefetch-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118047 }, /* node-prefetch-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118637 }, /* node-prefetches\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00000\000\000\000\000\000 */ -{ 119032 }, /* node-prefetches-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119234 }, /* node-prefetches-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 119133 }, /* node-prefetches-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 118934 }, /* node-prefetches-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118830 }, /* node-prefetches-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 118731 }, /* node-prefetches-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 115422 }, /* node-read\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 115761 }, /* node-read-access\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 115939 }, /* node-read-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 115848 }, /* node-read-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0x10006\000\00010\000\000\000\000\000 */ -{ 115677 }, /* node-read-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 115587 }, /* node-read-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 115502 }, /* node-read-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 120889 }, /* node-reference\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 120809 }, /* node-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\00010\000\000\000\000\000 */ -{ 120071 }, /* node-speculative-load\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 120490 }, /* node-speculative-load-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 120704 }, /* node-speculative-load-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 120597 }, /* node-speculative-load-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 120386 }, /* node-speculative-load-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 120276 }, /* node-speculative-load-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 120171 }, /* node-speculative-load-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119333 }, /* node-speculative-read\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119752 }, /* node-speculative-read-access\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119966 }, /* node-speculative-read-miss\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 119859 }, /* node-speculative-read-misses\000legacy cache\000Local memory prefetch misses\000legacy-cache-config=0x10206\000\00010\000\000\000\000\000 */ -{ 119648 }, /* node-speculative-read-ops\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119538 }, /* node-speculative-read-reference\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 119433 }, /* node-speculative-read-refs\000legacy cache\000Local memory prefetch accesses\000legacy-cache-config=0x206\000\00010\000\000\000\000\000 */ -{ 116028 }, /* node-store\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116391 }, /* node-store-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116577 }, /* node-store-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000 */ -{ 116484 }, /* node-store-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00000\000\000\000\000\000 */ -{ 116301 }, /* node-store-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116205 }, /* node-store-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116114 }, /* node-store-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116668 }, /* node-stores\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00000\000\000\000\000\000 */ -{ 117035 }, /* node-stores-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117223 }, /* node-stores-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000 */ -{ 117129 }, /* node-stores-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000 */ -{ 116944 }, /* node-stores-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116847 }, /* node-stores-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 116755 }, /* node-stores-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117315 }, /* node-write\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117678 }, /* node-write-access\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117864 }, /* node-write-miss\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000 */ -{ 117771 }, /* node-write-misses\000legacy cache\000Local memory write misses\000legacy-cache-config=0x10106\000\00010\000\000\000\000\000 */ -{ 117588 }, /* node-write-ops\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117492 }, /* node-write-reference\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 117401 }, /* node-write-refs\000legacy cache\000Local memory write accesses\000legacy-cache-config=0x106\000\00010\000\000\000\000\000 */ -{ 123400 }, /* ref-cycles\000legacy hardware\000Total cycles; not affected by CPU frequency scaling\000legacy-hardware-config=9\000\00000\000\000\000\000\000 */ -{ 123094 }, /* stalled-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This event is an alias of idle-cycles-backend]\000legacy-hardware-config=8\000\00000\000\000\000\000\000 */ -{ 122795 }, /* stalled-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This event is an alias of idle-cycles-frontend]\000legacy-hardware-config=7\000\00000\000\000\000\000\000 */ + /* bpc\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-conf... */ + { 111480 }, + /* bpc-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cac... */ + { 113849 }, + /* bpc-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 111564 }, + /* bpc-load-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 111939 }, + /* bpc-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 112135 }, + /* bpc-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 112035 }, + /* bpc-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 111846 }, + /* bpc-load-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 111747 }, + /* bpc-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 111653 }, + /* bpc-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cach... */ + { 112233 }, + /* bpc-loads-access\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 112612 }, + /* bpc-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-c... */ + { 112810 }, + /* bpc-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy... */ + { 112709 }, + /* bpc-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 112518 }, + /* bpc-loads-reference\000legacy cache\000Branch prediction unit read accesses\000l... */ + { 112418 }, + /* bpc-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy... */ + { 112323 }, + /* bpc-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-c... */ + { 114035 }, + /* bpc-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache... */ + { 113940 }, + /* bpc-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-... */ + { 113761 }, + /* bpc-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 112909 }, + /* bpc-read-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 113284 }, + /* bpc-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 113480 }, + /* bpc-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 113380 }, + /* bpc-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 113191 }, + /* bpc-read-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 113092 }, + /* bpc-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 112998 }, + /* bpc-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 113667 }, + /* bpc-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 113578 }, + /* bpu\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-conf... */ + { 106184 }, + /* bpu-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cac... */ + { 108553 }, + /* bpu-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 106268 }, + /* bpu-load-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 106643 }, + /* bpu-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 106839 }, + /* bpu-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 106739 }, + /* bpu-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 106550 }, + /* bpu-load-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 106451 }, + /* bpu-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 106357 }, + /* bpu-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cach... */ + { 106937 }, + /* bpu-loads-access\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 107316 }, + /* bpu-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-c... */ + { 107514 }, + /* bpu-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy... */ + { 107413 }, + /* bpu-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 107222 }, + /* bpu-loads-reference\000legacy cache\000Branch prediction unit read accesses\000l... */ + { 107122 }, + /* bpu-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy... */ + { 107027 }, + /* bpu-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-c... */ + { 108739 }, + /* bpu-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache... */ + { 108644 }, + /* bpu-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-... */ + { 108465 }, + /* bpu-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 107613 }, + /* bpu-read-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 107988 }, + /* bpu-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 108184 }, + /* bpu-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 108084 }, + /* bpu-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 107895 }, + /* bpu-read-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 107796 }, + /* bpu-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 107702 }, + /* bpu-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 108371 }, + /* bpu-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 108282 }, + /* branch\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-c... */ + { 100851 }, + /* branch-access\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 103295 }, + /* branch-instructions\000legacy hardware\000Retired branch instructions [This even... */ + { 122452 }, + /* branch-load\000legacy cache\000Branch prediction unit read accesses\000legacy-ca... */ + { 100938 }, + /* branch-load-access\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 101325 }, + /* branch-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy... */ + { 101527 }, + /* branch-load-misses\000legacy cache\000Branch prediction unit read misses\000lega... */ + { 101424 }, + /* branch-load-ops\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 101229 }, + /* branch-load-reference\000legacy cache\000Branch prediction unit read accesses\00... */ + { 101127 }, + /* branch-load-refs\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 101030 }, + /* branch-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 101628 }, + /* branch-loads-access\000legacy cache\000Branch prediction unit read accesses\000l... */ + { 102019 }, + /* branch-loads-miss\000legacy cache\000Branch prediction unit read misses\000legac... */ + { 102223 }, + /* branch-loads-misses\000legacy cache\000Branch prediction unit read misses\000leg... */ + { 102119 }, + /* branch-loads-ops\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 101922 }, + /* branch-loads-reference\000legacy cache\000Branch prediction unit read accesses\0... */ + { 101819 }, + /* branch-loads-refs\000legacy cache\000Branch prediction unit read accesses\000leg... */ + { 101721 }, + /* branch-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cach... */ + { 103389 }, + /* branch-misses\000legacy hardware\000Mispredicted branch instructions\000legacy-h... */ + { 122586 }, + /* branch-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cac... */ + { 103204 }, + /* branch-read\000legacy cache\000Branch prediction unit read accesses\000legacy-ca... */ + { 102325 }, + /* branch-read-access\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 102712 }, + /* branch-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy... */ + { 102914 }, + /* branch-read-misses\000legacy cache\000Branch prediction unit read misses\000lega... */ + { 102811 }, + /* branch-read-ops\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 102616 }, + /* branch-read-reference\000legacy cache\000Branch prediction unit read accesses\00... */ + { 102514 }, + /* branch-read-refs\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 102417 }, + /* branch-reference\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 103107 }, + /* branch-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-ca... */ + { 103015 }, + /* branches\000legacy hardware\000Retired branch instructions [This event is an ali... */ + { 122318 }, + /* branches-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 105890 }, + /* branches-load\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 103485 }, + /* branches-load-access\000legacy cache\000Branch prediction unit read accesses\000... */ + { 103880 }, + /* branches-load-miss\000legacy cache\000Branch prediction unit read misses\000lega... */ + { 104086 }, + /* branches-load-misses\000legacy cache\000Branch prediction unit read misses\000le... */ + { 103981 }, + /* branches-load-ops\000legacy cache\000Branch prediction unit read accesses\000leg... */ + { 103782 }, + /* branches-load-reference\000legacy cache\000Branch prediction unit read accesses\... */ + { 103678 }, + /* branches-load-refs\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 103579 }, + /* branches-loads\000legacy cache\000Branch prediction unit read accesses\000legacy... */ + { 104189 }, + /* branches-loads-access\000legacy cache\000Branch prediction unit read accesses\00... */ + { 104588 }, + /* branches-loads-miss\000legacy cache\000Branch prediction unit read misses\000leg... */ + { 104796 }, + /* branches-loads-misses\000legacy cache\000Branch prediction unit read misses\000l... */ + { 104690 }, + /* branches-loads-ops\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 104489 }, + /* branches-loads-reference\000legacy cache\000Branch prediction unit read accesses... */ + { 104384 }, + /* branches-loads-refs\000legacy cache\000Branch prediction unit read accesses\000l... */ + { 104284 }, + /* branches-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 106086 }, + /* branches-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 105986 }, + /* branches-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 105797 }, + /* branches-read\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 104900 }, + /* branches-read-access\000legacy cache\000Branch prediction unit read accesses\000... */ + { 105295 }, + /* branches-read-miss\000legacy cache\000Branch prediction unit read misses\000lega... */ + { 105501 }, + /* branches-read-misses\000legacy cache\000Branch prediction unit read misses\000le... */ + { 105396 }, + /* branches-read-ops\000legacy cache\000Branch prediction unit read accesses\000leg... */ + { 105197 }, + /* branches-read-reference\000legacy cache\000Branch prediction unit read accesses\... */ + { 105093 }, + /* branches-read-refs\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 104994 }, + /* branches-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 105698 }, + /* branches-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 105604 }, + /* btb\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-conf... */ + { 108832 }, + /* btb-access\000legacy cache\000Branch prediction unit read accesses\000legacy-cac... */ + { 111201 }, + /* btb-load\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 108916 }, + /* btb-load-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 109291 }, + /* btb-load-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 109487 }, + /* btb-load-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 109387 }, + /* btb-load-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 109198 }, + /* btb-load-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 109099 }, + /* btb-load-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 109005 }, + /* btb-loads\000legacy cache\000Branch prediction unit read accesses\000legacy-cach... */ + { 109585 }, + /* btb-loads-access\000legacy cache\000Branch prediction unit read accesses\000lega... */ + { 109964 }, + /* btb-loads-miss\000legacy cache\000Branch prediction unit read misses\000legacy-c... */ + { 110162 }, + /* btb-loads-misses\000legacy cache\000Branch prediction unit read misses\000legacy... */ + { 110061 }, + /* btb-loads-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 109870 }, + /* btb-loads-reference\000legacy cache\000Branch prediction unit read accesses\000l... */ + { 109770 }, + /* btb-loads-refs\000legacy cache\000Branch prediction unit read accesses\000legacy... */ + { 109675 }, + /* btb-miss\000legacy cache\000Branch prediction unit read misses\000legacy-cache-c... */ + { 111387 }, + /* btb-misses\000legacy cache\000Branch prediction unit read misses\000legacy-cache... */ + { 111292 }, + /* btb-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-cache-... */ + { 111113 }, + /* btb-read\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 110261 }, + /* btb-read-access\000legacy cache\000Branch prediction unit read accesses\000legac... */ + { 110636 }, + /* btb-read-miss\000legacy cache\000Branch prediction unit read misses\000legacy-ca... */ + { 110832 }, + /* btb-read-misses\000legacy cache\000Branch prediction unit read misses\000legacy-... */ + { 110732 }, + /* btb-read-ops\000legacy cache\000Branch prediction unit read accesses\000legacy-c... */ + { 110543 }, + /* btb-read-reference\000legacy cache\000Branch prediction unit read accesses\000le... */ + { 110444 }, + /* btb-read-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 110350 }, + /* btb-reference\000legacy cache\000Branch prediction unit read accesses\000legacy-... */ + { 111019 }, + /* btb-refs\000legacy cache\000Branch prediction unit read accesses\000legacy-cache... */ + { 110930 }, + /* bus-cycles\000legacy hardware\000Bus cycles, which can be different from total c... */ + { 122682 }, + /* cache-misses\000legacy hardware\000Cache misses. Usually this indicates Last Lev... */ + { 122075 }, + /* cache-references\000legacy hardware\000Cache accesses. Usually this indicates La... */ + { 121805 }, + /* cpu-cycles\000legacy hardware\000Total cycles. Be wary of what happens during CP... */ + { 121305 }, + /* cycles\000legacy hardware\000Total cycles. Be wary of what happens during CPU fr... */ + { 121467 }, + /* d-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\000... */ + { 78952 }, + /* d-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\... */ + { 85655 }, + /* d-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\00... */ + { 79024 }, + /* d-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-conf... */ + { 79351 }, + /* d-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0... */ + { 79523 }, + /* d-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config... */ + { 79435 }, + /* d-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 79270 }, + /* d-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-c... */ + { 79183 }, + /* d-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 79101 }, + /* d-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\0... */ + { 79609 }, + /* d-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-con... */ + { 79940 }, + /* d-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=... */ + { 80114 }, + /* d-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-confi... */ + { 80025 }, + /* d-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 79858 }, + /* d-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-... */ + { 79770 }, + /* d-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-confi... */ + { 79687 }, + /* d-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x1000... */ + { 85817 }, + /* d-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10... */ + { 85734 }, + /* d-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000... */ + { 85579 }, + /* d-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-con... */ + { 82650 }, + /* d-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-ca... */ + { 83025 }, + /* d-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-... */ + { 83217 }, + /* d-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cach... */ + { 83121 }, + /* d-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache... */ + { 82932 }, + /* d-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy... */ + { 82833 }, + /* d-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cach... */ + { 82739 }, + /* d-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-c... */ + { 83311 }, + /* d-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-... */ + { 83694 }, + /* d-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cach... */ + { 83890 }, + /* d-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-ca... */ + { 83792 }, + /* d-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cac... */ + { 83599 }, + /* d-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 83498 }, + /* d-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-ca... */ + { 83402 }, + /* d-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\00... */ + { 80201 }, + /* d-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-conf... */ + { 80528 }, + /* d-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0... */ + { 80700 }, + /* d-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config... */ + { 80612 }, + /* d-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 80447 }, + /* d-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-c... */ + { 80360 }, + /* d-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 80278 }, + /* d-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 85497 }, + /* d-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\00... */ + { 85420 }, + /* d-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-c... */ + { 84703 }, + /* d-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000l... */ + { 85110 }, + /* d-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legac... */ + { 85318 }, + /* d-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000leg... */ + { 85214 }, + /* d-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 85009 }, + /* d-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\0... */ + { 84902 }, + /* d-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000leg... */ + { 84800 }, + /* d-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-c... */ + { 83986 }, + /* d-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000l... */ + { 84393 }, + /* d-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legac... */ + { 84601 }, + /* d-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000leg... */ + { 84497 }, + /* d-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 84292 }, + /* d-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\0... */ + { 84185 }, + /* d-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000leg... */ + { 84083 }, + /* d-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x... */ + { 80786 }, + /* d-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-co... */ + { 81137 }, + /* d-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config... */ + { 81317 }, + /* d-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-conf... */ + { 81227 }, + /* d-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 81050 }, + /* d-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache... */ + { 80957 }, + /* d-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-conf... */ + { 80869 }, + /* d-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0... */ + { 81405 }, + /* d-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-c... */ + { 81760 }, + /* d-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-confi... */ + { 81942 }, + /* d-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-con... */ + { 81851 }, + /* d-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-conf... */ + { 81672 }, + /* d-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cach... */ + { 81578 }, + /* d-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-con... */ + { 81489 }, + /* d-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x... */ + { 82031 }, + /* d-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-co... */ + { 82382 }, + /* d-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config... */ + { 82562 }, + /* d-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-conf... */ + { 82472 }, + /* d-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 82295 }, + /* d-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache... */ + { 82202 }, + /* d-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-conf... */ + { 82114 }, + /* data-tlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\... */ + { 85898 }, + /* data-tlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 92823 }, + /* data-tlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3... */ + { 85973 }, + /* data-tlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-c... */ + { 86312 }, + /* data-tlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-confi... */ + { 86490 }, + /* data-tlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-con... */ + { 86399 }, + /* data-tlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-conf... */ + { 86228 }, + /* data-tlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cach... */ + { 86138 }, + /* data-tlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-con... */ + { 86053 }, + /* data-tlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 86579 }, + /* data-tlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-... */ + { 86922 }, + /* data-tlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-conf... */ + { 87102 }, + /* data-tlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-co... */ + { 87010 }, + /* data-tlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-con... */ + { 86837 }, + /* data-tlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cac... */ + { 86746 }, + /* data-tlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-co... */ + { 86660 }, + /* data-tlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x1... */ + { 92991 }, + /* data-tlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0... */ + { 92905 }, + /* data-tlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\... */ + { 92744 }, + /* data-tlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-... */ + { 89725 }, + /* data-tlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy... */ + { 90112 }, + /* data-tlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cac... */ + { 90310 }, + /* data-tlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-c... */ + { 90211 }, + /* data-tlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-ca... */ + { 90016 }, + /* data-tlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000leg... */ + { 89914 }, + /* data-tlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-c... */ + { 89817 }, + /* data-tlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cach... */ + { 90407 }, + /* data-tlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 90802 }, + /* data-tlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-c... */ + { 91004 }, + /* data-tlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy... */ + { 90903 }, + /* data-tlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-... */ + { 90704 }, + /* data-tlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000l... */ + { 90600 }, + /* data-tlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy... */ + { 90501 }, + /* data-tlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3... */ + { 87192 }, + /* data-tlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-c... */ + { 87531 }, + /* data-tlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-confi... */ + { 87709 }, + /* data-tlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-con... */ + { 87618 }, + /* data-tlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-conf... */ + { 87447 }, + /* data-tlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cach... */ + { 87357 }, + /* data-tlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-con... */ + { 87272 }, + /* data-tlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-con... */ + { 92659 }, + /* data-tlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3... */ + { 92579 }, + /* data-tlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legac... */ + { 91841 }, + /* data-tlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\0... */ + { 92260 }, + /* data-tlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000le... */ + { 92474 }, + /* data-tlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000... */ + { 92367 }, + /* data-tlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000l... */ + { 92156 }, + /* data-tlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesse... */ + { 92046 }, + /* data-tlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000... */ + { 91941 }, + /* data-tlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legac... */ + { 91103 }, + /* data-tlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\0... */ + { 91522 }, + /* data-tlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000le... */ + { 91736 }, + /* data-tlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000... */ + { 91629 }, + /* data-tlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000l... */ + { 91418 }, + /* data-tlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesse... */ + { 91308 }, + /* data-tlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000... */ + { 91203 }, + /* data-tlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config... */ + { 87798 }, + /* data-tlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache... */ + { 88161 }, + /* data-tlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-con... */ + { 88347 }, + /* data-tlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-c... */ + { 88254 }, + /* data-tlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-co... */ + { 88071 }, + /* data-tlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-ca... */ + { 87975 }, + /* data-tlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-c... */ + { 87884 }, + /* data-tlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 88438 }, + /* data-tlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cach... */ + { 88805 }, + /* data-tlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-co... */ + { 88993 }, + /* data-tlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-... */ + { 88899 }, + /* data-tlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-c... */ + { 88714 }, + /* data-tlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-c... */ + { 88617 }, + /* data-tlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-... */ + { 88525 }, + /* data-tlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config... */ + { 89085 }, + /* data-tlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache... */ + { 89448 }, + /* data-tlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-con... */ + { 89634 }, + /* data-tlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-c... */ + { 89541 }, + /* data-tlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-co... */ + { 89358 }, + /* data-tlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-ca... */ + { 89262 }, + /* data-tlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-c... */ + { 89171 }, + /* dtlb\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\0001... */ + { 72083 }, + /* dtlb-access\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\0... */ + { 78712 }, + /* dtlb-load\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000... */ + { 72154 }, + /* dtlb-load-access\000legacy cache\000Data TLB read accesses\000legacy-cache-confi... */ + { 72477 }, + /* dtlb-load-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x... */ + { 72647 }, + /* dtlb-load-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=... */ + { 72560 }, + /* dtlb-load-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3... */ + { 72397 }, + /* dtlb-load-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-co... */ + { 72311 }, + /* dtlb-load-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 72230 }, + /* dtlb-loads\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\00... */ + { 72732 }, + /* dtlb-loads-access\000legacy cache\000Data TLB read accesses\000legacy-cache-conf... */ + { 73059 }, + /* dtlb-loads-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0... */ + { 73231 }, + /* dtlb-loads-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config... */ + { 73143 }, + /* dtlb-loads-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 72978 }, + /* dtlb-loads-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-c... */ + { 72891 }, + /* dtlb-loads-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config... */ + { 72809 }, + /* dtlb-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x10003... */ + { 78872 }, + /* dtlb-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x100... */ + { 78790 }, + /* dtlb-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000\... */ + { 78637 }, + /* dtlb-prefetch\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-conf... */ + { 75738 }, + /* dtlb-prefetch-access\000legacy cache\000Data TLB prefetch accesses\000legacy-cac... */ + { 76109 }, + /* dtlb-prefetch-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache-c... */ + { 76299 }, + /* dtlb-prefetch-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cache... */ + { 76204 }, + /* dtlb-prefetch-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-... */ + { 76017 }, + /* dtlb-prefetch-reference\000legacy cache\000Data TLB prefetch accesses\000legacy-... */ + { 75919 }, + /* dtlb-prefetch-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cache... */ + { 75826 }, + /* dtlb-prefetches\000legacy cache\000Data TLB prefetch accesses\000legacy-cache-co... */ + { 76392 }, + /* dtlb-prefetches-access\000legacy cache\000Data TLB prefetch accesses\000legacy-c... */ + { 76771 }, + /* dtlb-prefetches-miss\000legacy cache\000Data TLB prefetch misses\000legacy-cache... */ + { 76965 }, + /* dtlb-prefetches-misses\000legacy cache\000Data TLB prefetch misses\000legacy-cac... */ + { 76868 }, + /* dtlb-prefetches-ops\000legacy cache\000Data TLB prefetch accesses\000legacy-cach... */ + { 76677 }, + /* dtlb-prefetches-reference\000legacy cache\000Data TLB prefetch accesses\000legac... */ + { 76577 }, + /* dtlb-prefetches-refs\000legacy cache\000Data TLB prefetch accesses\000legacy-cac... */ + { 76482 }, + /* dtlb-read\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000... */ + { 73317 }, + /* dtlb-read-access\000legacy cache\000Data TLB read accesses\000legacy-cache-confi... */ + { 73640 }, + /* dtlb-read-miss\000legacy cache\000Data TLB read misses\000legacy-cache-config=0x... */ + { 73810 }, + /* dtlb-read-misses\000legacy cache\000Data TLB read misses\000legacy-cache-config=... */ + { 73723 }, + /* dtlb-read-ops\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3... */ + { 73560 }, + /* dtlb-read-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-co... */ + { 73474 }, + /* dtlb-read-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 73393 }, + /* dtlb-reference\000legacy cache\000Data TLB read accesses\000legacy-cache-config=... */ + { 78556 }, + /* dtlb-refs\000legacy cache\000Data TLB read accesses\000legacy-cache-config=3\000... */ + { 78480 }, + /* dtlb-speculative-load\000legacy cache\000Data TLB prefetch accesses\000legacy-ca... */ + { 77770 }, + /* dtlb-speculative-load-access\000legacy cache\000Data TLB prefetch accesses\000le... */ + { 78173 }, + /* dtlb-speculative-load-miss\000legacy cache\000Data TLB prefetch misses\000legacy... */ + { 78379 }, + /* dtlb-speculative-load-misses\000legacy cache\000Data TLB prefetch misses\000lega... */ + { 78276 }, + /* dtlb-speculative-load-ops\000legacy cache\000Data TLB prefetch accesses\000legac... */ + { 78073 }, + /* dtlb-speculative-load-reference\000legacy cache\000Data TLB prefetch accesses\00... */ + { 77967 }, + /* dtlb-speculative-load-refs\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 77866 }, + /* dtlb-speculative-read\000legacy cache\000Data TLB prefetch accesses\000legacy-ca... */ + { 77060 }, + /* dtlb-speculative-read-access\000legacy cache\000Data TLB prefetch accesses\000le... */ + { 77463 }, + /* dtlb-speculative-read-miss\000legacy cache\000Data TLB prefetch misses\000legacy... */ + { 77669 }, + /* dtlb-speculative-read-misses\000legacy cache\000Data TLB prefetch misses\000lega... */ + { 77566 }, + /* dtlb-speculative-read-ops\000legacy cache\000Data TLB prefetch accesses\000legac... */ + { 77363 }, + /* dtlb-speculative-read-reference\000legacy cache\000Data TLB prefetch accesses\00... */ + { 77257 }, + /* dtlb-speculative-read-refs\000legacy cache\000Data TLB prefetch accesses\000lega... */ + { 77156 }, + /* dtlb-store\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x1... */ + { 73895 }, + /* dtlb-store-access\000legacy cache\000Data TLB write accesses\000legacy-cache-con... */ + { 74242 }, + /* dtlb-store-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=... */ + { 74420 }, + /* dtlb-store-misses\000legacy cache\000Data TLB write misses\000legacy-cache-confi... */ + { 74331 }, + /* dtlb-store-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config... */ + { 74156 }, + /* dtlb-store-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-... */ + { 74064 }, + /* dtlb-store-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 73977 }, + /* dtlb-stores\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x... */ + { 74507 }, + /* dtlb-stores-access\000legacy cache\000Data TLB write accesses\000legacy-cache-co... */ + { 74858 }, + /* dtlb-stores-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config... */ + { 75038 }, + /* dtlb-stores-misses\000legacy cache\000Data TLB write misses\000legacy-cache-conf... */ + { 74948 }, + /* dtlb-stores-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 74771 }, + /* dtlb-stores-reference\000legacy cache\000Data TLB write accesses\000legacy-cache... */ + { 74678 }, + /* dtlb-stores-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-conf... */ + { 74590 }, + /* dtlb-write\000legacy cache\000Data TLB write accesses\000legacy-cache-config=0x1... */ + { 75126 }, + /* dtlb-write-access\000legacy cache\000Data TLB write accesses\000legacy-cache-con... */ + { 75473 }, + /* dtlb-write-miss\000legacy cache\000Data TLB write misses\000legacy-cache-config=... */ + { 75651 }, + /* dtlb-write-misses\000legacy cache\000Data TLB write misses\000legacy-cache-confi... */ + { 75562 }, + /* dtlb-write-ops\000legacy cache\000Data TLB write accesses\000legacy-cache-config... */ + { 75387 }, + /* dtlb-write-reference\000legacy cache\000Data TLB write accesses\000legacy-cache-... */ + { 75295 }, + /* dtlb-write-refs\000legacy cache\000Data TLB write accesses\000legacy-cache-confi... */ + { 75208 }, + /* i-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\... */ + { 95555 }, + /* i-tlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-co... */ + { 97799 }, + /* i-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-conf... */ + { 95634 }, + /* i-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cac... */ + { 95989 }, + /* i-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-c... */ + { 96175 }, + /* i-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache... */ + { 96080 }, + /* i-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 95901 }, + /* i-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 95807 }, + /* i-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 95718 }, + /* i-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-con... */ + { 96268 }, + /* i-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-ca... */ + { 96627 }, + /* i-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-... */ + { 96815 }, + /* i-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cach... */ + { 96719 }, + /* i-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 96538 }, + /* i-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy... */ + { 96443 }, + /* i-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cach... */ + { 96353 }, + /* i-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config... */ + { 97975 }, + /* i-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-conf... */ + { 97885 }, + /* i-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-confi... */ + { 97716 }, + /* i-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-conf... */ + { 96909 }, + /* i-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cac... */ + { 97264 }, + /* i-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-c... */ + { 97450 }, + /* i-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache... */ + { 97355 }, + /* i-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 97176 }, + /* i-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 97082 }, + /* i-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 96993 }, + /* i-tlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 97627 }, + /* i-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-conf... */ + { 97543 }, + /* idle-cycles-backend\000legacy hardware\000Stalled cycles during retirement [This... */ + { 123247 }, + /* idle-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This eve... */ + { 122945 }, + /* instruction-tlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 98063 }, + /* instruction-tlb-access\000legacy cache\000Instruction TLB read accesses\000legac... */ + { 100557 }, + /* instruction-tlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 98152 }, + /* instruction-tlb-load-access\000legacy cache\000Instruction TLB read accesses\000... */ + { 98547 }, + /* instruction-tlb-load-miss\000legacy cache\000Instruction TLB read misses\000lega... */ + { 98753 }, + /* instruction-tlb-load-misses\000legacy cache\000Instruction TLB read misses\000le... */ + { 98648 }, + /* instruction-tlb-load-ops\000legacy cache\000Instruction TLB read accesses\000leg... */ + { 98449 }, + /* instruction-tlb-load-reference\000legacy cache\000Instruction TLB read accesses\... */ + { 98345 }, + /* instruction-tlb-load-refs\000legacy cache\000Instruction TLB read accesses\000le... */ + { 98246 }, + /* instruction-tlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy... */ + { 98856 }, + /* instruction-tlb-loads-access\000legacy cache\000Instruction TLB read accesses\00... */ + { 99255 }, + /* instruction-tlb-loads-miss\000legacy cache\000Instruction TLB read misses\000leg... */ + { 99463 }, + /* instruction-tlb-loads-misses\000legacy cache\000Instruction TLB read misses\000l... */ + { 99357 }, + /* instruction-tlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000le... */ + { 99156 }, + /* instruction-tlb-loads-reference\000legacy cache\000Instruction TLB read accesses... */ + { 99051 }, + /* instruction-tlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000l... */ + { 98951 }, + /* instruction-tlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-ca... */ + { 100753 }, + /* instruction-tlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-... */ + { 100653 }, + /* instruction-tlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-c... */ + { 100464 }, + /* instruction-tlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 99567 }, + /* instruction-tlb-read-access\000legacy cache\000Instruction TLB read accesses\000... */ + { 99962 }, + /* instruction-tlb-read-miss\000legacy cache\000Instruction TLB read misses\000lega... */ + { 100168 }, + /* instruction-tlb-read-misses\000legacy cache\000Instruction TLB read misses\000le... */ + { 100063 }, + /* instruction-tlb-read-ops\000legacy cache\000Instruction TLB read accesses\000leg... */ + { 99864 }, + /* instruction-tlb-read-reference\000legacy cache\000Instruction TLB read accesses\... */ + { 99760 }, + /* instruction-tlb-read-refs\000legacy cache\000Instruction TLB read accesses\000le... */ + { 99661 }, + /* instruction-tlb-reference\000legacy cache\000Instruction TLB read accesses\000le... */ + { 100365 }, + /* instruction-tlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 100271 }, + /* instructions\000legacy hardware\000Retired instructions. Be careful, these can b... */ + { 121629 }, + /* itlb\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config=4\0... */ + { 93075 }, + /* itlb-access\000legacy cache\000Instruction TLB read accesses\000legacy-cache-con... */ + { 95294 }, + /* itlb-load\000legacy cache\000Instruction TLB read accesses\000legacy-cache-confi... */ + { 93153 }, + /* itlb-load-access\000legacy cache\000Instruction TLB read accesses\000legacy-cach... */ + { 93504 }, + /* itlb-load-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-co... */ + { 93688 }, + /* itlb-load-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-... */ + { 93594 }, + /* itlb-load-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-c... */ + { 93417 }, + /* itlb-load-reference\000legacy cache\000Instruction TLB read accesses\000legacy-c... */ + { 93324 }, + /* itlb-load-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 93236 }, + /* itlb-loads\000legacy cache\000Instruction TLB read accesses\000legacy-cache-conf... */ + { 93780 }, + /* itlb-loads-access\000legacy cache\000Instruction TLB read accesses\000legacy-cac... */ + { 94135 }, + /* itlb-loads-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-c... */ + { 94321 }, + /* itlb-loads-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache... */ + { 94226 }, + /* itlb-loads-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 94047 }, + /* itlb-loads-reference\000legacy cache\000Instruction TLB read accesses\000legacy-... */ + { 93953 }, + /* itlb-loads-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache... */ + { 93864 }, + /* itlb-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-config=... */ + { 95468 }, + /* itlb-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-confi... */ + { 95379 }, + /* itlb-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-config... */ + { 95212 }, + /* itlb-read\000legacy cache\000Instruction TLB read accesses\000legacy-cache-confi... */ + { 94414 }, + /* itlb-read-access\000legacy cache\000Instruction TLB read accesses\000legacy-cach... */ + { 94765 }, + /* itlb-read-miss\000legacy cache\000Instruction TLB read misses\000legacy-cache-co... */ + { 94949 }, + /* itlb-read-misses\000legacy cache\000Instruction TLB read misses\000legacy-cache-... */ + { 94855 }, + /* itlb-read-ops\000legacy cache\000Instruction TLB read accesses\000legacy-cache-c... */ + { 94678 }, + /* itlb-read-reference\000legacy cache\000Instruction TLB read accesses\000legacy-c... */ + { 94585 }, + /* itlb-read-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 94497 }, + /* itlb-reference\000legacy cache\000Instruction TLB read accesses\000legacy-cache-... */ + { 95124 }, + /* itlb-refs\000legacy cache\000Instruction TLB read accesses\000legacy-cache-confi... */ + { 95041 }, + /* l1-d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=... */ + { 8037 }, + /* l1-d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-... */ + { 15406 }, + /* l1-d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-co... */ + { 8118 }, + /* l1-d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 8481 }, + /* l1-d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache... */ + { 8671 }, + /* l1-d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cac... */ + { 8574 }, + /* l1-d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 8391 }, + /* l1-d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 8295 }, + /* l1-d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 8204 }, + /* l1-d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-c... */ + { 8766 }, + /* l1-d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-... */ + { 9133 }, + /* l1-d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cach... */ + { 9325 }, + /* l1-d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-ca... */ + { 9227 }, + /* l1-d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 9042 }, + /* l1-d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000lega... */ + { 8945 }, + /* l1-d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-ca... */ + { 8853 }, + /* l1-d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-conf... */ + { 15586 }, + /* l1-d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-co... */ + { 15494 }, + /* l1-d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-con... */ + { 15321 }, + /* l1-d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-... */ + { 12122 }, + /* l1-d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 12533 }, + /* l1-d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000lega... */ + { 12743 }, + /* l1-d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000le... */ + { 12638 }, + /* l1-d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000leg... */ + { 12431 }, + /* l1-d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\... */ + { 12323 }, + /* l1-d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000le... */ + { 12220 }, + /* l1-d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legac... */ + { 12846 }, + /* l1-d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\0... */ + { 13265 }, + /* l1-d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000le... */ + { 13479 }, + /* l1-d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000... */ + { 13372 }, + /* l1-d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000l... */ + { 13161 }, + /* l1-d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 13051 }, + /* l1-d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 12946 }, + /* l1-d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-co... */ + { 9421 }, + /* l1-d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 9784 }, + /* l1-d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache... */ + { 9974 }, + /* l1-d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cac... */ + { 9877 }, + /* l1-d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 9694 }, + /* l1-d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 9598 }, + /* l1-d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 9507 }, + /* l1-d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 15230 }, + /* l1-d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-co... */ + { 15144 }, + /* l1-d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\00... */ + { 14364 }, + /* l1-d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch acce... */ + { 14807 }, + /* l1-d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses... */ + { 15033 }, + /* l1-d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch miss... */ + { 14920 }, + /* l1-d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 14697 }, + /* l1-d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch a... */ + { 14581 }, + /* l1-d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch access... */ + { 14470 }, + /* l1-d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\00... */ + { 13584 }, + /* l1-d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch acce... */ + { 14027 }, + /* l1-d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses... */ + { 14253 }, + /* l1-d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch miss... */ + { 14140 }, + /* l1-d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 13917 }, + /* l1-d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch a... */ + { 13801 }, + /* l1-d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch access... */ + { 13690 }, + /* l1-d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-... */ + { 10069 }, + /* l1-d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy... */ + { 10456 }, + /* l1-d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cac... */ + { 10654 }, + /* l1-d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-c... */ + { 10555 }, + /* l1-d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 10360 }, + /* l1-d-store-reference\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 10258 }, + /* l1-d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 10161 }, + /* l1-d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache... */ + { 10751 }, + /* l1-d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legac... */ + { 11142 }, + /* l1-d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-ca... */ + { 11342 }, + /* l1-d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-... */ + { 11242 }, + /* l1-d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 11045 }, + /* l1-d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000le... */ + { 10942 }, + /* l1-d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-... */ + { 10844 }, + /* l1-d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-... */ + { 11440 }, + /* l1-d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy... */ + { 11827 }, + /* l1-d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cac... */ + { 12025 }, + /* l1-d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-c... */ + { 11926 }, + /* l1-d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 11731 }, + /* l1-d-write-reference\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 11629 }, + /* l1-d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 11532 }, + /* l1-data\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-conf... */ + { 23238 }, + /* l1-data-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 30829 }, + /* l1-data-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache... */ + { 23322 }, + /* l1-data-load-access\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 23697 }, + /* l1-data-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-ca... */ + { 23893 }, + /* l1-data-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-... */ + { 23793 }, + /* l1-data-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 23604 }, + /* l1-data-load-reference\000legacy cache\000Level 1 data cache read accesses\000le... */ + { 23505 }, + /* l1-data-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-... */ + { 23411 }, + /* l1-data-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 23991 }, + /* l1-data-loads-access\000legacy cache\000Level 1 data cache read accesses\000lega... */ + { 24370 }, + /* l1-data-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-c... */ + { 24568 }, + /* l1-data-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy... */ + { 24467 }, + /* l1-data-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-... */ + { 24276 }, + /* l1-data-loads-reference\000legacy cache\000Level 1 data cache read accesses\000l... */ + { 24176 }, + /* l1-data-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy... */ + { 24081 }, + /* l1-data-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-c... */ + { 31015 }, + /* l1-data-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache... */ + { 30920 }, + /* l1-data-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-... */ + { 30741 }, + /* l1-data-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000lega... */ + { 27452 }, + /* l1-data-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\... */ + { 27875 }, + /* l1-data-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000l... */ + { 28091 }, + /* l1-data-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\00... */ + { 27983 }, + /* l1-data-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 27770 }, + /* l1-data-prefetch-reference\000legacy cache\000Level 1 data cache prefetch access... */ + { 27659 }, + /* l1-data-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\00... */ + { 27553 }, + /* l1-data-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000le... */ + { 28197 }, + /* l1-data-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 28628 }, + /* l1-data-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\00... */ + { 28848 }, + /* l1-data-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\... */ + { 28738 }, + /* l1-data-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\0... */ + { 28521 }, + /* l1-data-prefetches-reference\000legacy cache\000Level 1 data cache prefetch acce... */ + { 28408 }, + /* l1-data-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\... */ + { 28300 }, + /* l1-data-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache... */ + { 24667 }, + /* l1-data-read-access\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 25042 }, + /* l1-data-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-ca... */ + { 25238 }, + /* l1-data-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-... */ + { 25138 }, + /* l1-data-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 24949 }, + /* l1-data-read-reference\000legacy cache\000Level 1 data cache read accesses\000le... */ + { 24850 }, + /* l1-data-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-... */ + { 24756 }, + /* l1-data-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-... */ + { 30647 }, + /* l1-data-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache... */ + { 30558 }, + /* l1-data-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 29757 }, + /* l1-data-speculative-load-access\000legacy cache\000Level 1 data cache prefetch a... */ + { 30212 }, + /* l1-data-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch mis... */ + { 30444 }, + /* l1-data-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch m... */ + { 30328 }, + /* l1-data-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch acce... */ + { 30099 }, + /* l1-data-speculative-load-reference\000legacy cache\000Level 1 data cache prefetc... */ + { 29980 }, + /* l1-data-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch acc... */ + { 29866 }, + /* l1-data-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 28956 }, + /* l1-data-speculative-read-access\000legacy cache\000Level 1 data cache prefetch a... */ + { 29411 }, + /* l1-data-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch mis... */ + { 29643 }, + /* l1-data-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch m... */ + { 29527 }, + /* l1-data-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch acce... */ + { 29298 }, + /* l1-data-speculative-read-reference\000legacy cache\000Level 1 data cache prefetc... */ + { 29179 }, + /* l1-data-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch acc... */ + { 29065 }, + /* l1-data-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cac... */ + { 25336 }, + /* l1-data-store-access\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 25735 }, + /* l1-data-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-... */ + { 25939 }, + /* l1-data-store-misses\000legacy cache\000Level 1 data cache write misses\000legac... */ + { 25837 }, + /* l1-data-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy... */ + { 25636 }, + /* l1-data-store-reference\000legacy cache\000Level 1 data cache write accesses\000... */ + { 25531 }, + /* l1-data-store-refs\000legacy cache\000Level 1 data cache write accesses\000legac... */ + { 25431 }, + /* l1-data-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 26039 }, + /* l1-data-stores-access\000legacy cache\000Level 1 data cache write accesses\000le... */ + { 26442 }, + /* l1-data-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy... */ + { 26648 }, + /* l1-data-stores-misses\000legacy cache\000Level 1 data cache write misses\000lega... */ + { 26545 }, + /* l1-data-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legac... */ + { 26342 }, + /* l1-data-stores-reference\000legacy cache\000Level 1 data cache write accesses\00... */ + { 26236 }, + /* l1-data-stores-refs\000legacy cache\000Level 1 data cache write accesses\000lega... */ + { 26135 }, + /* l1-data-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cac... */ + { 26749 }, + /* l1-data-write-access\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 27148 }, + /* l1-data-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-... */ + { 27352 }, + /* l1-data-write-misses\000legacy cache\000Level 1 data cache write misses\000legac... */ + { 27250 }, + /* l1-data-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy... */ + { 27049 }, + /* l1-data-write-reference\000legacy cache\000Level 1 data cache write accesses\000... */ + { 26944 }, + /* l1-data-write-refs\000legacy cache\000Level 1 data cache write accesses\000legac... */ + { 26844 }, + /* l1-dcache\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-co... */ + { 13 }, + /* l1-dcache-access\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 7752 }, + /* l1-dcache-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 99 }, + /* l1-dcache-load-access\000legacy cache\000Level 1 data cache read accesses\000leg... */ + { 482 }, + /* l1-dcache-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-... */ + { 682 }, + /* l1-dcache-load-misses\000legacy cache\000Level 1 data cache read misses\000legac... */ + { 580 }, + /* l1-dcache-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy... */ + { 387 }, + /* l1-dcache-load-reference\000legacy cache\000Level 1 data cache read accesses\000... */ + { 286 }, + /* l1-dcache-load-refs\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 190 }, + /* l1-dcache-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-ca... */ + { 782 }, + /* l1-dcache-loads-access\000legacy cache\000Level 1 data cache read accesses\000le... */ + { 1169 }, + /* l1-dcache-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy... */ + { 1371 }, + /* l1-dcache-loads-misses\000legacy cache\000Level 1 data cache read misses\000lega... */ + { 1268 }, + /* l1-dcache-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 1073 }, + /* l1-dcache-loads-reference\000legacy cache\000Level 1 data cache read accesses\00... */ + { 971 }, + /* l1-dcache-loads-refs\000legacy cache\000Level 1 data cache read accesses\000lega... */ + { 874 }, + /* l1-dcache-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache... */ + { 7942 }, + /* l1-dcache-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cac... */ + { 7845 }, + /* l1-dcache-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 7662 }, + /* l1-dcache-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000le... */ + { 4313 }, + /* l1-dcache-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 4744 }, + /* l1-dcache-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\00... */ + { 4964 }, + /* l1-dcache-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\... */ + { 4854 }, + /* l1-dcache-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\0... */ + { 4637 }, + /* l1-dcache-prefetch-reference\000legacy cache\000Level 1 data cache prefetch acce... */ + { 4524 }, + /* l1-dcache-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\... */ + { 4416 }, + /* l1-dcache-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 5072 }, + /* l1-dcache-prefetches-access\000legacy cache\000Level 1 data cache prefetch acces... */ + { 5511 }, + /* l1-dcache-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\... */ + { 5735 }, + /* l1-dcache-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misse... */ + { 5623 }, + /* l1-dcache-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 5402 }, + /* l1-dcache-prefetches-reference\000legacy cache\000Level 1 data cache prefetch ac... */ + { 5287 }, + /* l1-dcache-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 5177 }, + /* l1-dcache-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 1472 }, + /* l1-dcache-read-access\000legacy cache\000Level 1 data cache read accesses\000leg... */ + { 1855 }, + /* l1-dcache-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-... */ + { 2055 }, + /* l1-dcache-read-misses\000legacy cache\000Level 1 data cache read misses\000legac... */ + { 1953 }, + /* l1-dcache-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy... */ + { 1760 }, + /* l1-dcache-read-reference\000legacy cache\000Level 1 data cache read accesses\000... */ + { 1659 }, + /* l1-dcache-read-refs\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 1563 }, + /* l1-dcache-reference\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 7566 }, + /* l1-dcache-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 7475 }, + /* l1-dcache-speculative-load\000legacy cache\000Level 1 data cache prefetch access... */ + { 6660 }, + /* l1-dcache-speculative-load-access\000legacy cache\000Level 1 data cache prefetch... */ + { 7123 }, + /* l1-dcache-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch m... */ + { 7359 }, + /* l1-dcache-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch... */ + { 7241 }, + /* l1-dcache-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch ac... */ + { 7008 }, + /* l1-dcache-speculative-load-reference\000legacy cache\000Level 1 data cache prefe... */ + { 6887 }, + /* l1-dcache-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch a... */ + { 6771 }, + /* l1-dcache-speculative-read\000legacy cache\000Level 1 data cache prefetch access... */ + { 5845 }, + /* l1-dcache-speculative-read-access\000legacy cache\000Level 1 data cache prefetch... */ + { 6308 }, + /* l1-dcache-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch m... */ + { 6544 }, + /* l1-dcache-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch... */ + { 6426 }, + /* l1-dcache-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch ac... */ + { 6193 }, + /* l1-dcache-speculative-read-reference\000legacy cache\000Level 1 data cache prefe... */ + { 6072 }, + /* l1-dcache-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch a... */ + { 5956 }, + /* l1-dcache-store\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 2155 }, + /* l1-dcache-store-access\000legacy cache\000Level 1 data cache write accesses\000l... */ + { 2562 }, + /* l1-dcache-store-miss\000legacy cache\000Level 1 data cache write misses\000legac... */ + { 2770 }, + /* l1-dcache-store-misses\000legacy cache\000Level 1 data cache write misses\000leg... */ + { 2666 }, + /* l1-dcache-store-ops\000legacy cache\000Level 1 data cache write accesses\000lega... */ + { 2461 }, + /* l1-dcache-store-reference\000legacy cache\000Level 1 data cache write accesses\0... */ + { 2354 }, + /* l1-dcache-store-refs\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 2252 }, + /* l1-dcache-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-... */ + { 2872 }, + /* l1-dcache-stores-access\000legacy cache\000Level 1 data cache write accesses\000... */ + { 3283 }, + /* l1-dcache-stores-miss\000legacy cache\000Level 1 data cache write misses\000lega... */ + { 3493 }, + /* l1-dcache-stores-misses\000legacy cache\000Level 1 data cache write misses\000le... */ + { 3388 }, + /* l1-dcache-stores-ops\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 3181 }, + /* l1-dcache-stores-reference\000legacy cache\000Level 1 data cache write accesses\... */ + { 3073 }, + /* l1-dcache-stores-refs\000legacy cache\000Level 1 data cache write accesses\000le... */ + { 2970 }, + /* l1-dcache-write\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 3596 }, + /* l1-dcache-write-access\000legacy cache\000Level 1 data cache write accesses\000l... */ + { 4003 }, + /* l1-dcache-write-miss\000legacy cache\000Level 1 data cache write misses\000legac... */ + { 4211 }, + /* l1-dcache-write-misses\000legacy cache\000Level 1 data cache write misses\000leg... */ + { 4107 }, + /* l1-dcache-write-ops\000legacy cache\000Level 1 data cache write accesses\000lega... */ + { 3902 }, + /* l1-dcache-write-reference\000legacy cache\000Level 1 data cache write accesses\0... */ + { 3795 }, + /* l1-dcache-write-refs\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 3693 }, + /* l1-i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-... */ + { 37366 }, + /* l1-i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy... */ + { 43053 }, + /* l1-i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-c... */ + { 37454 }, + /* l1-i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000l... */ + { 37845 }, + /* l1-i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legac... */ + { 38049 }, + /* l1-i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000leg... */ + { 37945 }, + /* l1-i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 37748 }, + /* l1-i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 37645 }, + /* l1-i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 37547 }, + /* l1-i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-... */ + { 38151 }, + /* l1-i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000... */ + { 38546 }, + /* l1-i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000lega... */ + { 38752 }, + /* l1-i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000le... */ + { 38647 }, + /* l1-i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 38448 }, + /* l1-i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\... */ + { 38344 }, + /* l1-i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000le... */ + { 38245 }, + /* l1-i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cac... */ + { 43247 }, + /* l1-i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-c... */ + { 43148 }, + /* l1-i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-ca... */ + { 42961 }, + /* l1-i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000... */ + { 39552 }, + /* l1-i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch acces... */ + { 39991 }, + /* l1-i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\... */ + { 40215 }, + /* l1-i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misse... */ + { 40103 }, + /* l1-i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses... */ + { 39882 }, + /* l1-i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch ac... */ + { 39767 }, + /* l1-i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesse... */ + { 39657 }, + /* l1-i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\0... */ + { 40325 }, + /* l1-i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch acc... */ + { 40772 }, + /* l1-i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misse... */ + { 41000 }, + /* l1-i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch mis... */ + { 40886 }, + /* l1-i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch access... */ + { 40661 }, + /* l1-i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 40544 }, + /* l1-i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch acces... */ + { 40432 }, + /* l1-i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-c... */ + { 38855 }, + /* l1-i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000l... */ + { 39246 }, + /* l1-i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legac... */ + { 39450 }, + /* l1-i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000leg... */ + { 39346 }, + /* l1-i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 39149 }, + /* l1-i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 39046 }, + /* l1-i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 38948 }, + /* l1-i-reference\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 42863 }, + /* l1-i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-c... */ + { 42770 }, + /* l1-i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch acce... */ + { 41941 }, + /* l1-i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefet... */ + { 42412 }, + /* l1-i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch... */ + { 42652 }, + /* l1-i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefet... */ + { 42532 }, + /* l1-i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 42295 }, + /* l1-i-speculative-load-reference\000legacy cache\000Level 1 instruction cache pre... */ + { 42172 }, + /* l1-i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch... */ + { 42054 }, + /* l1-i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch acce... */ + { 41112 }, + /* l1-i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefet... */ + { 41583 }, + /* l1-i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch... */ + { 41823 }, + /* l1-i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefet... */ + { 41703 }, + /* l1-i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 41466 }, + /* l1-i-speculative-read-reference\000legacy cache\000Level 1 instruction cache pre... */ + { 41343 }, + /* l1-i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch... */ + { 41225 }, + /* l1-icache\000legacy cache\000Level 1 instruction cache read accesses\000legacy-c... */ + { 31108 }, + /* l1-icache-access\000legacy cache\000Level 1 instruction cache read accesses\000l... */ + { 37060 }, + /* l1-icache-load\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 31201 }, + /* l1-icache-load-access\000legacy cache\000Level 1 instruction cache read accesses... */ + { 31612 }, + /* l1-icache-load-miss\000legacy cache\000Level 1 instruction cache read misses\000... */ + { 31826 }, + /* l1-icache-load-misses\000legacy cache\000Level 1 instruction cache read misses\0... */ + { 31717 }, + /* l1-icache-load-ops\000legacy cache\000Level 1 instruction cache read accesses\00... */ + { 31510 }, + /* l1-icache-load-reference\000legacy cache\000Level 1 instruction cache read acces... */ + { 31402 }, + /* l1-icache-load-refs\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 31299 }, + /* l1-icache-loads\000legacy cache\000Level 1 instruction cache read accesses\000le... */ + { 31933 }, + /* l1-icache-loads-access\000legacy cache\000Level 1 instruction cache read accesse... */ + { 32348 }, + /* l1-icache-loads-miss\000legacy cache\000Level 1 instruction cache read misses\00... */ + { 32564 }, + /* l1-icache-loads-misses\000legacy cache\000Level 1 instruction cache read misses\... */ + { 32454 }, + /* l1-icache-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 32245 }, + /* l1-icache-loads-reference\000legacy cache\000Level 1 instruction cache read acce... */ + { 32136 }, + /* l1-icache-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\... */ + { 32032 }, + /* l1-icache-miss\000legacy cache\000Level 1 instruction cache read misses\000legac... */ + { 37264 }, + /* l1-icache-misses\000legacy cache\000Level 1 instruction cache read misses\000leg... */ + { 37160 }, + /* l1-icache-ops\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 36963 }, + /* l1-icache-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesse... */ + { 33404 }, + /* l1-icache-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 33863 }, + /* l1-icache-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch mi... */ + { 34097 }, + /* l1-icache-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 33980 }, + /* l1-icache-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch acc... */ + { 33749 }, + /* l1-icache-prefetch-reference\000legacy cache\000Level 1 instruction cache prefet... */ + { 33629 }, + /* l1-icache-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch ac... */ + { 33514 }, + /* l1-icache-prefetches\000legacy cache\000Level 1 instruction cache prefetch acces... */ + { 34212 }, + /* l1-icache-prefetches-access\000legacy cache\000Level 1 instruction cache prefetc... */ + { 34679 }, + /* l1-icache-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 34917 }, + /* l1-icache-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetc... */ + { 34798 }, + /* l1-icache-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch a... */ + { 34563 }, + /* l1-icache-prefetches-reference\000legacy cache\000Level 1 instruction cache pref... */ + { 34441 }, + /* l1-icache-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 34324 }, + /* l1-icache-read\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 32672 }, + /* l1-icache-read-access\000legacy cache\000Level 1 instruction cache read accesses... */ + { 33083 }, + /* l1-icache-read-miss\000legacy cache\000Level 1 instruction cache read misses\000... */ + { 33297 }, + /* l1-icache-read-misses\000legacy cache\000Level 1 instruction cache read misses\0... */ + { 33188 }, + /* l1-icache-read-ops\000legacy cache\000Level 1 instruction cache read accesses\00... */ + { 32981 }, + /* l1-icache-read-reference\000legacy cache\000Level 1 instruction cache read acces... */ + { 32873 }, + /* l1-icache-read-refs\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 32770 }, + /* l1-icache-reference\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 36860 }, + /* l1-icache-refs\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 36762 }, + /* l1-icache-speculative-load\000legacy cache\000Level 1 instruction cache prefetch... */ + { 35898 }, + /* l1-icache-speculative-load-access\000legacy cache\000Level 1 instruction cache p... */ + { 36389 }, + /* l1-icache-speculative-load-miss\000legacy cache\000Level 1 instruction cache pre... */ + { 36639 }, + /* l1-icache-speculative-load-misses\000legacy cache\000Level 1 instruction cache p... */ + { 36514 }, + /* l1-icache-speculative-load-ops\000legacy cache\000Level 1 instruction cache pref... */ + { 36267 }, + /* l1-icache-speculative-load-reference\000legacy cache\000Level 1 instruction cach... */ + { 36139 }, + /* l1-icache-speculative-load-refs\000legacy cache\000Level 1 instruction cache pre... */ + { 36016 }, + /* l1-icache-speculative-read\000legacy cache\000Level 1 instruction cache prefetch... */ + { 35034 }, + /* l1-icache-speculative-read-access\000legacy cache\000Level 1 instruction cache p... */ + { 35525 }, + /* l1-icache-speculative-read-miss\000legacy cache\000Level 1 instruction cache pre... */ + { 35775 }, + /* l1-icache-speculative-read-misses\000legacy cache\000Level 1 instruction cache p... */ + { 35650 }, + /* l1-icache-speculative-read-ops\000legacy cache\000Level 1 instruction cache pref... */ + { 35403 }, + /* l1-icache-speculative-read-reference\000legacy cache\000Level 1 instruction cach... */ + { 35275 }, + /* l1-icache-speculative-read-refs\000legacy cache\000Level 1 instruction cache pre... */ + { 35152 }, + /* l1-instruction\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 49266 }, + /* l1-instruction-access\000legacy cache\000Level 1 instruction cache read accesses... */ + { 55483 }, + /* l1-instruction-load\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 49364 }, + /* l1-instruction-load-access\000legacy cache\000Level 1 instruction cache read acc... */ + { 49795 }, + /* l1-instruction-load-miss\000legacy cache\000Level 1 instruction cache read misse... */ + { 50019 }, + /* l1-instruction-load-misses\000legacy cache\000Level 1 instruction cache read mis... */ + { 49905 }, + /* l1-instruction-load-ops\000legacy cache\000Level 1 instruction cache read access... */ + { 49688 }, + /* l1-instruction-load-reference\000legacy cache\000Level 1 instruction cache read ... */ + { 49575 }, + /* l1-instruction-load-refs\000legacy cache\000Level 1 instruction cache read acces... */ + { 49467 }, + /* l1-instruction-loads\000legacy cache\000Level 1 instruction cache read accesses\... */ + { 50131 }, + /* l1-instruction-loads-access\000legacy cache\000Level 1 instruction cache read ac... */ + { 50566 }, + /* l1-instruction-loads-miss\000legacy cache\000Level 1 instruction cache read miss... */ + { 50792 }, + /* l1-instruction-loads-misses\000legacy cache\000Level 1 instruction cache read mi... */ + { 50677 }, + /* l1-instruction-loads-ops\000legacy cache\000Level 1 instruction cache read acces... */ + { 50458 }, + /* l1-instruction-loads-reference\000legacy cache\000Level 1 instruction cache read... */ + { 50344 }, + /* l1-instruction-loads-refs\000legacy cache\000Level 1 instruction cache read acce... */ + { 50235 }, + /* l1-instruction-miss\000legacy cache\000Level 1 instruction cache read misses\000... */ + { 55697 }, + /* l1-instruction-misses\000legacy cache\000Level 1 instruction cache read misses\0... */ + { 55588 }, + /* l1-instruction-ops\000legacy cache\000Level 1 instruction cache read accesses\00... */ + { 55381 }, + /* l1-instruction-prefetch\000legacy cache\000Level 1 instruction cache prefetch ac... */ + { 51672 }, + /* l1-instruction-prefetch-access\000legacy cache\000Level 1 instruction cache pref... */ + { 52151 }, + /* l1-instruction-prefetch-miss\000legacy cache\000Level 1 instruction cache prefet... */ + { 52395 }, + /* l1-instruction-prefetch-misses\000legacy cache\000Level 1 instruction cache pref... */ + { 52273 }, + /* l1-instruction-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetc... */ + { 52032 }, + /* l1-instruction-prefetch-reference\000legacy cache\000Level 1 instruction cache p... */ + { 51907 }, + /* l1-instruction-prefetch-refs\000legacy cache\000Level 1 instruction cache prefet... */ + { 51787 }, + /* l1-instruction-prefetches\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 52515 }, + /* l1-instruction-prefetches-access\000legacy cache\000Level 1 instruction cache pr... */ + { 53002 }, + /* l1-instruction-prefetches-miss\000legacy cache\000Level 1 instruction cache pref... */ + { 53250 }, + /* l1-instruction-prefetches-misses\000legacy cache\000Level 1 instruction cache pr... */ + { 53126 }, + /* l1-instruction-prefetches-ops\000legacy cache\000Level 1 instruction cache prefe... */ + { 52881 }, + /* l1-instruction-prefetches-reference\000legacy cache\000Level 1 instruction cache... */ + { 52754 }, + /* l1-instruction-prefetches-refs\000legacy cache\000Level 1 instruction cache pref... */ + { 52632 }, + /* l1-instruction-read\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 50905 }, + /* l1-instruction-read-access\000legacy cache\000Level 1 instruction cache read acc... */ + { 51336 }, + /* l1-instruction-read-miss\000legacy cache\000Level 1 instruction cache read misse... */ + { 51560 }, + /* l1-instruction-read-misses\000legacy cache\000Level 1 instruction cache read mis... */ + { 51446 }, + /* l1-instruction-read-ops\000legacy cache\000Level 1 instruction cache read access... */ + { 51229 }, + /* l1-instruction-read-reference\000legacy cache\000Level 1 instruction cache read ... */ + { 51116 }, + /* l1-instruction-read-refs\000legacy cache\000Level 1 instruction cache read acces... */ + { 51008 }, + /* l1-instruction-reference\000legacy cache\000Level 1 instruction cache read acces... */ + { 55273 }, + /* l1-instruction-refs\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 55170 }, + /* l1-instruction-speculative-load\000legacy cache\000Level 1 instruction cache pre... */ + { 54271 }, + /* l1-instruction-speculative-load-access\000legacy cache\000Level 1 instruction ca... */ + { 54782 }, + /* l1-instruction-speculative-load-miss\000legacy cache\000Level 1 instruction cach... */ + { 55042 }, + /* l1-instruction-speculative-load-misses\000legacy cache\000Level 1 instruction ca... */ + { 54912 }, + /* l1-instruction-speculative-load-ops\000legacy cache\000Level 1 instruction cache... */ + { 54655 }, + /* l1-instruction-speculative-load-reference\000legacy cache\000Level 1 instruction... */ + { 54522 }, + /* l1-instruction-speculative-load-refs\000legacy cache\000Level 1 instruction cach... */ + { 54394 }, + /* l1-instruction-speculative-read\000legacy cache\000Level 1 instruction cache pre... */ + { 53372 }, + /* l1-instruction-speculative-read-access\000legacy cache\000Level 1 instruction ca... */ + { 53883 }, + /* l1-instruction-speculative-read-miss\000legacy cache\000Level 1 instruction cach... */ + { 54143 }, + /* l1-instruction-speculative-read-misses\000legacy cache\000Level 1 instruction ca... */ + { 54013 }, + /* l1-instruction-speculative-read-ops\000legacy cache\000Level 1 instruction cache... */ + { 53756 }, + /* l1-instruction-speculative-read-reference\000legacy cache\000Level 1 instruction... */ + { 53623 }, + /* l1-instruction-speculative-read-refs\000legacy cache\000Level 1 instruction cach... */ + { 53495 }, + /* l1d\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-config=0... */ + { 15676 }, + /* l1d-access\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-c... */ + { 22971 }, + /* l1d-load\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-con... */ + { 15756 }, + /* l1d-load-access\000legacy cache\000Level 1 data cache read accesses\000legacy-ca... */ + { 16115 }, + /* l1d-load-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-... */ + { 16303 }, + /* l1d-load-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cach... */ + { 16207 }, + /* l1d-load-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache... */ + { 16026 }, + /* l1d-load-reference\000legacy cache\000Level 1 data cache read accesses\000legacy... */ + { 15931 }, + /* l1d-load-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 15841 }, + /* l1d-loads\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-co... */ + { 16397 }, + /* l1d-loads-access\000legacy cache\000Level 1 data cache read accesses\000legacy-c... */ + { 16760 }, + /* l1d-loads-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache... */ + { 16950 }, + /* l1d-loads-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cac... */ + { 16853 }, + /* l1d-loads-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 16670 }, + /* l1d-loads-reference\000legacy cache\000Level 1 data cache read accesses\000legac... */ + { 16574 }, + /* l1d-loads-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cac... */ + { 16483 }, + /* l1d-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-confi... */ + { 23149 }, + /* l1d-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cache-con... */ + { 23058 }, + /* l1d-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-conf... */ + { 22887 }, + /* l1d-prefetch\000legacy cache\000Level 1 data cache prefetch accesses\000legacy-c... */ + { 19718 }, + /* l1d-prefetch-access\000legacy cache\000Level 1 data cache prefetch accesses\000l... */ + { 20125 }, + /* l1d-prefetch-miss\000legacy cache\000Level 1 data cache prefetch misses\000legac... */ + { 20333 }, + /* l1d-prefetch-misses\000legacy cache\000Level 1 data cache prefetch misses\000leg... */ + { 20229 }, + /* l1d-prefetch-ops\000legacy cache\000Level 1 data cache prefetch accesses\000lega... */ + { 20024 }, + /* l1d-prefetch-reference\000legacy cache\000Level 1 data cache prefetch accesses\0... */ + { 19917 }, + /* l1d-prefetch-refs\000legacy cache\000Level 1 data cache prefetch accesses\000leg... */ + { 19815 }, + /* l1d-prefetches\000legacy cache\000Level 1 data cache prefetch accesses\000legacy... */ + { 20435 }, + /* l1d-prefetches-access\000legacy cache\000Level 1 data cache prefetch accesses\00... */ + { 20850 }, + /* l1d-prefetches-miss\000legacy cache\000Level 1 data cache prefetch misses\000leg... */ + { 21062 }, + /* l1d-prefetches-misses\000legacy cache\000Level 1 data cache prefetch misses\000l... */ + { 20956 }, + /* l1d-prefetches-ops\000legacy cache\000Level 1 data cache prefetch accesses\000le... */ + { 20747 }, + /* l1d-prefetches-reference\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 20638 }, + /* l1d-prefetches-refs\000legacy cache\000Level 1 data cache prefetch accesses\000l... */ + { 20534 }, + /* l1d-read\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-con... */ + { 17045 }, + /* l1d-read-access\000legacy cache\000Level 1 data cache read accesses\000legacy-ca... */ + { 17404 }, + /* l1d-read-miss\000legacy cache\000Level 1 data cache read misses\000legacy-cache-... */ + { 17592 }, + /* l1d-read-misses\000legacy cache\000Level 1 data cache read misses\000legacy-cach... */ + { 17496 }, + /* l1d-read-ops\000legacy cache\000Level 1 data cache read accesses\000legacy-cache... */ + { 17315 }, + /* l1d-read-reference\000legacy cache\000Level 1 data cache read accesses\000legacy... */ + { 17220 }, + /* l1d-read-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 17130 }, + /* l1d-reference\000legacy cache\000Level 1 data cache read accesses\000legacy-cach... */ + { 22797 }, + /* l1d-refs\000legacy cache\000Level 1 data cache read accesses\000legacy-cache-con... */ + { 22712 }, + /* l1d-speculative-load\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 21939 }, + /* l1d-speculative-load-access\000legacy cache\000Level 1 data cache prefetch acces... */ + { 22378 }, + /* l1d-speculative-load-miss\000legacy cache\000Level 1 data cache prefetch misses\... */ + { 22602 }, + /* l1d-speculative-load-misses\000legacy cache\000Level 1 data cache prefetch misse... */ + { 22490 }, + /* l1d-speculative-load-ops\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 22269 }, + /* l1d-speculative-load-reference\000legacy cache\000Level 1 data cache prefetch ac... */ + { 22154 }, + /* l1d-speculative-load-refs\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 22044 }, + /* l1d-speculative-read\000legacy cache\000Level 1 data cache prefetch accesses\000... */ + { 21166 }, + /* l1d-speculative-read-access\000legacy cache\000Level 1 data cache prefetch acces... */ + { 21605 }, + /* l1d-speculative-read-miss\000legacy cache\000Level 1 data cache prefetch misses\... */ + { 21829 }, + /* l1d-speculative-read-misses\000legacy cache\000Level 1 data cache prefetch misse... */ + { 21717 }, + /* l1d-speculative-read-ops\000legacy cache\000Level 1 data cache prefetch accesses... */ + { 21496 }, + /* l1d-speculative-read-reference\000legacy cache\000Level 1 data cache prefetch ac... */ + { 21381 }, + /* l1d-speculative-read-refs\000legacy cache\000Level 1 data cache prefetch accesse... */ + { 21271 }, + /* l1d-store\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-c... */ + { 17686 }, + /* l1d-store-access\000legacy cache\000Level 1 data cache write accesses\000legacy-... */ + { 18069 }, + /* l1d-store-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cach... */ + { 18265 }, + /* l1d-store-misses\000legacy cache\000Level 1 data cache write misses\000legacy-ca... */ + { 18167 }, + /* l1d-store-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cac... */ + { 17974 }, + /* l1d-store-reference\000legacy cache\000Level 1 data cache write accesses\000lega... */ + { 17873 }, + /* l1d-store-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 17777 }, + /* l1d-stores\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-... */ + { 18361 }, + /* l1d-stores-access\000legacy cache\000Level 1 data cache write accesses\000legacy... */ + { 18748 }, + /* l1d-stores-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cac... */ + { 18946 }, + /* l1d-stores-misses\000legacy cache\000Level 1 data cache write misses\000legacy-c... */ + { 18847 }, + /* l1d-stores-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 18652 }, + /* l1d-stores-reference\000legacy cache\000Level 1 data cache write accesses\000leg... */ + { 18550 }, + /* l1d-stores-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-c... */ + { 18453 }, + /* l1d-write\000legacy cache\000Level 1 data cache write accesses\000legacy-cache-c... */ + { 19043 }, + /* l1d-write-access\000legacy cache\000Level 1 data cache write accesses\000legacy-... */ + { 19426 }, + /* l1d-write-miss\000legacy cache\000Level 1 data cache write misses\000legacy-cach... */ + { 19622 }, + /* l1d-write-misses\000legacy cache\000Level 1 data cache write misses\000legacy-ca... */ + { 19524 }, + /* l1d-write-ops\000legacy cache\000Level 1 data cache write accesses\000legacy-cac... */ + { 19331 }, + /* l1d-write-reference\000legacy cache\000Level 1 data cache write accesses\000lega... */ + { 19230 }, + /* l1d-write-refs\000legacy cache\000Level 1 data cache write accesses\000legacy-ca... */ + { 19134 }, + /* l1i\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cache-c... */ + { 43344 }, + /* l1i-access\000legacy cache\000Level 1 instruction cache read accesses\000legacy-... */ + { 48978 }, + /* l1i-load\000legacy cache\000Level 1 instruction cache read accesses\000legacy-ca... */ + { 43431 }, + /* l1i-load-access\000legacy cache\000Level 1 instruction cache read accesses\000le... */ + { 43818 }, + /* l1i-load-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy... */ + { 44020 }, + /* l1i-load-misses\000legacy cache\000Level 1 instruction cache read misses\000lega... */ + { 43917 }, + /* l1i-load-ops\000legacy cache\000Level 1 instruction cache read accesses\000legac... */ + { 43722 }, + /* l1i-load-reference\000legacy cache\000Level 1 instruction cache read accesses\00... */ + { 43620 }, + /* l1i-load-refs\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 43523 }, + /* l1i-loads\000legacy cache\000Level 1 instruction cache read accesses\000legacy-c... */ + { 44121 }, + /* l1i-loads-access\000legacy cache\000Level 1 instruction cache read accesses\000l... */ + { 44512 }, + /* l1i-loads-miss\000legacy cache\000Level 1 instruction cache read misses\000legac... */ + { 44716 }, + /* l1i-loads-misses\000legacy cache\000Level 1 instruction cache read misses\000leg... */ + { 44612 }, + /* l1i-loads-ops\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 44415 }, + /* l1i-loads-reference\000legacy cache\000Level 1 instruction cache read accesses\0... */ + { 44312 }, + /* l1i-loads-refs\000legacy cache\000Level 1 instruction cache read accesses\000leg... */ + { 44214 }, + /* l1i-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy-cach... */ + { 49170 }, + /* l1i-misses\000legacy cache\000Level 1 instruction cache read misses\000legacy-ca... */ + { 49072 }, + /* l1i-ops\000legacy cache\000Level 1 instruction cache read accesses\000legacy-cac... */ + { 48887 }, + /* l1i-prefetch\000legacy cache\000Level 1 instruction cache prefetch accesses\000l... */ + { 45508 }, + /* l1i-prefetch-access\000legacy cache\000Level 1 instruction cache prefetch access... */ + { 45943 }, + /* l1i-prefetch-miss\000legacy cache\000Level 1 instruction cache prefetch misses\0... */ + { 46165 }, + /* l1i-prefetch-misses\000legacy cache\000Level 1 instruction cache prefetch misses... */ + { 46054 }, + /* l1i-prefetch-ops\000legacy cache\000Level 1 instruction cache prefetch accesses\... */ + { 45835 }, + /* l1i-prefetch-reference\000legacy cache\000Level 1 instruction cache prefetch acc... */ + { 45721 }, + /* l1i-prefetch-refs\000legacy cache\000Level 1 instruction cache prefetch accesses... */ + { 45612 }, + /* l1i-prefetches\000legacy cache\000Level 1 instruction cache prefetch accesses\00... */ + { 46274 }, + /* l1i-prefetches-access\000legacy cache\000Level 1 instruction cache prefetch acce... */ + { 46717 }, + /* l1i-prefetches-miss\000legacy cache\000Level 1 instruction cache prefetch misses... */ + { 46943 }, + /* l1i-prefetches-misses\000legacy cache\000Level 1 instruction cache prefetch miss... */ + { 46830 }, + /* l1i-prefetches-ops\000legacy cache\000Level 1 instruction cache prefetch accesse... */ + { 46607 }, + /* l1i-prefetches-reference\000legacy cache\000Level 1 instruction cache prefetch a... */ + { 46491 }, + /* l1i-prefetches-refs\000legacy cache\000Level 1 instruction cache prefetch access... */ + { 46380 }, + /* l1i-read\000legacy cache\000Level 1 instruction cache read accesses\000legacy-ca... */ + { 44818 }, + /* l1i-read-access\000legacy cache\000Level 1 instruction cache read accesses\000le... */ + { 45205 }, + /* l1i-read-miss\000legacy cache\000Level 1 instruction cache read misses\000legacy... */ + { 45407 }, + /* l1i-read-misses\000legacy cache\000Level 1 instruction cache read misses\000lega... */ + { 45304 }, + /* l1i-read-ops\000legacy cache\000Level 1 instruction cache read accesses\000legac... */ + { 45109 }, + /* l1i-read-reference\000legacy cache\000Level 1 instruction cache read accesses\00... */ + { 45007 }, + /* l1i-read-refs\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 44910 }, + /* l1i-reference\000legacy cache\000Level 1 instruction cache read accesses\000lega... */ + { 48790 }, + /* l1i-refs\000legacy cache\000Level 1 instruction cache read accesses\000legacy-ca... */ + { 48698 }, + /* l1i-speculative-load\000legacy cache\000Level 1 instruction cache prefetch acces... */ + { 47876 }, + /* l1i-speculative-load-access\000legacy cache\000Level 1 instruction cache prefetc... */ + { 48343 }, + /* l1i-speculative-load-miss\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 48581 }, + /* l1i-speculative-load-misses\000legacy cache\000Level 1 instruction cache prefetc... */ + { 48462 }, + /* l1i-speculative-load-ops\000legacy cache\000Level 1 instruction cache prefetch a... */ + { 48227 }, + /* l1i-speculative-load-reference\000legacy cache\000Level 1 instruction cache pref... */ + { 48105 }, + /* l1i-speculative-load-refs\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 47988 }, + /* l1i-speculative-read\000legacy cache\000Level 1 instruction cache prefetch acces... */ + { 47054 }, + /* l1i-speculative-read-access\000legacy cache\000Level 1 instruction cache prefetc... */ + { 47521 }, + /* l1i-speculative-read-miss\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 47759 }, + /* l1i-speculative-read-misses\000legacy cache\000Level 1 instruction cache prefetc... */ + { 47640 }, + /* l1i-speculative-read-ops\000legacy cache\000Level 1 instruction cache prefetch a... */ + { 47405 }, + /* l1i-speculative-read-reference\000legacy cache\000Level 1 instruction cache pref... */ + { 47283 }, + /* l1i-speculative-read-refs\000legacy cache\000Level 1 instruction cache prefetch ... */ + { 47166 }, + /* l2\000legacy cache\000Level 2 (or higher) last level cache read accesses\000lega... */ + { 63212 }, + /* l2-access\000legacy cache\000Level 2 (or higher) last level cache read accesses\... */ + { 71765 }, + /* l2-load\000legacy cache\000Level 2 (or higher) last level cache read accesses\00... */ + { 63309 }, + /* l2-load-access\000legacy cache\000Level 2 (or higher) last level cache read acce... */ + { 63736 }, + /* l2-load-miss\000legacy cache\000Level 2 (or higher) last level cache read misses... */ + { 63958 }, + /* l2-load-misses\000legacy cache\000Level 2 (or higher) last level cache read miss... */ + { 63845 }, + /* l2-load-ops\000legacy cache\000Level 2 (or higher) last level cache read accesse... */ + { 63630 }, + /* l2-load-reference\000legacy cache\000Level 2 (or higher) last level cache read a... */ + { 63518 }, + /* l2-load-refs\000legacy cache\000Level 2 (or higher) last level cache read access... */ + { 63411 }, + /* l2-loads\000legacy cache\000Level 2 (or higher) last level cache read accesses\0... */ + { 64069 }, + /* l2-loads-access\000legacy cache\000Level 2 (or higher) last level cache read acc... */ + { 64500 }, + /* l2-loads-miss\000legacy cache\000Level 2 (or higher) last level cache read misse... */ + { 64724 }, + /* l2-loads-misses\000legacy cache\000Level 2 (or higher) last level cache read mis... */ + { 64610 }, + /* l2-loads-ops\000legacy cache\000Level 2 (or higher) last level cache read access... */ + { 64393 }, + /* l2-loads-reference\000legacy cache\000Level 2 (or higher) last level cache read ... */ + { 64280 }, + /* l2-loads-refs\000legacy cache\000Level 2 (or higher) last level cache read acces... */ + { 64172 }, + /* l2-miss\000legacy cache\000Level 2 (or higher) last level cache read misses\000l... */ + { 71977 }, + /* l2-misses\000legacy cache\000Level 2 (or higher) last level cache read misses\00... */ + { 71869 }, + /* l2-ops\000legacy cache\000Level 2 (or higher) last level cache read accesses\000... */ + { 71664 }, + /* l2-prefetch\000legacy cache\000Level 2 (or higher) last level cache prefetch acc... */ + { 67985 }, + /* l2-prefetch-access\000legacy cache\000Level 2 (or higher) last level cache prefe... */ + { 68460 }, + /* l2-prefetch-miss\000legacy cache\000Level 2 (or higher) last level cache prefetc... */ + { 68702 }, + /* l2-prefetch-misses\000legacy cache\000Level 2 (or higher) last level cache prefe... */ + { 68581 }, + /* l2-prefetch-ops\000legacy cache\000Level 2 (or higher) last level cache prefetch... */ + { 68342 }, + /* l2-prefetch-reference\000legacy cache\000Level 2 (or higher) last level cache pr... */ + { 68218 }, + /* l2-prefetch-refs\000legacy cache\000Level 2 (or higher) last level cache prefetc... */ + { 68099 }, + /* l2-prefetches\000legacy cache\000Level 2 (or higher) last level cache prefetch a... */ + { 68821 }, + /* l2-prefetches-access\000legacy cache\000Level 2 (or higher) last level cache pre... */ + { 69304 }, + /* l2-prefetches-miss\000legacy cache\000Level 2 (or higher) last level cache prefe... */ + { 69550 }, + /* l2-prefetches-misses\000legacy cache\000Level 2 (or higher) last level cache pre... */ + { 69427 }, + /* l2-prefetches-ops\000legacy cache\000Level 2 (or higher) last level cache prefet... */ + { 69184 }, + /* l2-prefetches-reference\000legacy cache\000Level 2 (or higher) last level cache ... */ + { 69058 }, + /* l2-prefetches-refs\000legacy cache\000Level 2 (or higher) last level cache prefe... */ + { 68937 }, + /* l2-read\000legacy cache\000Level 2 (or higher) last level cache read accesses\00... */ + { 64836 }, + /* l2-read-access\000legacy cache\000Level 2 (or higher) last level cache read acce... */ + { 65263 }, + /* l2-read-miss\000legacy cache\000Level 2 (or higher) last level cache read misses... */ + { 65485 }, + /* l2-read-misses\000legacy cache\000Level 2 (or higher) last level cache read miss... */ + { 65372 }, + /* l2-read-ops\000legacy cache\000Level 2 (or higher) last level cache read accesse... */ + { 65157 }, + /* l2-read-reference\000legacy cache\000Level 2 (or higher) last level cache read a... */ + { 65045 }, + /* l2-read-refs\000legacy cache\000Level 2 (or higher) last level cache read access... */ + { 64938 }, + /* l2-reference\000legacy cache\000Level 2 (or higher) last level cache read access... */ + { 71557 }, + /* l2-refs\000legacy cache\000Level 2 (or higher) last level cache read accesses\00... */ + { 71455 }, + /* l2-speculative-load\000legacy cache\000Level 2 (or higher) last level cache pref... */ + { 70563 }, + /* l2-speculative-load-access\000legacy cache\000Level 2 (or higher) last level cac... */ + { 71070 }, + /* l2-speculative-load-miss\000legacy cache\000Level 2 (or higher) last level cache... */ + { 71328 }, + /* l2-speculative-load-misses\000legacy cache\000Level 2 (or higher) last level cac... */ + { 71199 }, + /* l2-speculative-load-ops\000legacy cache\000Level 2 (or higher) last level cache ... */ + { 70944 }, + /* l2-speculative-load-reference\000legacy cache\000Level 2 (or higher) last level ... */ + { 70812 }, + /* l2-speculative-load-refs\000legacy cache\000Level 2 (or higher) last level cache... */ + { 70685 }, + /* l2-speculative-read\000legacy cache\000Level 2 (or higher) last level cache pref... */ + { 69671 }, + /* l2-speculative-read-access\000legacy cache\000Level 2 (or higher) last level cac... */ + { 70178 }, + /* l2-speculative-read-miss\000legacy cache\000Level 2 (or higher) last level cache... */ + { 70436 }, + /* l2-speculative-read-misses\000legacy cache\000Level 2 (or higher) last level cac... */ + { 70307 }, + /* l2-speculative-read-ops\000legacy cache\000Level 2 (or higher) last level cache ... */ + { 70052 }, + /* l2-speculative-read-reference\000legacy cache\000Level 2 (or higher) last level ... */ + { 69920 }, + /* l2-speculative-read-refs\000legacy cache\000Level 2 (or higher) last level cache... */ + { 69793 }, + /* l2-store\000legacy cache\000Level 2 (or higher) last level cache write accesses\... */ + { 65596 }, + /* l2-store-access\000legacy cache\000Level 2 (or higher) last level cache write ac... */ + { 66047 }, + /* l2-store-miss\000legacy cache\000Level 2 (or higher) last level cache write miss... */ + { 66277 }, + /* l2-store-misses\000legacy cache\000Level 2 (or higher) last level cache write mi... */ + { 66162 }, + /* l2-store-ops\000legacy cache\000Level 2 (or higher) last level cache write acces... */ + { 65935 }, + /* l2-store-reference\000legacy cache\000Level 2 (or higher) last level cache write... */ + { 65817 }, + /* l2-store-refs\000legacy cache\000Level 2 (or higher) last level cache write acce... */ + { 65704 }, + /* l2-stores\000legacy cache\000Level 2 (or higher) last level cache write accesses... */ + { 66390 }, + /* l2-stores-access\000legacy cache\000Level 2 (or higher) last level cache write a... */ + { 66845 }, + /* l2-stores-miss\000legacy cache\000Level 2 (or higher) last level cache write mis... */ + { 67077 }, + /* l2-stores-misses\000legacy cache\000Level 2 (or higher) last level cache write m... */ + { 66961 }, + /* l2-stores-ops\000legacy cache\000Level 2 (or higher) last level cache write acce... */ + { 66732 }, + /* l2-stores-reference\000legacy cache\000Level 2 (or higher) last level cache writ... */ + { 66613 }, + /* l2-stores-refs\000legacy cache\000Level 2 (or higher) last level cache write acc... */ + { 66499 }, + /* l2-write\000legacy cache\000Level 2 (or higher) last level cache write accesses\... */ + { 67191 }, + /* l2-write-access\000legacy cache\000Level 2 (or higher) last level cache write ac... */ + { 67642 }, + /* l2-write-miss\000legacy cache\000Level 2 (or higher) last level cache write miss... */ + { 67872 }, + /* l2-write-misses\000legacy cache\000Level 2 (or higher) last level cache write mi... */ + { 67757 }, + /* l2-write-ops\000legacy cache\000Level 2 (or higher) last level cache write acces... */ + { 67530 }, + /* l2-write-reference\000legacy cache\000Level 2 (or higher) last level cache write... */ + { 67412 }, + /* l2-write-refs\000legacy cache\000Level 2 (or higher) last level cache write acce... */ + { 67299 }, + /* llc\000legacy cache\000Last level cache read accesses\000legacy-cache-config=2\0... */ + { 55804 }, + /* llc-access\000legacy cache\000Last level cache read accesses\000legacy-cache-con... */ + { 62951 }, + /* llc-load\000legacy cache\000Last level cache read accesses\000legacy-cache-confi... */ + { 55882 }, + /* llc-load-access\000legacy cache\000Last level cache read accesses\000legacy-cach... */ + { 56233 }, + /* llc-load-miss\000legacy cache\000Last level cache read misses\000legacy-cache-co... */ + { 56417 }, + /* llc-load-misses\000legacy cache\000Last level cache read misses\000legacy-cache-... */ + { 56323 }, + /* llc-load-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-c... */ + { 56146 }, + /* llc-load-reference\000legacy cache\000Last level cache read accesses\000legacy-c... */ + { 56053 }, + /* llc-load-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-... */ + { 55965 }, + /* llc-loads\000legacy cache\000Last level cache read accesses\000legacy-cache-conf... */ + { 56509 }, + /* llc-loads-access\000legacy cache\000Last level cache read accesses\000legacy-cac... */ + { 56864 }, + /* llc-loads-miss\000legacy cache\000Last level cache read misses\000legacy-cache-c... */ + { 57050 }, + /* llc-loads-misses\000legacy cache\000Last level cache read misses\000legacy-cache... */ + { 56955 }, + /* llc-loads-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-... */ + { 56776 }, + /* llc-loads-reference\000legacy cache\000Last level cache read accesses\000legacy-... */ + { 56682 }, + /* llc-loads-refs\000legacy cache\000Last level cache read accesses\000legacy-cache... */ + { 56593 }, + /* llc-miss\000legacy cache\000Last level cache read misses\000legacy-cache-config=... */ + { 63125 }, + /* llc-misses\000legacy cache\000Last level cache read misses\000legacy-cache-confi... */ + { 63036 }, + /* llc-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-config... */ + { 62869 }, + /* llc-prefetch\000legacy cache\000Last level cache prefetch accesses\000legacy-cac... */ + { 59760 }, + /* llc-prefetch-access\000legacy cache\000Last level cache prefetch accesses\000leg... */ + { 60159 }, + /* llc-prefetch-miss\000legacy cache\000Last level cache prefetch misses\000legacy-... */ + { 60363 }, + /* llc-prefetch-misses\000legacy cache\000Last level cache prefetch misses\000legac... */ + { 60261 }, + /* llc-prefetch-ops\000legacy cache\000Last level cache prefetch accesses\000legacy... */ + { 60060 }, + /* llc-prefetch-reference\000legacy cache\000Last level cache prefetch accesses\000... */ + { 59955 }, + /* llc-prefetch-refs\000legacy cache\000Last level cache prefetch accesses\000legac... */ + { 59855 }, + /* llc-prefetches\000legacy cache\000Last level cache prefetch accesses\000legacy-c... */ + { 60463 }, + /* llc-prefetches-access\000legacy cache\000Last level cache prefetch accesses\000l... */ + { 60870 }, + /* llc-prefetches-miss\000legacy cache\000Last level cache prefetch misses\000legac... */ + { 61078 }, + /* llc-prefetches-misses\000legacy cache\000Last level cache prefetch misses\000leg... */ + { 60974 }, + /* llc-prefetches-ops\000legacy cache\000Last level cache prefetch accesses\000lega... */ + { 60769 }, + /* llc-prefetches-reference\000legacy cache\000Last level cache prefetch accesses\0... */ + { 60662 }, + /* llc-prefetches-refs\000legacy cache\000Last level cache prefetch accesses\000leg... */ + { 60560 }, + /* llc-read\000legacy cache\000Last level cache read accesses\000legacy-cache-confi... */ + { 57143 }, + /* llc-read-access\000legacy cache\000Last level cache read accesses\000legacy-cach... */ + { 57494 }, + /* llc-read-miss\000legacy cache\000Last level cache read misses\000legacy-cache-co... */ + { 57678 }, + /* llc-read-misses\000legacy cache\000Last level cache read misses\000legacy-cache-... */ + { 57584 }, + /* llc-read-ops\000legacy cache\000Last level cache read accesses\000legacy-cache-c... */ + { 57407 }, + /* llc-read-reference\000legacy cache\000Last level cache read accesses\000legacy-c... */ + { 57314 }, + /* llc-read-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-... */ + { 57226 }, + /* llc-reference\000legacy cache\000Last level cache read accesses\000legacy-cache-... */ + { 62781 }, + /* llc-refs\000legacy cache\000Last level cache read accesses\000legacy-cache-confi... */ + { 62698 }, + /* llc-speculative-load\000legacy cache\000Last level cache prefetch accesses\000le... */ + { 61939 }, + /* llc-speculative-load-access\000legacy cache\000Last level cache prefetch accesse... */ + { 62370 }, + /* llc-speculative-load-miss\000legacy cache\000Last level cache prefetch misses\00... */ + { 62590 }, + /* llc-speculative-load-misses\000legacy cache\000Last level cache prefetch misses\... */ + { 62480 }, + /* llc-speculative-load-ops\000legacy cache\000Last level cache prefetch accesses\0... */ + { 62263 }, + /* llc-speculative-load-reference\000legacy cache\000Last level cache prefetch acce... */ + { 62150 }, + /* llc-speculative-load-refs\000legacy cache\000Last level cache prefetch accesses\... */ + { 62042 }, + /* llc-speculative-read\000legacy cache\000Last level cache prefetch accesses\000le... */ + { 61180 }, + /* llc-speculative-read-access\000legacy cache\000Last level cache prefetch accesse... */ + { 61611 }, + /* llc-speculative-read-miss\000legacy cache\000Last level cache prefetch misses\00... */ + { 61831 }, + /* llc-speculative-read-misses\000legacy cache\000Last level cache prefetch misses\... */ + { 61721 }, + /* llc-speculative-read-ops\000legacy cache\000Last level cache prefetch accesses\0... */ + { 61504 }, + /* llc-speculative-read-reference\000legacy cache\000Last level cache prefetch acce... */ + { 61391 }, + /* llc-speculative-read-refs\000legacy cache\000Last level cache prefetch accesses\... */ + { 61283 }, + /* llc-store\000legacy cache\000Last level cache write accesses\000legacy-cache-con... */ + { 57770 }, + /* llc-store-access\000legacy cache\000Last level cache write accesses\000legacy-ca... */ + { 58145 }, + /* llc-store-miss\000legacy cache\000Last level cache write misses\000legacy-cache-... */ + { 58337 }, + /* llc-store-misses\000legacy cache\000Last level cache write misses\000legacy-cach... */ + { 58241 }, + /* llc-store-ops\000legacy cache\000Last level cache write accesses\000legacy-cache... */ + { 58052 }, + /* llc-store-reference\000legacy cache\000Last level cache write accesses\000legacy... */ + { 57953 }, + /* llc-store-refs\000legacy cache\000Last level cache write accesses\000legacy-cach... */ + { 57859 }, + /* llc-stores\000legacy cache\000Last level cache write accesses\000legacy-cache-co... */ + { 58431 }, + /* llc-stores-access\000legacy cache\000Last level cache write accesses\000legacy-c... */ + { 58810 }, + /* llc-stores-miss\000legacy cache\000Last level cache write misses\000legacy-cache... */ + { 59004 }, + /* llc-stores-misses\000legacy cache\000Last level cache write misses\000legacy-cac... */ + { 58907 }, + /* llc-stores-ops\000legacy cache\000Last level cache write accesses\000legacy-cach... */ + { 58716 }, + /* llc-stores-reference\000legacy cache\000Last level cache write accesses\000legac... */ + { 58616 }, + /* llc-stores-refs\000legacy cache\000Last level cache write accesses\000legacy-cac... */ + { 58521 }, + /* llc-write\000legacy cache\000Last level cache write accesses\000legacy-cache-con... */ + { 59099 }, + /* llc-write-access\000legacy cache\000Last level cache write accesses\000legacy-ca... */ + { 59474 }, + /* llc-write-miss\000legacy cache\000Last level cache write misses\000legacy-cache-... */ + { 59666 }, + /* llc-write-misses\000legacy cache\000Last level cache write misses\000legacy-cach... */ + { 59570 }, + /* llc-write-ops\000legacy cache\000Last level cache write accesses\000legacy-cache... */ + { 59381 }, + /* llc-write-reference\000legacy cache\000Last level cache write accesses\000legacy... */ + { 59282 }, + /* llc-write-refs\000legacy cache\000Last level cache write accesses\000legacy-cach... */ + { 59188 }, + /* node\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\000\... */ + { 114128 }, + /* node-access\000legacy cache\000Local memory read accesses\000legacy-cache-config... */ + { 121053 }, + /* node-load\000legacy cache\000Local memory read accesses\000legacy-cache-config=6... */ + { 114203 }, + /* node-load-access\000legacy cache\000Local memory read accesses\000legacy-cache-c... */ + { 114542 }, + /* node-load-miss\000legacy cache\000Local memory read misses\000legacy-cache-confi... */ + { 114720 }, + /* node-load-misses\000legacy cache\000Local memory read misses\000legacy-cache-con... */ + { 114629 }, + /* node-load-ops\000legacy cache\000Local memory read accesses\000legacy-cache-conf... */ + { 114458 }, + /* node-load-reference\000legacy cache\000Local memory read accesses\000legacy-cach... */ + { 114368 }, + /* node-load-refs\000legacy cache\000Local memory read accesses\000legacy-cache-con... */ + { 114283 }, + /* node-loads\000legacy cache\000Local memory read accesses\000legacy-cache-config=... */ + { 114809 }, + /* node-loads-access\000legacy cache\000Local memory read accesses\000legacy-cache-... */ + { 115152 }, + /* node-loads-miss\000legacy cache\000Local memory read misses\000legacy-cache-conf... */ + { 115332 }, + /* node-loads-misses\000legacy cache\000Local memory read misses\000legacy-cache-co... */ + { 115240 }, + /* node-loads-ops\000legacy cache\000Local memory read accesses\000legacy-cache-con... */ + { 115067 }, + /* node-loads-reference\000legacy cache\000Local memory read accesses\000legacy-cac... */ + { 114976 }, + /* node-loads-refs\000legacy cache\000Local memory read accesses\000legacy-cache-co... */ + { 114890 }, + /* node-miss\000legacy cache\000Local memory read misses\000legacy-cache-config=0x1... */ + { 121221 }, + /* node-misses\000legacy cache\000Local memory read misses\000legacy-cache-config=0... */ + { 121135 }, + /* node-ops\000legacy cache\000Local memory read accesses\000legacy-cache-config=6\... */ + { 120974 }, + /* node-prefetch\000legacy cache\000Local memory prefetch accesses\000legacy-cache-... */ + { 117955 }, + /* node-prefetch-access\000legacy cache\000Local memory prefetch accesses\000legacy... */ + { 118342 }, + /* node-prefetch-miss\000legacy cache\000Local memory prefetch misses\000legacy-cac... */ + { 118540 }, + /* node-prefetch-misses\000legacy cache\000Local memory prefetch misses\000legacy-c... */ + { 118441 }, + /* node-prefetch-ops\000legacy cache\000Local memory prefetch accesses\000legacy-ca... */ + { 118246 }, + /* node-prefetch-reference\000legacy cache\000Local memory prefetch accesses\000leg... */ + { 118144 }, + /* node-prefetch-refs\000legacy cache\000Local memory prefetch accesses\000legacy-c... */ + { 118047 }, + /* node-prefetches\000legacy cache\000Local memory prefetch accesses\000legacy-cach... */ + { 118637 }, + /* node-prefetches-access\000legacy cache\000Local memory prefetch accesses\000lega... */ + { 119032 }, + /* node-prefetches-miss\000legacy cache\000Local memory prefetch misses\000legacy-c... */ + { 119234 }, + /* node-prefetches-misses\000legacy cache\000Local memory prefetch misses\000legacy... */ + { 119133 }, + /* node-prefetches-ops\000legacy cache\000Local memory prefetch accesses\000legacy-... */ + { 118934 }, + /* node-prefetches-reference\000legacy cache\000Local memory prefetch accesses\000l... */ + { 118830 }, + /* node-prefetches-refs\000legacy cache\000Local memory prefetch accesses\000legacy... */ + { 118731 }, + /* node-read\000legacy cache\000Local memory read accesses\000legacy-cache-config=6... */ + { 115422 }, + /* node-read-access\000legacy cache\000Local memory read accesses\000legacy-cache-c... */ + { 115761 }, + /* node-read-miss\000legacy cache\000Local memory read misses\000legacy-cache-confi... */ + { 115939 }, + /* node-read-misses\000legacy cache\000Local memory read misses\000legacy-cache-con... */ + { 115848 }, + /* node-read-ops\000legacy cache\000Local memory read accesses\000legacy-cache-conf... */ + { 115677 }, + /* node-read-reference\000legacy cache\000Local memory read accesses\000legacy-cach... */ + { 115587 }, + /* node-read-refs\000legacy cache\000Local memory read accesses\000legacy-cache-con... */ + { 115502 }, + /* node-reference\000legacy cache\000Local memory read accesses\000legacy-cache-con... */ + { 120889 }, + /* node-refs\000legacy cache\000Local memory read accesses\000legacy-cache-config=6... */ + { 120809 }, + /* node-speculative-load\000legacy cache\000Local memory prefetch accesses\000legac... */ + { 120071 }, + /* node-speculative-load-access\000legacy cache\000Local memory prefetch accesses\0... */ + { 120490 }, + /* node-speculative-load-miss\000legacy cache\000Local memory prefetch misses\000le... */ + { 120704 }, + /* node-speculative-load-misses\000legacy cache\000Local memory prefetch misses\000... */ + { 120597 }, + /* node-speculative-load-ops\000legacy cache\000Local memory prefetch accesses\000l... */ + { 120386 }, + /* node-speculative-load-reference\000legacy cache\000Local memory prefetch accesse... */ + { 120276 }, + /* node-speculative-load-refs\000legacy cache\000Local memory prefetch accesses\000... */ + { 120171 }, + /* node-speculative-read\000legacy cache\000Local memory prefetch accesses\000legac... */ + { 119333 }, + /* node-speculative-read-access\000legacy cache\000Local memory prefetch accesses\0... */ + { 119752 }, + /* node-speculative-read-miss\000legacy cache\000Local memory prefetch misses\000le... */ + { 119966 }, + /* node-speculative-read-misses\000legacy cache\000Local memory prefetch misses\000... */ + { 119859 }, + /* node-speculative-read-ops\000legacy cache\000Local memory prefetch accesses\000l... */ + { 119648 }, + /* node-speculative-read-reference\000legacy cache\000Local memory prefetch accesse... */ + { 119538 }, + /* node-speculative-read-refs\000legacy cache\000Local memory prefetch accesses\000... */ + { 119433 }, + /* node-store\000legacy cache\000Local memory write accesses\000legacy-cache-config... */ + { 116028 }, + /* node-store-access\000legacy cache\000Local memory write accesses\000legacy-cache... */ + { 116391 }, + /* node-store-miss\000legacy cache\000Local memory write misses\000legacy-cache-con... */ + { 116577 }, + /* node-store-misses\000legacy cache\000Local memory write misses\000legacy-cache-c... */ + { 116484 }, + /* node-store-ops\000legacy cache\000Local memory write accesses\000legacy-cache-co... */ + { 116301 }, + /* node-store-reference\000legacy cache\000Local memory write accesses\000legacy-ca... */ + { 116205 }, + /* node-store-refs\000legacy cache\000Local memory write accesses\000legacy-cache-c... */ + { 116114 }, + /* node-stores\000legacy cache\000Local memory write accesses\000legacy-cache-confi... */ + { 116668 }, + /* node-stores-access\000legacy cache\000Local memory write accesses\000legacy-cach... */ + { 117035 }, + /* node-stores-miss\000legacy cache\000Local memory write misses\000legacy-cache-co... */ + { 117223 }, + /* node-stores-misses\000legacy cache\000Local memory write misses\000legacy-cache-... */ + { 117129 }, + /* node-stores-ops\000legacy cache\000Local memory write accesses\000legacy-cache-c... */ + { 116944 }, + /* node-stores-reference\000legacy cache\000Local memory write accesses\000legacy-c... */ + { 116847 }, + /* node-stores-refs\000legacy cache\000Local memory write accesses\000legacy-cache-... */ + { 116755 }, + /* node-write\000legacy cache\000Local memory write accesses\000legacy-cache-config... */ + { 117315 }, + /* node-write-access\000legacy cache\000Local memory write accesses\000legacy-cache... */ + { 117678 }, + /* node-write-miss\000legacy cache\000Local memory write misses\000legacy-cache-con... */ + { 117864 }, + /* node-write-misses\000legacy cache\000Local memory write misses\000legacy-cache-c... */ + { 117771 }, + /* node-write-ops\000legacy cache\000Local memory write accesses\000legacy-cache-co... */ + { 117588 }, + /* node-write-reference\000legacy cache\000Local memory write accesses\000legacy-ca... */ + { 117492 }, + /* node-write-refs\000legacy cache\000Local memory write accesses\000legacy-cache-c... */ + { 117401 }, + /* ref-cycles\000legacy hardware\000Total cycles; not affected by CPU frequency sca... */ + { 123400 }, + /* stalled-cycles-backend\000legacy hardware\000Stalled cycles during retirement [T... */ + { 123094 }, + /* stalled-cycles-frontend\000legacy hardware\000Stalled cycles during issue [This ... */ + { 122795 }, }; static const struct compact_pmu_event pmu_events__common_software[] = { -{ 124563 }, /* alignment-faults\000software\000Number of kernel handled memory alignment faults\000config=7\000\00000\000\000\000\000\000 */ -{ 124862 }, /* bpf-output\000software\000An event used by BPF programs to write to the perf ring buffer\000config=0xa\000\00000\000\000\000\000\000 */ -{ 124964 }, /* cgroup-switches\000software\000Number of context switches to a task in a different cgroup\000config=0xb\000\00000\000\000\000\000\000 */ -{ 123885 }, /* context-switches\000software\000Number of context switches [This event is an alias of cs]\000config=3\000\00000\000\000\000\000\000 */ -{ 123521 }, /* cpu-clock\000software\000Per-CPU high-resolution timer based event\000config=0\000\000001e-6msec\000\000\000\000\000 */ -{ 124087 }, /* cpu-migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of migrations]\000config=4\000\00000\000\000\000\000\000 */ -{ 123986 }, /* cs\000software\000Number of context switches [This event is an alias of context-switches]\000config=3\000\00000\000\000\000\000\000 */ -{ 124782 }, /* dummy\000software\000A placeholder event that doesn't count anything\000config=9\000\00000\000\000\000\000\000 */ -{ 124655 }, /* emulation-faults\000software\000Number of kernel handled unimplemented instruction faults handled through emulation\000config=8\000\00000\000\000\000\000\000 */ -{ 123695 }, /* faults\000software\000Number of page faults [This event is an alias of page-faults]\000config=2\000\00000\000\000\000\000\000 */ -{ 124460 }, /* major-faults\000software\000Number of major page faults. Major faults require I/O to handle\000config=6\000\00000\000\000\000\000\000 */ -{ 124219 }, /* migrations\000software\000Number of times a process has migrated to a new CPU [This event is an alias of cpu-migrations]\000config=4\000\00000\000\000\000\000\000 */ -{ 124351 }, /* minor-faults\000software\000Number of minor page faults. Minor faults don't require I/O to handle\000config=5\000\00000\000\000\000\000\000 */ -{ 123790 }, /* page-faults\000software\000Number of page faults [This event is an alias of faults]\000config=2\000\00000\000\000\000\000\000 */ -{ 123607 }, /* task-clock\000software\000Per-task high-resolution timer based event\000config=1\000\000001e-6msec\000\000\000\000\000 */ + /* alignment-faults\000software\000Number of kernel handled memory alignment faults... */ + { 124563 }, + /* bpf-output\000software\000An event used by BPF programs to write to the perf rin... */ + { 124862 }, + /* cgroup-switches\000software\000Number of context switches to a task in a differe... */ + { 124964 }, + /* context-switches\000software\000Number of context switches [This event is an ali... */ + { 123885 }, + /* cpu-clock\000software\000Per-CPU high-resolution timer based event\000config=0\0... */ + { 123521 }, + /* cpu-migrations\000software\000Number of times a process has migrated to a new CP... */ + { 124087 }, + /* cs\000software\000Number of context switches [This event is an alias of context-... */ + { 123986 }, + /* dummy\000software\000A placeholder event that doesn't count anything\000config=9... */ + { 124782 }, + /* emulation-faults\000software\000Number of kernel handled unimplemented instructi... */ + { 124655 }, + /* faults\000software\000Number of page faults [This event is an alias of page-faul... */ + { 123695 }, + /* major-faults\000software\000Number of major page faults. Major faults require I/... */ + { 124460 }, + /* migrations\000software\000Number of times a process has migrated to a new CPU [T... */ + { 124219 }, + /* minor-faults\000software\000Number of minor page faults. Minor faults don't requ... */ + { 124351 }, + /* page-faults\000software\000Number of page faults [This event is an alias of faul... */ + { 123790 }, + /* task-clock\000software\000Per-task high-resolution timer based event\000config=1... */ + { 123607 }, }; static const struct compact_pmu_event pmu_events__common_tool[] = { -{ 126205 }, /* core_wide\000tool\0001 if not SMT, if SMT are events being gathered on all SMT threads 1 otherwise 0\000config=0xd\000\00000\000\000\000\000\000 */ -{ 125072 }, /* duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000\00000\000\000\000\000\000 */ -{ 125286 }, /* has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000\00000\000\000\000\000\000 */ -{ 125362 }, /* num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with each thread being associated with a logical Linux CPU\000config=5\000\00000\000\000\000\000\000 */ -{ 125507 }, /* num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPUs on a core\000config=6\000\00000\000\000\000\000\000 */ -{ 125610 }, /* num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be multiple such CPUs on a core\000config=7\000\00000\000\000\000\000\000 */ -{ 125727 }, /* num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000\00000\000\000\000\000\000 */ -{ 125803 }, /* num_packages\000tool\000Number of packages. Each package has 1 or more die\000config=9\000\00000\000\000\000\000\000 */ -{ 125889 }, /* slots\000tool\000Number of functional units that in parallel can execute parts of an instruction\000config=0xa\000\00000\000\000\000\000\000 */ -{ 125999 }, /* smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enable otherwise 0\000config=0xb\000\00000\000\000\000\000\000 */ -{ 125218 }, /* system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\000\000\000\000\000 */ -{ 126106 }, /* system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per second\000config=0xc\000\00000\000\000\000\000\000 */ -{ 126319 }, /* target_cpu\000tool\0001 if CPUs being analyzed, 0 if threads/processes\000config=0xe\000\00000\000\000\000\000\000 */ -{ 125148 }, /* user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000\000\000\000\000\000 */ + /* core_wide\000tool\0001 if not SMT, if SMT are events being gathered on all SMT t... */ + { 126205 }, + /* duration_time\000tool\000Wall clock interval time in nanoseconds\000config=1\000... */ + { 125072 }, + /* has_pmem\000tool\0001 if persistent memory installed otherwise 0\000config=4\000... */ + { 125286 }, + /* num_cores\000tool\000Number of cores. A core consists of 1 or more thread, with ... */ + { 125362 }, + /* num_cpus\000tool\000Number of logical Linux CPUs. There may be multiple such CPU... */ + { 125507 }, + /* num_cpus_online\000tool\000Number of online logical Linux CPUs. There may be mul... */ + { 125610 }, + /* num_dies\000tool\000Number of dies. Each die has 1 or more cores\000config=8\000... */ + { 125727 }, + /* num_packages\000tool\000Number of packages. Each package has 1 or more die\000co... */ + { 125803 }, + /* slots\000tool\000Number of functional units that in parallel can execute parts o... */ + { 125889 }, + /* smt_on\000tool\0001 if simultaneous multithreading (aka hyperthreading) is enabl... */ + { 125999 }, + /* system_time\000tool\000System/kernel time in nanoseconds\000config=3\000\00000\0... */ + { 125218 }, + /* system_tsc_freq\000tool\000The amount a Time Stamp Counter (TSC) increases per s... */ + { 126106 }, + /* target_cpu\000tool\0001 if CPUs being analyzed, 0 if threads/processes\000config... */ + { 126319 }, + /* user_time\000tool\000User (non-kernel) time in nanoseconds\000config=2\000\00000... */ + { 125148 }, }; static const struct pmu_table_entry pmu_events__common[] = { -{ - .entries = pmu_events__common_default_core, - .num_entries = ARRAY_SIZE(pmu_events__common_default_core), - .pmu_name = { 0 /* default_core\000 */ }, -}, -{ - .entries = pmu_events__common_software, - .num_entries = ARRAY_SIZE(pmu_events__common_software), - .pmu_name = { 123512 /* software\000 */ }, -}, -{ - .entries = pmu_events__common_tool, - .num_entries = ARRAY_SIZE(pmu_events__common_tool), - .pmu_name = { 125067 /* tool\000 */ }, -}, + { + .entries = pmu_events__common_default_core, + .num_entries = ARRAY_SIZE(pmu_events__common_default_core), + .pmu_name = { 0 /* default_core\000 */ }, + }, + { + .entries = pmu_events__common_software, + .num_entries = ARRAY_SIZE(pmu_events__common_software), + .pmu_name = { 123512 /* software\000 */ }, + }, + { + .entries = pmu_events__common_tool, + .num_entries = ARRAY_SIZE(pmu_events__common_tool), + .pmu_name = { 125067 /* tool\000 */ }, + }, }; static const struct compact_pmu_event pmu_metrics__common_default_core[] = { -{ 127956 }, /* CPUs_utilized\000Default\000(software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@) / (duration_time * 1e9)\000\000Average CPU utilization\000\0001CPUs\000\000\000\000011 */ -{ 129583 }, /* backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if has_event(stalled\\-cycles\\-backend) else 0)\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001 */ -{ 129933 }, /* branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011 */ -{ 130113 }, /* branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001 */ -{ 128142 }, /* cs_per_second\000Default\000software@context\\-switches\\,name\\=context\\-switches@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Context switches per CPU second\000\0001cs/sec\000\000\000\000011 */ -{ 129757 }, /* cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011 */ -{ 130549 }, /* dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001 */ -{ 129404 }, /* frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles if has_event(stalled\\-cycles\\-frontend) else 0)\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001 */ -{ 128866 }, /* insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\000Instructions Per Cycle\000\0001instructions\000\000\000\000001 */ -{ 130655 }, /* itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001 */ -{ 130761 }, /* l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001 */ -{ 130217 }, /* l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001 */ -{ 130434 }, /* l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001 */ -{ 130333 }, /* llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001 */ -{ 128375 }, /* migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-migrations@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Process migrations to a new CPU per CPU second\000\0001migrations/sec\000\000\000\000011 */ -{ 128635 }, /* page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faults@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Page faults per CPU second\000\0001faults/sec\000\000\000\000011 */ -{ 128979 }, /* stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions if has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend) else (stalled\\-cycles\\-frontend / instructions if has_event(stalled\\-cycles\\-frontend) else (stalled\\-cycles\\-backend / instructions if has_event(stalled\\-cycles\\-backend) else 0)))\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001 */ + /* CPUs_utilized\000Default\000(software@cpu\\-clock\\,name\\=cpu\\-clock@ if #targ... */ + { 127956 }, + /* backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if ... */ + { 129583 }, + /* branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-c... */ + { 129933 }, + /* branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > ... */ + { 130113 }, + /* cs_per_second\000Default\000software@context\\-switches\\,name\\=context\\-switc... */ + { 128142 }, + /* cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu... */ + { 129757 }, + /* dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_r... */ + { 130549 }, + /* frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles i... */ + { 129404 }, + /* insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\0... */ + { 128866 }, + /* itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_r... */ + { 130655 }, + /* l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcac... */ + { 130761 }, + /* l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\0... */ + { 130217 }, + /* l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\0... */ + { 130434 }, + /* llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate ... */ + { 130333 }, + /* migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-mig... */ + { 128375 }, + /* page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faul... */ + { 128635 }, + /* stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, s... */ + { 128979 }, }; static const struct pmu_table_entry pmu_metrics__common[] = { -{ - .entries = pmu_metrics__common_default_core, - .num_entries = ARRAY_SIZE(pmu_metrics__common_default_core), - .pmu_name = { 0 /* default_core\000 */ }, -}, + { + .entries = pmu_metrics__common_default_core, + .num_entries = ARRAY_SIZE(pmu_metrics__common_default_core), + .pmu_name = { 0 /* default_core\000 */ }, + }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_default_core[] = { -{ 126403 }, /* bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000\000\000\000 */ -{ 126465 }, /* bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000\000\000\000 */ -{ 126727 }, /* dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispatch for any reason\000event=9,period=200000,umask=0x20\000\00000\000\000\000\000\000 */ -{ 126860 }, /* eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) transitions\000event=0x3a,period=200000\000\00000\000\000\000\000\000 */ -{ 126527 }, /* l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000\000\000\000Attributable Level 3 cache access, read\000 */ -{ 126625 }, /* segment_reg_loads.any\000other\000Number of segment register loads\000event=6,period=200000,umask=0x80\000\00000\000\000\000\000\000 */ + /* bp_l1_btb_correct\000branch\000L1 BTB Correction\000event=0x8a\000\00000\000\000... */ + { 126403 }, + /* bp_l2_btb_correct\000branch\000L2 BTB Correction\000event=0x8b\000\00000\000\000... */ + { 126465 }, + /* dispatch_blocked.any\000other\000Memory cluster signals to block micro-op dispat... */ + { 126727 }, + /* eist_trans\000other\000Number of Enhanced Intel SpeedStep(R) Technology (EIST) t... */ + { 126860 }, + /* l3_cache_rd\000cache\000L3 cache access, read\000event=0x40\000\00000\000\000\00... */ + { 126527 }, + /* segment_reg_loads.any\000other\000Number of segment register loads\000event=6,pe... */ + { 126625 }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_hisi_sccl_ddrc[] = { -{ 126993 }, /* uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000\000\000\000\000\000 */ + /* uncore_hisi_ddrc.flux_wcmd\000uncore\000DDRC write commands\000event=2\000\00000... */ + { 126993 }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_hisi_sccl_l3c[] = { -{ 127355 }, /* uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\000\000\000\000\000 */ + /* uncore_hisi_l3c.rd_hit_cpipe\000uncore\000Total read hits\000event=7\000\00000\0... */ + { 127355 }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_cbox[] = { -{ 127229 }, /* event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000\000\000\000\000 */ -{ 127283 }, /* event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000\000\000\000\000 */ -{ 127075 }, /* unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted from L3 Eviction which misses in some processor core\000event=0x22,umask=0x81\000\00000\000\000\000\000\000 */ + /* event-hyphen\000uncore\000UNC_CBO_HYPHEN\000event=0xe0\000\00000\000\000\000\000... */ + { 127229 }, + /* event-two-hyph\000uncore\000UNC_CBO_TWO_HYPH\000event=0xc0\000\00000\000\000\000... */ + { 127283 }, + /* unc_cbo_xsnp_response.miss_eviction\000uncore\000A cross-core snoop resulted fro... */ + { 127075 }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_imc[] = { -{ 127538 }, /* uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000\000\000\000\000 */ + /* uncore_imc.cache_hits\000uncore\000Total cache hits\000event=0x34\000\00000\000\... */ + { 127538 }, }; static const struct compact_pmu_event pmu_events__test_soc_cpu_uncore_imc_free_running[] = { -{ 127447 }, /* uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12\000\00000\000\000\000\000\000 */ + /* uncore_imc_free_running.cache_miss\000uncore\000Total cache misses\000event=0x12... */ + { 127447 }, }; static const struct pmu_table_entry pmu_events__test_soc_cpu[] = { -{ - .entries = pmu_events__test_soc_cpu_default_core, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_default_core), - .pmu_name = { 0 /* default_core\000 */ }, -}, -{ - .entries = pmu_events__test_soc_cpu_hisi_sccl_ddrc, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_ddrc), - .pmu_name = { 126978 /* hisi_sccl,ddrc\000 */ }, -}, -{ - .entries = pmu_events__test_soc_cpu_hisi_sccl_l3c, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_l3c), - .pmu_name = { 127341 /* hisi_sccl,l3c\000 */ }, -}, -{ - .entries = pmu_events__test_soc_cpu_uncore_cbox, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_cbox), - .pmu_name = { 127063 /* uncore_cbox\000 */ }, -}, -{ - .entries = pmu_events__test_soc_cpu_uncore_imc, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc), - .pmu_name = { 127527 /* uncore_imc\000 */ }, -}, -{ - .entries = pmu_events__test_soc_cpu_uncore_imc_free_running, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc_free_running), - .pmu_name = { 127423 /* uncore_imc_free_running\000 */ }, -}, + { + .entries = pmu_events__test_soc_cpu_default_core, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_default_core), + .pmu_name = { 0 /* default_core\000 */ }, + }, + { + .entries = pmu_events__test_soc_cpu_hisi_sccl_ddrc, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_ddrc), + .pmu_name = { 126978 /* hisi_sccl,ddrc\000 */ }, + }, + { + .entries = pmu_events__test_soc_cpu_hisi_sccl_l3c, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_hisi_sccl_l3c), + .pmu_name = { 127341 /* hisi_sccl,l3c\000 */ }, + }, + { + .entries = pmu_events__test_soc_cpu_uncore_cbox, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_cbox), + .pmu_name = { 127063 /* uncore_cbox\000 */ }, + }, + { + .entries = pmu_events__test_soc_cpu_uncore_imc, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc), + .pmu_name = { 127527 /* uncore_imc\000 */ }, + }, + { + .entries = pmu_events__test_soc_cpu_uncore_imc_free_running, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_cpu_uncore_imc_free_running), + .pmu_name = { 127423 /* uncore_imc_free_running\000 */ }, + }, }; static const struct compact_pmu_event pmu_metrics__test_soc_cpu_default_core[] = { -{ 130909 }, /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000 */ -{ 131598 }, /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000 */ -{ 131368 }, /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000 */ -{ 131463 }, /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000 */ -{ 131663 }, /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ -{ 131732 }, /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ -{ 130996 }, /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000 */ -{ 130932 }, /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000 */ -{ 131870 }, /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000 */ -{ 131803 }, /* M1\000\000ipc + M2\000\000\000\000\000\000\000\000000 */ -{ 131826 }, /* M2\000\000ipc + M1\000\000\000\000\000\000\000\000000 */ -{ 131849 }, /* M3\000\0001 / M3\000\000\000\000\000\000\000\000000 */ -{ 131296 }, /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000 */ -{ 131163 }, /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ -{ 131228 }, /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ + /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000 */ + { 130909 }, + /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000... */ + { 131598 }, + /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqs... */ + { 131368 }, + /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data... */ + { 131463 }, + /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000... */ + { 131663 }, + /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\0... */ + { 131732 }, + /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.t... */ + { 130996 }, + /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\... */ + { 130932 }, + /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\... */ + { 131870 }, + /* M1\000\000ipc + M2\000\000\000\000\000\000\000\000000 */ + { 131803 }, + /* M2\000\000ipc + M1\000\000\000\000\000\000\000\000000 */ + { 131826 }, + /* M3\000\0001 / M3\000\000\000\000\000\000\000\000000 */ + { 131849 }, + /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\... */ + { 131296 }, + /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\0... */ + { 131163 }, + /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\00... */ + { 131228 }, }; static const struct pmu_table_entry pmu_metrics__test_soc_cpu[] = { -{ - .entries = pmu_metrics__test_soc_cpu_default_core, - .num_entries = ARRAY_SIZE(pmu_metrics__test_soc_cpu_default_core), - .pmu_name = { 0 /* default_core\000 */ }, -}, + { + .entries = pmu_metrics__test_soc_cpu_default_core, + .num_entries = ARRAY_SIZE(pmu_metrics__test_soc_cpu_default_core), + .pmu_name = { 0 /* default_core\000 */ }, + }, }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_ccn_pmu[] = { -{ 127717 }, /* sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x01\00000\000\000\000\000\000 */ + /* sys_ccn_pmu.read_cycles\000uncore\000ccn read-cycles event\000config=0x2c\0000x0... */ + { 127717 }, }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_cmn_pmu[] = { -{ 127813 }, /* sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first lookup result (high priority)\000eventid=1,type=5\000(434|436|43c|43a).*\00000\000\000\000\000\000 */ + /* sys_cmn_pmu.hnf_cache_miss\000uncore\000Counts total cache misses in first looku... */ + { 127813 }, }; static const struct compact_pmu_event pmu_events__test_soc_sys_uncore_sys_ddr_pmu[] = { -{ 127622 }, /* sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8\00000\000\000\000\000\000 */ + /* sys_ddr_pmu.write_cycles\000uncore\000ddr write-cycles event\000event=0x2b\000v8... */ + { 127622 }, }; static const struct pmu_table_entry pmu_events__test_soc_sys[] = { -{ - .entries = pmu_events__test_soc_sys_uncore_sys_ccn_pmu, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ccn_pmu), - .pmu_name = { 127698 /* uncore_sys_ccn_pmu\000 */ }, -}, -{ - .entries = pmu_events__test_soc_sys_uncore_sys_cmn_pmu, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_cmn_pmu), - .pmu_name = { 127794 /* uncore_sys_cmn_pmu\000 */ }, -}, -{ - .entries = pmu_events__test_soc_sys_uncore_sys_ddr_pmu, - .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ddr_pmu), - .pmu_name = { 127603 /* uncore_sys_ddr_pmu\000 */ }, -}, + { + .entries = pmu_events__test_soc_sys_uncore_sys_ccn_pmu, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ccn_pmu), + .pmu_name = { 127698 /* uncore_sys_ccn_pmu\000 */ }, + }, + { + .entries = pmu_events__test_soc_sys_uncore_sys_cmn_pmu, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_cmn_pmu), + .pmu_name = { 127794 /* uncore_sys_cmn_pmu\000 */ }, + }, + { + .entries = pmu_events__test_soc_sys_uncore_sys_ddr_pmu, + .num_entries = ARRAY_SIZE(pmu_events__test_soc_sys_uncore_sys_ddr_pmu), + .pmu_name = { 127603 /* uncore_sys_ddr_pmu\000 */ }, + }, }; /* Struct used to make the PMU event table implementation opaque to callers. */ struct pmu_events_table { - const struct pmu_table_entry *pmus; - uint32_t num_pmus; + const struct pmu_table_entry *pmus; + uint32_t num_pmus; }; /* Struct used to make the PMU metric table implementation opaque to callers. */ struct pmu_metrics_table { - const struct pmu_table_entry *pmus; - uint32_t num_pmus; + const char *name; + const struct pmu_table_entry *pmus; + uint32_t num_pmus; }; /* @@ -2791,10 +5417,10 @@ struct pmu_metrics_table { * The cpuid can contain any character other than the comma. */ struct pmu_events_map { - const char *arch; - const char *cpuid; - struct pmu_events_table event_table; - struct pmu_metrics_table metric_table; + const char *arch; + const char *cpuid; + struct pmu_events_table event_table; + struct pmu_metrics_table metric_table; }; /* @@ -2810,6 +5436,7 @@ static const struct pmu_events_map pmu_events_map[] = { .num_pmus = ARRAY_SIZE(pmu_events__common), }, .metric_table = { + .name = "common", .pmus = pmu_metrics__common, .num_pmus = ARRAY_SIZE(pmu_metrics__common), }, @@ -2822,6 +5449,7 @@ static const struct pmu_events_map pmu_events_map[] = { .num_pmus = ARRAY_SIZE(pmu_events__test_soc_cpu), }, .metric_table = { + .name = "test_soc_cpu", .pmus = pmu_metrics__test_soc_cpu, .num_pmus = ARRAY_SIZE(pmu_metrics__test_soc_cpu), } @@ -2830,7 +5458,7 @@ static const struct pmu_events_map pmu_events_map[] = { .arch = 0, .cpuid = 0, .event_table = { 0, 0 }, - .metric_table = { 0, 0 }, + .metric_table = { 0 }, } }; @@ -2850,7 +5478,7 @@ static const struct pmu_sys_events pmu_sys_event_tables[] = { }, { .event_table = { 0, 0 }, - .metric_table = { 0, 0 }, + .metric_table = { 0 }, }, }; @@ -2915,455 +5543,493 @@ static void decompress_metric(int offset, struct pmu_metric *pm) } static int pmu_events_table__for_each_event_pmu(const struct pmu_events_table *table, - const struct pmu_table_entry *pmu, - pmu_event_iter_fn fn, - void *data) + const struct pmu_table_entry *pmu, + pmu_event_iter_fn fn, + void *data) { - int ret; - struct pmu_event pe = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - - for (uint32_t i = 0; i < pmu->num_entries; i++) { - decompress_event(pmu->entries[i].offset, &pe); - if (!pe.name) - continue; - ret = fn(&pe, table, data); - if (ret) - return ret; - } - return 0; + int ret; + struct pmu_event pe = { + .pmu = &big_c_string[pmu->pmu_name.offset], + }; + + for (uint32_t i = 0; i < pmu->num_entries; i++) { + decompress_event(pmu->entries[i].offset, &pe); + if (!pe.name) + continue; + ret = fn(&pe, table, data); + if (ret) + return ret; + } + return 0; } static int pmu_events_table__find_event_pmu(const struct pmu_events_table *table, - const struct pmu_table_entry *pmu, - const char *name, - pmu_event_iter_fn fn, - void *data) + const struct pmu_table_entry *pmu, + const char *name, + pmu_event_iter_fn fn, + void *data) { - struct pmu_event pe = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - int low = 0, high = pmu->num_entries - 1; - - while (low <= high) { - int cmp, mid = (low + high) / 2; - - decompress_event(pmu->entries[mid].offset, &pe); - - if (!pe.name && !name) - goto do_call; - - if (!pe.name && name) { - low = mid + 1; - continue; - } - if (pe.name && !name) { - high = mid - 1; - continue; - } - - cmp = strcasecmp(pe.name, name); - if (cmp < 0) { - low = mid + 1; - continue; - } - if (cmp > 0) { - high = mid - 1; - continue; - } + struct pmu_event pe = { + .pmu = &big_c_string[pmu->pmu_name.offset], + }; + int low = 0, high = pmu->num_entries - 1; + + while (low <= high) { + int cmp, mid = (low + high) / 2; + + decompress_event(pmu->entries[mid].offset, &pe); + + if (!pe.name && !name) + goto do_call; + + if (!pe.name && name) { + low = mid + 1; + continue; + } + if (pe.name && !name) { + high = mid - 1; + continue; + } + + cmp = strcasecmp(pe.name, name); + if (cmp < 0) { + low = mid + 1; + continue; + } + if (cmp > 0) { + high = mid - 1; + continue; + } do_call: - return fn ? fn(&pe, table, data) : 0; - } - return PMU_EVENTS__NOT_FOUND; + return fn ? fn(&pe, table, data) : 0; + } + return PMU_EVENTS__NOT_FOUND; } int pmu_events_table__for_each_event(const struct pmu_events_table *table, - struct perf_pmu *pmu, - pmu_event_iter_fn fn, - void *data) + struct perf_pmu *pmu, + pmu_event_iter_fn fn, + void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); - if (ret) - return ret; - } - return 0; + if (!table) + return 0; + for (size_t i = 0; i < table->num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &table->pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + int ret; + + if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) + continue; + + ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); + if (ret) + return ret; + } + return 0; } int pmu_events_table__find_event(const struct pmu_events_table *table, - struct perf_pmu *pmu, - const char *name, - pmu_event_iter_fn fn, - void *data) + struct perf_pmu *pmu, + const char *name, + pmu_event_iter_fn fn, + void *data) { - if (!table) - return PMU_EVENTS__NOT_FOUND; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data); - if (ret != PMU_EVENTS__NOT_FOUND) - return ret; - } - return PMU_EVENTS__NOT_FOUND; + if (!table) + return PMU_EVENTS__NOT_FOUND; + for (size_t i = 0; i < table->num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &table->pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + int ret; + + if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) + continue; + + ret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data); + if (ret != PMU_EVENTS__NOT_FOUND) + return ret; + } + return PMU_EVENTS__NOT_FOUND; } -size_t pmu_events_table__num_events(const struct pmu_events_table *table, - struct perf_pmu *pmu) +size_t pmu_events_table__num_events(const struct pmu_events_table *table, struct perf_pmu *pmu) { - size_t count = 0; - - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - - if (perf_pmu__name_wildcard_match(pmu, pmu_name)) - count += table_pmu->num_entries; - } - return count; + size_t count = 0; + + if (!table) + return 0; + for (size_t i = 0; i < table->num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &table->pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + + if (perf_pmu__name_wildcard_match(pmu, pmu_name)) + count += table_pmu->num_entries; + } + return count; } static int pmu_metrics_table__for_each_metric_pmu(const struct pmu_metrics_table *table, - const struct pmu_table_entry *pmu, - pmu_metric_iter_fn fn, - void *data) + const struct pmu_table_entry *pmu, + pmu_metric_iter_fn fn, + void *data) { - int ret; - struct pmu_metric pm = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - - for (uint32_t i = 0; i < pmu->num_entries; i++) { - decompress_metric(pmu->entries[i].offset, &pm); - if (!pm.metric_expr) - continue; - ret = fn(&pm, table, data); - if (ret) - return ret; - } - return 0; + int ret; + struct pmu_metric pm = { + .pmu = &big_c_string[pmu->pmu_name.offset], + }; + + for (uint32_t i = 0; i < pmu->num_entries; i++) { + decompress_metric(pmu->entries[i].offset, &pm); + if (!pm.metric_expr) + continue; + ret = fn(&pm, table, data); + if (ret) + return ret; + } + return 0; } static int pmu_metrics_table__find_metric_pmu(const struct pmu_metrics_table *table, - const struct pmu_table_entry *pmu, - const char *metric, - pmu_metric_iter_fn fn, - void *data) + const struct pmu_table_entry *pmu, + const char *metric, + pmu_metric_iter_fn fn, + void *data) { - struct pmu_metric pm = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - int low = 0, high = pmu->num_entries - 1; - - while (low <= high) { - int cmp, mid = (low + high) / 2; - - decompress_metric(pmu->entries[mid].offset, &pm); - - if (!pm.metric_name && !metric) - goto do_call; - - if (!pm.metric_name && metric) { - low = mid + 1; - continue; - } - if (pm.metric_name && !metric) { - high = mid - 1; - continue; - } - - cmp = strcmp(pm.metric_name, metric); - if (cmp < 0) { - low = mid + 1; - continue; - } - if (cmp > 0) { - high = mid - 1; - continue; - } + struct pmu_metric pm = { + .pmu = &big_c_string[pmu->pmu_name.offset], + }; + int low = 0, high = pmu->num_entries - 1; + + while (low <= high) { + int cmp, mid = (low + high) / 2; + + decompress_metric(pmu->entries[mid].offset, &pm); + + if (!pm.metric_name && !metric) + goto do_call; + + if (!pm.metric_name && metric) { + low = mid + 1; + continue; + } + if (pm.metric_name && !metric) { + high = mid - 1; + continue; + } + + cmp = strcmp(pm.metric_name, metric); + if (cmp < 0) { + low = mid + 1; + continue; + } + if (cmp > 0) { + high = mid - 1; + continue; + } do_call: - return fn ? fn(&pm, table, data) : 0; - } - return PMU_METRICS__NOT_FOUND; + return fn ? fn(&pm, table, data) : 0; + } + return PMU_METRICS__NOT_FOUND; } int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, - pmu_metric_iter_fn fn, - void *data) + pmu_metric_iter_fn fn, + void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - int ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i], - fn, data); - - if (ret) - return ret; - } - return 0; + if (!table) + return 0; + for (size_t i = 0; i < table->num_pmus; i++) { + int ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i], fn, data); + + if (ret) + return ret; + } + return 0; } int pmu_metrics_table__find_metric(const struct pmu_metrics_table *table, - struct perf_pmu *pmu, - const char *metric, - pmu_metric_iter_fn fn, - void *data) + struct perf_pmu *pmu, + const char *metric, + pmu_metric_iter_fn fn, + void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_metrics_table__find_metric_pmu(table, table_pmu, metric, fn, data); - if (ret != PMU_METRICS__NOT_FOUND) - return ret; - } - return PMU_METRICS__NOT_FOUND; + if (!table) + return 0; + for (size_t i = 0; i < table->num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &table->pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + int ret; + + if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) + continue; + + ret = pmu_metrics_table__find_metric_pmu(table, table_pmu, metric, fn, data); + if (ret != PMU_METRICS__NOT_FOUND) + return ret; + } + return PMU_METRICS__NOT_FOUND; } static const struct pmu_events_map *map_for_cpu(struct perf_cpu cpu) { - static struct { - const struct pmu_events_map *map; - struct perf_cpu cpu; - } last_result; - static struct { - const struct pmu_events_map *map; - char *cpuid; - } last_map_search; - static bool has_last_result, has_last_map_search; - const struct pmu_events_map *map = NULL; - char *cpuid = NULL; - size_t i; - - if (has_last_result && last_result.cpu.cpu == cpu.cpu) - return last_result.map; - - cpuid = get_cpuid_allow_env_override(cpu); - - /* - * On some platforms which uses cpus map, cpuid can be NULL for - * PMUs other than CORE PMUs. - */ - if (!cpuid) - goto out_update_last_result; - - if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) { - map = last_map_search.map; - free(cpuid); - } else { - i = 0; - for (;;) { - map = &pmu_events_map[i++]; - - if (!map->arch) { - map = NULL; - break; - } - - if (!strcmp_cpuid_str(map->cpuid, cpuid)) - break; - } - free(last_map_search.cpuid); - last_map_search.cpuid = cpuid; - last_map_search.map = map; - has_last_map_search = true; - } + static struct { + const struct pmu_events_map *map; + struct perf_cpu cpu; + } last_result; + static struct { + const struct pmu_events_map *map; + char *cpuid; + } last_map_search; + static bool has_last_result, has_last_map_search; + const struct pmu_events_map *map = NULL; + char *cpuid = NULL; + size_t i; + + if (has_last_result && last_result.cpu.cpu == cpu.cpu) + return last_result.map; + + cpuid = get_cpuid_allow_env_override(cpu); + + /* + * On some platforms which uses cpus map, cpuid can be NULL for + * PMUs other than CORE PMUs. + */ + if (!cpuid) + goto out_update_last_result; + + if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) { + map = last_map_search.map; + free(cpuid); + } else { + i = 0; + for (;;) { + map = &pmu_events_map[i++]; + + if (!map->arch) { + map = NULL; + break; + } + + if (!strcmp_cpuid_str(map->cpuid, cpuid)) + break; + } + free(last_map_search.cpuid); + last_map_search.cpuid = cpuid; + last_map_search.map = map; + has_last_map_search = true; + } out_update_last_result: - last_result.cpu = cpu; - last_result.map = map; - has_last_result = true; - return map; + last_result.cpu = cpu; + last_result.map = map; + has_last_result = true; + return map; } static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) { - struct perf_cpu cpu = {-1}; - - if (pmu) { - for (size_t i = 0; i < ARRAY_SIZE(pmu_events__common); i++) { - const char *pmu_name = &big_c_string[pmu_events__common[i].pmu_name.offset]; - - if (!strcmp(pmu_name, pmu->name)) { - const struct pmu_events_map *map = &pmu_events_map[0]; - - while (strcmp("common", map->arch)) - map++; - return map; - } - } - cpu = perf_cpu_map__min(pmu->cpus); - } - return map_for_cpu(cpu); + struct perf_cpu cpu = { -1 }; + + if (pmu) { + for (size_t i = 0; i < ARRAY_SIZE(pmu_events__common); i++) { + const char *pmu_name = &big_c_string[pmu_events__common[i].pmu_name.offset]; + + if (!strcmp(pmu_name, pmu->name)) { + const struct pmu_events_map *map = &pmu_events_map[0]; + + while (strcmp("common", map->arch)) + map++; + return map; + } + } + cpu = perf_cpu_map__min(pmu->cpus); + } + return map_for_cpu(cpu); } const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) { - const struct pmu_events_map *map = map_for_pmu(pmu); + const struct pmu_events_map *map = map_for_pmu(pmu); - if (!map) - return NULL; + if (!map) + return NULL; - if (!pmu) - return &map->event_table; + if (!pmu) + return &map->event_table; - for (size_t i = 0; i < map->event_table.num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + for (size_t i = 0; i < map->event_table.num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - if (perf_pmu__name_wildcard_match(pmu, pmu_name)) - return &map->event_table; - } - return NULL; + if (perf_pmu__name_wildcard_match(pmu, pmu_name)) + return &map->event_table; + } + return NULL; } const struct pmu_events_table *perf_pmu__default_core_events_table(void) { - int i = 0; + int i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; + for (;;) { + const struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; + if (!map->arch) + break; - if (!strcmp(map->cpuid, "common")) - return &map->event_table; - } - return NULL; + if (!strcmp(map->cpuid, "common")) + return &map->event_table; + } + return NULL; } const struct pmu_metrics_table *pmu_metrics_table__find(void) { - struct perf_cpu cpu = {-1}; - const struct pmu_events_map *map = map_for_cpu(cpu); + struct perf_cpu cpu = { -1 }; + const struct pmu_events_map *map = map_for_cpu(cpu); - return map ? &map->metric_table : NULL; + return map ? &map->metric_table : NULL; } const struct pmu_metrics_table *pmu_metrics_table__default(void) { - int i = 0; + int i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; + for (;;) { + const struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; + if (!map->arch) + break; - if (!strcmp(map->cpuid, "common")) - return &map->metric_table; - } - return NULL; + if (!strcmp(map->cpuid, "common")) + return &map->metric_table; + } + return NULL; } const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) - return &tables->event_table; - } - return NULL; + for (const struct pmu_events_map *tables = &pmu_events_map[0]; + tables->arch; + tables++) { + if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) + return &tables->event_table; + } + return NULL; } const struct pmu_metrics_table *find_core_metrics_table(const char *arch, const char *cpuid) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) - return &tables->metric_table; - } - return NULL; + for (const struct pmu_events_map *tables = &pmu_events_map[0]; + tables->arch; + tables++) { + if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) + return &tables->metric_table; + } + return NULL; } int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - int ret = pmu_events_table__for_each_event(&tables->event_table, - /*pmu=*/ NULL, fn, data); - - if (ret) - return ret; - } - return 0; + for (const struct pmu_events_map *tables = &pmu_events_map[0]; + tables->arch; + tables++) { + int ret = pmu_events_table__for_each_event(&tables->event_table, + /*pmu=*/NULL, fn, data); + + if (ret) + return ret; + } + return 0; } int pmu_for_each_core_metric(pmu_metric_iter_fn fn, void *data) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); - - if (ret) - return ret; - } - return 0; + for (const struct pmu_events_map *tables = &pmu_events_map[0]; + tables->arch; + tables++) { + int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); + + if (ret) + return ret; + } + return 0; } const struct pmu_events_table *find_sys_events_table(const char *name) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - if (!strcmp(tables->name, name)) - return &tables->event_table; - } - return NULL; + for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; + tables->name; + tables++) { + if (!strcmp(tables->name, name)) + return &tables->event_table; + } + return NULL; } int pmu_for_each_sys_event(pmu_event_iter_fn fn, void *data) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - int ret = pmu_events_table__for_each_event(&tables->event_table, - /*pmu=*/ NULL, fn, data); - - if (ret) - return ret; - } - return 0; + for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; + tables->name; + tables++) { + int ret = pmu_events_table__for_each_event(&tables->event_table, + /*pmu=*/NULL, fn, data); + + if (ret) + return ret; + } + return 0; } int pmu_for_each_sys_metric(pmu_metric_iter_fn fn, void *data) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); - - if (ret) - return ret; - } - return 0; + for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; + tables->name; + tables++) { + int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); + + if (ret) + return ret; + } + return 0; +} +/* clang-format on */ + +const char *pmu_metrics_table__name(const struct pmu_metrics_table *table) +{ + return table ? table->name : NULL; +} + +int pmu_metrics_table__iterate_tables(pmu_metrics_table_iter_t fn, void *data) +{ + size_t i; + int ret; + + for (i = 0; pmu_events_map[i].cpuid; i++) { + size_t j; + bool found = false; + + if (!pmu_events_map[i].metric_table.pmus) + continue; + for (j = 0; j < i; j++) { + if (pmu_events_map[j].metric_table.pmus == + pmu_events_map[i].metric_table.pmus) { + found = true; + break; + } + } + if (found) + continue; + ret = fn(&pmu_events_map[i].metric_table, data); + if (ret) + return ret; + } + for (i = 0; pmu_sys_event_tables[i].name; i++) { + if (!pmu_sys_event_tables[i].metric_table.pmus) + continue; + ret = fn(&pmu_sys_event_tables[i].metric_table, data); + if (ret) + return ret; + } + return 0; } static const int metricgroups[][2] = { @@ -3372,20 +6038,19 @@ static const int metricgroups[][2] = { const char *describe_metricgroup(const char *group) { - int low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1; - - while (low <= high) { - int mid = (low + high) / 2; - const char *mgroup = &big_c_string[metricgroups[mid][0]]; - int cmp = strcmp(mgroup, group); - - if (cmp == 0) { - return &big_c_string[metricgroups[mid][1]]; - } else if (cmp < 0) { - low = mid + 1; - } else { - high = mid - 1; - } - } - return NULL; + int low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1; + + while (low <= high) { + int mid = (low + high) / 2; + const char *mgroup = &big_c_string[metricgroups[mid][0]]; + int cmp = strcmp(mgroup, group); + + if (cmp == 0) + return &big_c_string[metricgroups[mid][1]]; + else if (cmp < 0) + low = mid + 1; + else + high = mid - 1; + } + return NULL; } diff --git a/tools/perf/pmu-events/intel_metrics.py b/tools/perf/pmu-events/intel_metrics.py index 52035433b505..bc2b920d3a0d 100755 --- a/tools/perf/pmu-events/intel_metrics.py +++ b/tools/perf/pmu-events/intel_metrics.py @@ -7,7 +7,7 @@ import re from typing import Optional from common_metrics import Cycles -from metric import (d_ratio, has_event, max, source_count, CheckPmu, Event, +from metric import (d_ratio, has_event, max, aggr_nr, CheckPmu, Event, JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, Literal, LoadEvents, Metric, MetricConstraint, MetricGroup, MetricRef, Select) @@ -457,6 +457,67 @@ def IntelIlp() -> MetricGroup: ]) +def IntelIotlb() -> Optional[MetricGroup]: + try: + total_hit = ( + Event("UNC_IIO_IOMMU0.4K_HITS") + + Event("UNC_IIO_IOMMU0.2M_HITS") + + Event("UNC_IIO_IOMMU0.1G_HITS") + ) + total_miss = Event("UNC_IIO_IOMMU0.MISSES") + except: + return None + + miss_rate = d_ratio(total_miss, total_miss + total_hit) + metrics = [ + Metric("iotlb_total_hit", "IOTLB total hit", total_hit, "hits"), + Metric("iotlb_total_miss", "IOTLB total miss", total_miss, "misses"), + Metric("iotlb_miss_rate", "IOTLB miss rate", miss_rate, "100%"), + ] + + try: + interrupt_cache_hit = Event("UNC_IIO_IOMMU3.INT_CACHE_HITS") + interrupt_cache_lookup = Event("UNC_IIO_IOMMU3.INT_CACHE_LOOKUPS") + interrupt_cache_miss = max(interrupt_cache_lookup - interrupt_cache_hit, 0) + interrupt_cache_miss_rate = d_ratio( + interrupt_cache_miss, interrupt_cache_miss + interrupt_cache_hit + ) + metrics += [ + Metric( + "iotlb_interrupt_cache_hit", + "IOTLB interrupt cache hit", + interrupt_cache_hit, + "hits", + ), + Metric( + "iotlb_interrupt_cache_miss", + "IOTLB interrupt cache miss", + interrupt_cache_miss, + "misses", + ), + Metric( + "iotlb_interrupt_cache_lookup", + "IOTLB interrupt cache lookup", + interrupt_cache_lookup, + "lookups", + ), + Metric( + "iotlb_interrupt_cache_miss_rate", + "IOTLB interrupt cache miss rate", + interrupt_cache_miss_rate, + "100%", + ), + ] + except: + pass + + return MetricGroup( + "iotlb", + metrics, + description="IOMMU TLB metrics", + ) + + def IntelL2() -> Optional[MetricGroup]: try: DC_HIT = Event("L2_RQSTS.DEMAND_DATA_RD_HIT") @@ -674,10 +735,10 @@ def IntelMissLat() -> Optional[MetricGroup]: else: assert data_rd_loc_occ.name == "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL", data_rd_loc_occ - ticks_per_cha = ticks / source_count(data_rd_loc_ins) + ticks_per_cha = ticks / aggr_nr(data_rd_loc_ins) loc_lat = interval_sec * 1e9 * data_rd_loc_occ / \ (ticks_per_cha * data_rd_loc_ins) - ticks_per_cha = ticks / source_count(data_rd_rem_ins) + ticks_per_cha = ticks / aggr_nr(data_rd_rem_ins) rem_lat = interval_sec * 1e9 * data_rd_rem_occ / \ (ticks_per_cha * data_rd_rem_ins) return MetricGroup("lpm_miss_lat", [ @@ -1105,6 +1166,7 @@ def dir_path(path: str) -> str: IntelCtxSw(), IntelFpu(), IntelIlp(), + IntelIotlb(), IntelL2(), IntelLdSt(), IntelMissLat(), diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index 3a1bcdcdc685..376dc2d24162 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -183,7 +183,7 @@ def string_cmp_key(s: str) -> Tuple[bool, int, str]: for s in sorted(self.strings, key=string_cmp_key): if s not in folded_strings: self.offsets[s] = big_string_offset - self.big_string.append(f'/* offset={big_string_offset} */ "') + self.big_string.append(f'/* offset={big_string_offset} */\n"') self.big_string.append(s) self.big_string.append('"') if s in fold_into_strings: @@ -450,11 +450,12 @@ def build_c_string(self, metric: bool) -> str: def to_c_string(self, metric: bool) -> str: """Representation of the event as a C struct initializer.""" - def fix_comment(s: str) -> str: - return s.replace('*/', r'\*\/') + def make_comment(s: str) -> str: + s = s.replace('*/', r'\*\/') + return f'\t/* {s} */\n' if len(s) < 80 else f'\t/* {s[0:80]}... */\n' s = self.build_c_string(metric) - return f'{{ { _bcs.offsets[s] } }}, /* {fix_comment(s)} */\n' + return f'{make_comment(s)}\t{{ { _bcs.offsets[s] } }},\n' @lru_cache(maxsize=None) @@ -558,11 +559,11 @@ def fix_none(s: Optional[str]) -> str: """) for (pmu, tbl_pmu) in sorted(pmus): pmu_name = f"{pmu}\\000" - _args.output_file.write(f"""{{ - .entries = {_pending_events_tblname}_{tbl_pmu}, - .num_entries = ARRAY_SIZE({_pending_events_tblname}_{tbl_pmu}), - .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }}, -}}, + _args.output_file.write(f"""\t{{ +\t\t.entries = {_pending_events_tblname}_{tbl_pmu}, +\t\t.num_entries = ARRAY_SIZE({_pending_events_tblname}_{tbl_pmu}), +\t\t.pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }}, +\t}}, """) _args.output_file.write('};\n\n') @@ -613,11 +614,11 @@ def fix_none(s: Optional[str]) -> str: """) for (pmu, tbl_pmu) in sorted(pmus): pmu_name = f"{pmu}\\000" - _args.output_file.write(f"""{{ - .entries = {_pending_metrics_tblname}_{tbl_pmu}, - .num_entries = ARRAY_SIZE({_pending_metrics_tblname}_{tbl_pmu}), - .pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }}, -}}, + _args.output_file.write(f"""\t{{ +\t\t.entries = {_pending_metrics_tblname}_{tbl_pmu}, +\t\t.num_entries = ARRAY_SIZE({_pending_metrics_tblname}_{tbl_pmu}), +\t\t.pmu_name = {{ {_bcs.offsets[pmu_name]} /* {pmu_name} */ }}, +\t}}, """) _args.output_file.write('};\n\n') @@ -705,14 +706,15 @@ def print_mapping_table(archs: Sequence[str]) -> None: _args.output_file.write(""" /* Struct used to make the PMU event table implementation opaque to callers. */ struct pmu_events_table { - const struct pmu_table_entry *pmus; - uint32_t num_pmus; +\tconst struct pmu_table_entry *pmus; +\tuint32_t num_pmus; }; /* Struct used to make the PMU metric table implementation opaque to callers. */ struct pmu_metrics_table { - const struct pmu_table_entry *pmus; - uint32_t num_pmus; +\tconst char *name; +\tconst struct pmu_table_entry *pmus; +\tuint32_t num_pmus; }; /* @@ -724,10 +726,10 @@ def print_mapping_table(archs: Sequence[str]) -> None: * The cpuid can contain any character other than the comma. */ struct pmu_events_map { - const char *arch; - const char *cpuid; - struct pmu_events_table event_table; - struct pmu_metrics_table metric_table; +\tconst char *arch; +\tconst char *cpuid; +\tstruct pmu_events_table event_table; +\tstruct pmu_metrics_table metric_table; }; /* @@ -746,6 +748,7 @@ def print_mapping_table(archs: Sequence[str]) -> None: \t\t.num_pmus = ARRAY_SIZE(pmu_events__test_soc_cpu), \t}, \t.metric_table = { +\t\t.name = "test_soc_cpu", \t\t.pmus = pmu_metrics__test_soc_cpu, \t\t.num_pmus = ARRAY_SIZE(pmu_metrics__test_soc_cpu), \t} @@ -760,6 +763,7 @@ def print_mapping_table(archs: Sequence[str]) -> None: \t\t.num_pmus = ARRAY_SIZE(pmu_events__common), \t}, \t.metric_table = { +\t\t.name = "common", \t\t.pmus = pmu_metrics__common, \t\t.num_pmus = ARRAY_SIZE(pmu_metrics__common), \t}, @@ -780,8 +784,10 @@ def print_mapping_table(archs: Sequence[str]) -> None: event_size = '0' metric_tblname = file_name_to_table_name('pmu_metrics_', [], row[2].replace('/', '_')) if metric_tblname in _metric_tables: + metric_name = f'"{metric_tblname.replace("pmu_metrics__", "")}"' metric_size = f'ARRAY_SIZE({metric_tblname})' else: + metric_name = 'NULL' metric_tblname = 'NULL' metric_size = '0' if event_size == '0' and metric_size == '0': @@ -795,6 +801,7 @@ def print_mapping_table(archs: Sequence[str]) -> None: \t\t.num_pmus = {event_size} \t}}, \t.metric_table = {{ +\t\t.name = {metric_name}, \t\t.pmus = {metric_tblname}, \t\t.num_pmus = {metric_size} \t}} @@ -806,12 +813,55 @@ def print_mapping_table(archs: Sequence[str]) -> None: \t.arch = 0, \t.cpuid = 0, \t.event_table = { 0, 0 }, -\t.metric_table = { 0, 0 }, +\t.metric_table = { 0 }, } }; """) +def print_metric_table_functions() -> None: + _args.output_file.write(""" +const char *pmu_metrics_table__name(const struct pmu_metrics_table *table) +{ +\treturn table ? table->name : NULL; +} + +int pmu_metrics_table__iterate_tables(pmu_metrics_table_iter_t fn, void *data) +{ +\tsize_t i; +\tint ret; + +\tfor (i = 0; pmu_events_map[i].cpuid; i++) { +\t\tsize_t j; +\t\tbool found = false; + +\t\tif (!pmu_events_map[i].metric_table.pmus) +\t\t\tcontinue; +\t\tfor (j = 0; j < i; j++) { +\t\t\tif (pmu_events_map[j].metric_table.pmus == +\t\t\t pmu_events_map[i].metric_table.pmus) { +\t\t\t\tfound = true; +\t\t\t\tbreak; +\t\t\t} +\t\t} +\t\tif (found) +\t\t\tcontinue; +\t\tret = fn(&pmu_events_map[i].metric_table, data); +\t\tif (ret) +\t\t\treturn ret; +\t} +\tfor (i = 0; pmu_sys_event_tables[i].name; i++) { +\t\tif (!pmu_sys_event_tables[i].metric_table.pmus) +\t\t\tcontinue; +\t\tret = fn(&pmu_sys_event_tables[i].metric_table, data); +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; +} +""") + + def print_system_mapping_table() -> None: """C struct mapping table array for tables from /sys directories.""" _args.output_file.write(""" @@ -834,6 +884,7 @@ def print_system_mapping_table() -> None: if metric_tblname in _sys_metric_tables: _args.output_file.write(f""" \t\t.metric_table = {{ +\t\t\t.name = "{metric_tblname.replace('pmu_metrics__', '')}", \t\t\t.pmus = {metric_tblname}, \t\t\t.num_pmus = ARRAY_SIZE({metric_tblname}) \t\t}},""") @@ -847,6 +898,7 @@ def print_system_mapping_table() -> None: continue _args.output_file.write(f"""\t{{ \t\t.metric_table = {{ +\t\t\t.name = "{tblname.replace('pmu_metrics__', '')}", \t\t\t.pmus = {tblname}, \t\t\t.num_pmus = ARRAY_SIZE({tblname}) \t\t}}, @@ -855,7 +907,7 @@ def print_system_mapping_table() -> None: """) _args.output_file.write("""\t{ \t\t.event_table = { 0, 0 }, -\t\t.metric_table = { 0, 0 }, +\t\t.metric_table = { 0 }, \t}, }; @@ -896,455 +948,453 @@ def print_system_mapping_table() -> None: _args.output_file.write("""} static int pmu_events_table__for_each_event_pmu(const struct pmu_events_table *table, - const struct pmu_table_entry *pmu, - pmu_event_iter_fn fn, - void *data) +\t\t\t\t\t\tconst struct pmu_table_entry *pmu, +\t\t\t\t\t\tpmu_event_iter_fn fn, +\t\t\t\t\t\tvoid *data) { - int ret; - struct pmu_event pe = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - - for (uint32_t i = 0; i < pmu->num_entries; i++) { - decompress_event(pmu->entries[i].offset, &pe); - if (!pe.name) - continue; - ret = fn(&pe, table, data); - if (ret) - return ret; - } - return 0; +\tint ret; +\tstruct pmu_event pe = { +\t\t.pmu = &big_c_string[pmu->pmu_name.offset], +\t}; + +\tfor (uint32_t i = 0; i < pmu->num_entries; i++) { +\t\tdecompress_event(pmu->entries[i].offset, &pe); +\t\tif (!pe.name) +\t\t\tcontinue; +\t\tret = fn(&pe, table, data); +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } static int pmu_events_table__find_event_pmu(const struct pmu_events_table *table, - const struct pmu_table_entry *pmu, - const char *name, - pmu_event_iter_fn fn, - void *data) +\t\t\t\t\t const struct pmu_table_entry *pmu, +\t\t\t\t\t const char *name, +\t\t\t\t\t pmu_event_iter_fn fn, +\t\t\t\t\t void *data) { - struct pmu_event pe = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - int low = 0, high = pmu->num_entries - 1; - - while (low <= high) { - int cmp, mid = (low + high) / 2; - - decompress_event(pmu->entries[mid].offset, &pe); - - if (!pe.name && !name) - goto do_call; - - if (!pe.name && name) { - low = mid + 1; - continue; - } - if (pe.name && !name) { - high = mid - 1; - continue; - } - - cmp = strcasecmp(pe.name, name); - if (cmp < 0) { - low = mid + 1; - continue; - } - if (cmp > 0) { - high = mid - 1; - continue; - } +\tstruct pmu_event pe = { +\t\t.pmu = &big_c_string[pmu->pmu_name.offset], +\t}; +\tint low = 0, high = pmu->num_entries - 1; + +\twhile (low <= high) { +\t\tint cmp, mid = (low + high) / 2; + +\t\tdecompress_event(pmu->entries[mid].offset, &pe); + +\t\tif (!pe.name && !name) +\t\t\tgoto do_call; + +\t\tif (!pe.name && name) { +\t\t\tlow = mid + 1; +\t\t\tcontinue; +\t\t} +\t\tif (pe.name && !name) { +\t\t\thigh = mid - 1; +\t\t\tcontinue; +\t\t} + +\t\tcmp = strcasecmp(pe.name, name); +\t\tif (cmp < 0) { +\t\t\tlow = mid + 1; +\t\t\tcontinue; +\t\t} +\t\tif (cmp > 0) { +\t\t\thigh = mid - 1; +\t\t\tcontinue; +\t\t} do_call: - return fn ? fn(&pe, table, data) : 0; - } - return PMU_EVENTS__NOT_FOUND; +\t\treturn fn ? fn(&pe, table, data) : 0; +\t} +\treturn PMU_EVENTS__NOT_FOUND; } int pmu_events_table__for_each_event(const struct pmu_events_table *table, - struct perf_pmu *pmu, - pmu_event_iter_fn fn, - void *data) +\t\t\t\t struct perf_pmu *pmu, +\t\t\t\t pmu_event_iter_fn fn, +\t\t\t\t void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); - if (ret) - return ret; - } - return 0; +\tif (!table) +\t\treturn 0; +\tfor (size_t i = 0; i < table->num_pmus; i++) { +\t\tconst struct pmu_table_entry *table_pmu = &table->pmus[i]; +\t\tconst char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; +\t\tint ret; + +\t\tif (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) +\t\t\tcontinue; + +\t\tret = pmu_events_table__for_each_event_pmu(table, table_pmu, fn, data); +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } int pmu_events_table__find_event(const struct pmu_events_table *table, - struct perf_pmu *pmu, - const char *name, - pmu_event_iter_fn fn, - void *data) +\t\t\t\t struct perf_pmu *pmu, +\t\t\t\t const char *name, +\t\t\t\t pmu_event_iter_fn fn, +\t\t\t\t void *data) { - if (!table) - return PMU_EVENTS__NOT_FOUND; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data); - if (ret != PMU_EVENTS__NOT_FOUND) - return ret; - } - return PMU_EVENTS__NOT_FOUND; +\tif (!table) +\t\treturn PMU_EVENTS__NOT_FOUND; +\tfor (size_t i = 0; i < table->num_pmus; i++) { +\t\tconst struct pmu_table_entry *table_pmu = &table->pmus[i]; +\t\tconst char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; +\t\tint ret; + +\t\tif (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) +\t\t\tcontinue; + +\t\tret = pmu_events_table__find_event_pmu(table, table_pmu, name, fn, data); +\t\tif (ret != PMU_EVENTS__NOT_FOUND) +\t\t\treturn ret; +\t} +\treturn PMU_EVENTS__NOT_FOUND; } -size_t pmu_events_table__num_events(const struct pmu_events_table *table, - struct perf_pmu *pmu) +size_t pmu_events_table__num_events(const struct pmu_events_table *table, struct perf_pmu *pmu) { - size_t count = 0; - - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - - if (perf_pmu__name_wildcard_match(pmu, pmu_name)) - count += table_pmu->num_entries; - } - return count; +\tsize_t count = 0; + +\tif (!table) +\t\treturn 0; +\tfor (size_t i = 0; i < table->num_pmus; i++) { +\t\tconst struct pmu_table_entry *table_pmu = &table->pmus[i]; +\t\tconst char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + +\t\tif (perf_pmu__name_wildcard_match(pmu, pmu_name)) +\t\t\tcount += table_pmu->num_entries; +\t} +\treturn count; } static int pmu_metrics_table__for_each_metric_pmu(const struct pmu_metrics_table *table, - const struct pmu_table_entry *pmu, - pmu_metric_iter_fn fn, - void *data) +\t\t\t\t\t\tconst struct pmu_table_entry *pmu, +\t\t\t\t\t\tpmu_metric_iter_fn fn, +\t\t\t\t\t\tvoid *data) { - int ret; - struct pmu_metric pm = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - - for (uint32_t i = 0; i < pmu->num_entries; i++) { - decompress_metric(pmu->entries[i].offset, &pm); - if (!pm.metric_expr) - continue; - ret = fn(&pm, table, data); - if (ret) - return ret; - } - return 0; +\tint ret; +\tstruct pmu_metric pm = { +\t\t.pmu = &big_c_string[pmu->pmu_name.offset], +\t}; + +\tfor (uint32_t i = 0; i < pmu->num_entries; i++) { +\t\tdecompress_metric(pmu->entries[i].offset, &pm); +\t\tif (!pm.metric_expr) +\t\t\tcontinue; +\t\tret = fn(&pm, table, data); +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } static int pmu_metrics_table__find_metric_pmu(const struct pmu_metrics_table *table, - const struct pmu_table_entry *pmu, - const char *metric, - pmu_metric_iter_fn fn, - void *data) +\t\t\t\t\t const struct pmu_table_entry *pmu, +\t\t\t\t\t const char *metric, +\t\t\t\t\t pmu_metric_iter_fn fn, +\t\t\t\t\t void *data) { - struct pmu_metric pm = { - .pmu = &big_c_string[pmu->pmu_name.offset], - }; - int low = 0, high = pmu->num_entries - 1; - - while (low <= high) { - int cmp, mid = (low + high) / 2; - - decompress_metric(pmu->entries[mid].offset, &pm); - - if (!pm.metric_name && !metric) - goto do_call; - - if (!pm.metric_name && metric) { - low = mid + 1; - continue; - } - if (pm.metric_name && !metric) { - high = mid - 1; - continue; - } - - cmp = strcmp(pm.metric_name, metric); - if (cmp < 0) { - low = mid + 1; - continue; - } - if (cmp > 0) { - high = mid - 1; - continue; - } +\tstruct pmu_metric pm = { +\t\t.pmu = &big_c_string[pmu->pmu_name.offset], +\t}; +\tint low = 0, high = pmu->num_entries - 1; + +\twhile (low <= high) { +\t\tint cmp, mid = (low + high) / 2; + +\t\tdecompress_metric(pmu->entries[mid].offset, &pm); + +\t\tif (!pm.metric_name && !metric) +\t\t\tgoto do_call; + +\t\tif (!pm.metric_name && metric) { +\t\t\tlow = mid + 1; +\t\t\tcontinue; +\t\t} +\t\tif (pm.metric_name && !metric) { +\t\t\thigh = mid - 1; +\t\t\tcontinue; +\t\t} + +\t\tcmp = strcmp(pm.metric_name, metric); +\t\tif (cmp < 0) { +\t\t\tlow = mid + 1; +\t\t\tcontinue; +\t\t} +\t\tif (cmp > 0) { +\t\t\thigh = mid - 1; +\t\t\tcontinue; +\t\t} do_call: - return fn ? fn(&pm, table, data) : 0; - } - return PMU_METRICS__NOT_FOUND; +\t\treturn fn ? fn(&pm, table, data) : 0; +\t} +\treturn PMU_METRICS__NOT_FOUND; } int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, - pmu_metric_iter_fn fn, - void *data) +\t\t\t\t pmu_metric_iter_fn fn, +\t\t\t\t void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - int ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i], - fn, data); - - if (ret) - return ret; - } - return 0; +\tif (!table) +\t\treturn 0; +\tfor (size_t i = 0; i < table->num_pmus; i++) { +\t\tint ret = pmu_metrics_table__for_each_metric_pmu(table, &table->pmus[i], fn, data); + +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } int pmu_metrics_table__find_metric(const struct pmu_metrics_table *table, - struct perf_pmu *pmu, - const char *metric, - pmu_metric_iter_fn fn, - void *data) +\t\t\t\t struct perf_pmu *pmu, +\t\t\t\t const char *metric, +\t\t\t\t pmu_metric_iter_fn fn, +\t\t\t\t void *data) { - if (!table) - return 0; - for (size_t i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - int ret; - - if (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) - continue; - - ret = pmu_metrics_table__find_metric_pmu(table, table_pmu, metric, fn, data); - if (ret != PMU_METRICS__NOT_FOUND) - return ret; - } - return PMU_METRICS__NOT_FOUND; +\tif (!table) +\t\treturn 0; +\tfor (size_t i = 0; i < table->num_pmus; i++) { +\t\tconst struct pmu_table_entry *table_pmu = &table->pmus[i]; +\t\tconst char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; +\t\tint ret; + +\t\tif (pmu && !perf_pmu__name_wildcard_match(pmu, pmu_name)) +\t\t\tcontinue; + +\t\tret = pmu_metrics_table__find_metric_pmu(table, table_pmu, metric, fn, data); +\t\tif (ret != PMU_METRICS__NOT_FOUND) +\t\t\treturn ret; +\t} +\treturn PMU_METRICS__NOT_FOUND; } static const struct pmu_events_map *map_for_cpu(struct perf_cpu cpu) { - static struct { - const struct pmu_events_map *map; - struct perf_cpu cpu; - } last_result; - static struct { - const struct pmu_events_map *map; - char *cpuid; - } last_map_search; - static bool has_last_result, has_last_map_search; - const struct pmu_events_map *map = NULL; - char *cpuid = NULL; - size_t i; - - if (has_last_result && last_result.cpu.cpu == cpu.cpu) - return last_result.map; - - cpuid = get_cpuid_allow_env_override(cpu); - - /* - * On some platforms which uses cpus map, cpuid can be NULL for - * PMUs other than CORE PMUs. - */ - if (!cpuid) - goto out_update_last_result; - - if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) { - map = last_map_search.map; - free(cpuid); - } else { - i = 0; - for (;;) { - map = &pmu_events_map[i++]; - - if (!map->arch) { - map = NULL; - break; - } - - if (!strcmp_cpuid_str(map->cpuid, cpuid)) - break; - } - free(last_map_search.cpuid); - last_map_search.cpuid = cpuid; - last_map_search.map = map; - has_last_map_search = true; - } +\tstatic struct { +\t\tconst struct pmu_events_map *map; +\t\tstruct perf_cpu cpu; +\t} last_result; +\tstatic struct { +\t\tconst struct pmu_events_map *map; +\t\tchar *cpuid; +\t} last_map_search; +\tstatic bool has_last_result, has_last_map_search; +\tconst struct pmu_events_map *map = NULL; +\tchar *cpuid = NULL; +\tsize_t i; + +\tif (has_last_result && last_result.cpu.cpu == cpu.cpu) +\t\treturn last_result.map; + +\tcpuid = get_cpuid_allow_env_override(cpu); + +\t/* +\t * On some platforms which uses cpus map, cpuid can be NULL for +\t * PMUs other than CORE PMUs. +\t */ +\tif (!cpuid) +\t\tgoto out_update_last_result; + +\tif (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) { +\t\tmap = last_map_search.map; +\t\tfree(cpuid); +\t} else { +\t\ti = 0; +\t\tfor (;;) { +\t\t\tmap = &pmu_events_map[i++]; + +\t\t\tif (!map->arch) { +\t\t\t\tmap = NULL; +\t\t\t\tbreak; +\t\t\t} + +\t\t\tif (!strcmp_cpuid_str(map->cpuid, cpuid)) +\t\t\t\tbreak; +\t\t} +\t\tfree(last_map_search.cpuid); +\t\tlast_map_search.cpuid = cpuid; +\t\tlast_map_search.map = map; +\t\thas_last_map_search = true; +\t} out_update_last_result: - last_result.cpu = cpu; - last_result.map = map; - has_last_result = true; - return map; +\tlast_result.cpu = cpu; +\tlast_result.map = map; +\thas_last_result = true; +\treturn map; } static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) { - struct perf_cpu cpu = {-1}; - - if (pmu) { - for (size_t i = 0; i < ARRAY_SIZE(pmu_events__common); i++) { - const char *pmu_name = &big_c_string[pmu_events__common[i].pmu_name.offset]; - - if (!strcmp(pmu_name, pmu->name)) { - const struct pmu_events_map *map = &pmu_events_map[0]; - - while (strcmp("common", map->arch)) - map++; - return map; - } - } - cpu = perf_cpu_map__min(pmu->cpus); - } - return map_for_cpu(cpu); +\tstruct perf_cpu cpu = { -1 }; + +\tif (pmu) { +\t\tfor (size_t i = 0; i < ARRAY_SIZE(pmu_events__common); i++) { +\t\t\tconst char *pmu_name = &big_c_string[pmu_events__common[i].pmu_name.offset]; + +\t\t\tif (!strcmp(pmu_name, pmu->name)) { +\t\t\t\tconst struct pmu_events_map *map = &pmu_events_map[0]; + +\t\t\t\twhile (strcmp("common", map->arch)) +\t\t\t\t\tmap++; +\t\t\t\treturn map; +\t\t\t} +\t\t} +\t\tcpu = perf_cpu_map__min(pmu->cpus); +\t} +\treturn map_for_cpu(cpu); } const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) { - const struct pmu_events_map *map = map_for_pmu(pmu); +\tconst struct pmu_events_map *map = map_for_pmu(pmu); - if (!map) - return NULL; +\tif (!map) +\t\treturn NULL; - if (!pmu) - return &map->event_table; +\tif (!pmu) +\t\treturn &map->event_table; - for (size_t i = 0; i < map->event_table.num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i]; - const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; +\tfor (size_t i = 0; i < map->event_table.num_pmus; i++) { +\t\tconst struct pmu_table_entry *table_pmu = &map->event_table.pmus[i]; +\t\tconst char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; - if (perf_pmu__name_wildcard_match(pmu, pmu_name)) - return &map->event_table; - } - return NULL; +\t\tif (perf_pmu__name_wildcard_match(pmu, pmu_name)) +\t\t\treturn &map->event_table; +\t} +\treturn NULL; } const struct pmu_events_table *perf_pmu__default_core_events_table(void) { - int i = 0; +\tint i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; +\tfor (;;) { +\t\tconst struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; +\t\tif (!map->arch) +\t\t\tbreak; - if (!strcmp(map->cpuid, "common")) - return &map->event_table; - } - return NULL; +\t\tif (!strcmp(map->cpuid, "common")) +\t\t\treturn &map->event_table; +\t} +\treturn NULL; } const struct pmu_metrics_table *pmu_metrics_table__find(void) { - struct perf_cpu cpu = {-1}; - const struct pmu_events_map *map = map_for_cpu(cpu); +\tstruct perf_cpu cpu = { -1 }; +\tconst struct pmu_events_map *map = map_for_cpu(cpu); - return map ? &map->metric_table : NULL; +\treturn map ? &map->metric_table : NULL; } const struct pmu_metrics_table *pmu_metrics_table__default(void) { - int i = 0; +\tint i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; +\tfor (;;) { +\t\tconst struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; +\t\tif (!map->arch) +\t\t\tbreak; - if (!strcmp(map->cpuid, "common")) - return &map->metric_table; - } - return NULL; +\t\tif (!strcmp(map->cpuid, "common")) +\t\t\treturn &map->metric_table; +\t} +\treturn NULL; } const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) - return &tables->event_table; - } - return NULL; +\tfor (const struct pmu_events_map *tables = &pmu_events_map[0]; +\t tables->arch; +\t tables++) { +\t\tif (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) +\t\t\treturn &tables->event_table; +\t} +\treturn NULL; } const struct pmu_metrics_table *find_core_metrics_table(const char *arch, const char *cpuid) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - if (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) - return &tables->metric_table; - } - return NULL; +\tfor (const struct pmu_events_map *tables = &pmu_events_map[0]; +\t tables->arch; +\t tables++) { +\t\tif (!strcmp(tables->arch, arch) && !strcmp_cpuid_str(tables->cpuid, cpuid)) +\t\t\treturn &tables->metric_table; +\t} +\treturn NULL; } int pmu_for_each_core_event(pmu_event_iter_fn fn, void *data) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - int ret = pmu_events_table__for_each_event(&tables->event_table, - /*pmu=*/ NULL, fn, data); - - if (ret) - return ret; - } - return 0; +\tfor (const struct pmu_events_map *tables = &pmu_events_map[0]; +\t tables->arch; +\t tables++) { +\t\tint ret = pmu_events_table__for_each_event(&tables->event_table, +\t\t\t\t\t\t\t /*pmu=*/NULL, fn, data); + +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } int pmu_for_each_core_metric(pmu_metric_iter_fn fn, void *data) { - for (const struct pmu_events_map *tables = &pmu_events_map[0]; - tables->arch; - tables++) { - int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); - - if (ret) - return ret; - } - return 0; +\tfor (const struct pmu_events_map *tables = &pmu_events_map[0]; +\t tables->arch; +\t tables++) { +\t\tint ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); + +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } const struct pmu_events_table *find_sys_events_table(const char *name) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - if (!strcmp(tables->name, name)) - return &tables->event_table; - } - return NULL; +\tfor (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; +\t tables->name; +\t tables++) { +\t\tif (!strcmp(tables->name, name)) +\t\t\treturn &tables->event_table; +\t} +\treturn NULL; } int pmu_for_each_sys_event(pmu_event_iter_fn fn, void *data) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - int ret = pmu_events_table__for_each_event(&tables->event_table, - /*pmu=*/ NULL, fn, data); - - if (ret) - return ret; - } - return 0; +\tfor (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; +\t tables->name; +\t tables++) { +\t\tint ret = pmu_events_table__for_each_event(&tables->event_table, +\t\t\t\t\t\t\t /*pmu=*/NULL, fn, data); + +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } int pmu_for_each_sys_metric(pmu_metric_iter_fn fn, void *data) { - for (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; - tables->name; - tables++) { - int ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); - - if (ret) - return ret; - } - return 0; +\tfor (const struct pmu_sys_events *tables = &pmu_sys_event_tables[0]; +\t tables->name; +\t tables++) { +\t\tint ret = pmu_metrics_table__for_each_metric(&tables->metric_table, fn, data); + +\t\tif (ret) +\t\t\treturn ret; +\t} +\treturn 0; } """) @@ -1362,22 +1412,21 @@ def print_metricgroups() -> None: const char *describe_metricgroup(const char *group) { - int low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1; - - while (low <= high) { - int mid = (low + high) / 2; - const char *mgroup = &big_c_string[metricgroups[mid][0]]; - int cmp = strcmp(mgroup, group); - - if (cmp == 0) { - return &big_c_string[metricgroups[mid][1]]; - } else if (cmp < 0) { - low = mid + 1; - } else { - high = mid - 1; - } - } - return NULL; +\tint low = 0, high = (int)ARRAY_SIZE(metricgroups) - 1; + +\twhile (low <= high) { +\t\tint mid = (low + high) / 2; +\t\tconst char *mgroup = &big_c_string[metricgroups[mid][0]]; +\t\tint cmp = strcmp(mgroup, group); + +\t\tif (cmp == 0) +\t\t\treturn &big_c_string[metricgroups[mid][1]]; +\t\telse if (cmp < 0) +\t\t\tlow = mid + 1; +\t\telse +\t\t\thigh = mid - 1; +\t} +\treturn NULL; } """) @@ -1422,11 +1471,12 @@ def ftw(path: str, parents: Sequence[str], ) ap.add_argument( 'output_file', type=argparse.FileType('w', encoding='utf-8'), nargs='?', default=sys.stdout) + ap.add_argument( + 'output_string_file', type=argparse.FileType('w', encoding='utf-8'), nargs='?', default=None) _args = ap.parse_args() - _args.output_file.write(f""" -/* SPDX-License-Identifier: GPL-2.0 */ -/* THIS FILE WAS AUTOGENERATED BY jevents.py arch={_args.arch} model={_args.model} ! */ + _args.output_file.write(f"""/* SPDX-License-Identifier: GPL-2.0 */ +/* THIS FILE WAS AUTOGENERATED BY `jevents.py arch={_args.arch} model={_args.model}` ! */ """) _args.output_file.write(""" #include @@ -1436,13 +1486,13 @@ def ftw(path: str, parents: Sequence[str], #include struct compact_pmu_event { - int offset; +\tint offset; }; struct pmu_table_entry { - const struct compact_pmu_event *entries; - uint32_t num_entries; - struct compact_pmu_event pmu_name; +\tconst struct compact_pmu_event *entries; +\tuint32_t num_entries; +\tstruct compact_pmu_event pmu_name; }; """) @@ -1463,10 +1513,21 @@ def ftw(path: str, parents: Sequence[str], ftw(arch_path, [], preprocess_one_file) _bcs.compute() - _args.output_file.write('static const char *const big_c_string =\n') - for s in _bcs.big_string: - _args.output_file.write(s) - _args.output_file.write(';\n\n') + _args.output_file.write('/* clang-format off */\n') + if not _args.output_string_file: + _args.output_file.write('static const char *const big_c_string =\n') + for s in _bcs.big_string: + _args.output_file.write(s) + _args.output_file.write(';\n\n') + else: + _args.output_string_file.write('/* SPDX-License-Identifier: GPL-2.0 */\n') + _args.output_string_file.write('/* Autogenerated by jevents.py */\n') + _args.output_string_file.write('extern const char big_c_string[];\n') + _args.output_string_file.write('const char big_c_string[] =\n') + for s in _bcs.big_string: + _args.output_string_file.write(s) + _args.output_string_file.write(';\n') + _args.output_file.write('extern const char big_c_string[];\n\n') for arch in archs: arch_path = f'{_args.starting_dir}/{arch}' ftw(arch_path, [], process_one_file) @@ -1475,7 +1536,12 @@ def ftw(path: str, parents: Sequence[str], print_mapping_table(archs) print_system_mapping_table() + _args.output_file.write('/* clang-format on */\n') + print_metric_table_functions() print_metricgroups() + _args.output_file.close() + if _args.output_string_file: + _args.output_string_file.close() if __name__ == '__main__': main() diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index 585454828c2f..a91ccb5977f0 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -25,7 +25,6 @@ def LoadEvents(directory: str) -> None: "cycles", "duration_time", "instructions", - "l2_itlb_misses", } for file in os.listdir(os.fsencode(directory)): filename = os.fsdecode(file) @@ -94,7 +93,7 @@ def CheckEveryEvent(*names: str) -> None: name = name[:name.find(':')] elif '/' in name: name = name[:name.find('/')] - if any([name.startswith(x) for x in ['amd', 'arm', 'cpu', 'msr', 'power']]): + if any([name.startswith(x) for x in ['amd', 'arm', 'cpu', 'msr', 'power', 'cha', 'uncore']]): continue if name not in all_events_all_models: raise Exception(f"Is {name} a named json event?") @@ -577,6 +576,11 @@ def source_count(event: Event) -> Function: return Function('source_count', event) +def aggr_nr(event: Event) -> Function: + # pylint: disable=invalid-name + return Function('aggr_nr', event) + + def has_event(event: Event) -> Function: # pylint: disable=redefined-builtin # pylint: disable=invalid-name @@ -763,7 +767,7 @@ def ParsePerfJson(orig: str) -> Expression: # Convert accidentally converted scientific notation constants back py = re.sub(r'([0-9]+)Event\(r"(e[0-9]*)"\)', r'\1\2', py) # Convert all the known keywords back from events to just the keyword - keywords = ['if', 'else', 'min', 'max', 'd_ratio', 'source_count', 'has_event', 'strcmp_cpuid_str'] + keywords = ['if', 'else', 'min', 'max', 'd_ratio', 'source_count', 'aggr_nr', 'has_event', 'strcmp_cpuid_str'] for kw in keywords: py = re.sub(rf'Event\(r"{kw}"\)', kw, py) try: diff --git a/tools/perf/pmu-events/pmu-events.h b/tools/perf/pmu-events/pmu-events.h index d3b24014c6ff..cb55c9fbca43 100644 --- a/tools/perf/pmu-events/pmu-events.h +++ b/tools/perf/pmu-events/pmu-events.h @@ -91,6 +91,9 @@ typedef int (*pmu_metric_iter_fn)(const struct pmu_metric *pm, const struct pmu_metrics_table *table, void *data); +typedef int (*pmu_metrics_table_iter_t)(const struct pmu_metrics_table *table, + void *data); + int pmu_events_table__for_each_event(const struct pmu_events_table *table, struct perf_pmu *pmu, pmu_event_iter_fn fn, @@ -112,6 +115,8 @@ size_t pmu_events_table__num_events(const struct pmu_events_table *table, int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, void *data); +const char *pmu_metrics_table__name(const struct pmu_metrics_table *table); +int pmu_metrics_table__iterate_tables(pmu_metrics_table_iter_t fn, void *data); /* * Search for a table and entry matching with pmu__name_wildcard_match or any * tables if pmu is NULL. Each matching metric has fn called on it. 0 implies to diff --git a/tools/perf/python/perf.pyi b/tools/perf/python/perf.pyi new file mode 100644 index 000000000000..f266c9baffd0 --- /dev/null +++ b/tools/perf/python/perf.pyi @@ -0,0 +1,672 @@ +"""Type stubs for the perf Python module.""" +from typing import Callable, Dict, List, Optional, Any, Iterator + +def config_get(name: str) -> Optional[str]: + """Get a configuration value from perf config. + + Args: + name: The configuration variable name (e.g., 'colors.top'). + + Returns: + The configuration value as a string, or None if not set. + """ + ... + +def metrics() -> List[Dict[str, str]]: + """Get a list of available metrics. + + Returns: + A list of dictionaries, each describing a metric. + """ + ... + +def syscall_name(id: int, *, elf_machine: Optional[int] = None) -> str: + """Convert a syscall number to its name. + + Args: + sc_id: The syscall number. + elf_machine: Optional ELF machine type. + + Returns: + The name of the syscall. + """ + ... + +def syscall_id(name: str, *, elf_machine: Optional[int] = None) -> int: + """Convert a syscall name to its number. + + Args: + name: The syscall name. + elf_machine: Optional ELF machine type. + + Returns: + The number of the syscall. + """ + ... + +def parse_events( + event_string: str, + cpus: Optional[cpu_map] = None, + threads: Optional['thread_map'] = None +) -> 'evlist': + """Parse an event string and return an evlist. + + Args: + event_string: The event string (e.g., 'cycles,instructions'). + cpus: Optional CPU map to bind events to. + threads: Optional thread map to bind events to. + + Returns: + An evlist containing the parsed events. + """ + ... + +def parse_metrics( + metrics_string: str, + pmu: Optional[str] = None, + cpus: Optional[cpu_map] = None, + threads: Optional['thread_map'] = None +) -> 'evlist': + """Parse a string of metrics or metric groups and return an evlist.""" + ... + +def tracepoint(sys: str, name: str) -> int: + """Returns the tracepoint ID for a given system and name.""" + ... + +def pmus() -> Iterator[Any]: + """Returns a sequence of pmus.""" + ... + +class data: + """Represents a perf data file.""" + def __init__(self, path: str = ..., fd: int = ...) -> None: ... + +class thread: + """Represents a thread in the system.""" + def comm(self) -> str: + """Get the command name of the thread.""" + ... + pid: int + tid: int + ppid: int + cpu: int + +class counts_values: + """Raw counter values.""" + id: int + val: int + ena: int + run: int + lost: int + values: List[int] + +class thread_map: + """Map of threads being monitored.""" + def __init__(self, pid: int = -1, tid: int = -1) -> None: + """Initialize a thread map. + + Args: + pid: Process ID to monitor (-1 for all). + tid: Thread ID to monitor (-1 for all). + """ + ... + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> int: ... + def __iter__(self) -> Iterator[int]: ... + +class evsel: + """Event selector, represents a single event being monitored.""" + def __init__( + self, + type: int = ..., + config: int = ..., + sample_freq: int = ..., + sample_period: int = ..., + sample_type: int = ..., + read_format: int = ..., + disabled: bool = ..., + inherit: bool = ..., + pinned: bool = ..., + exclusive: bool = ..., + exclude_user: bool = ..., + exclude_kernel: bool = ..., + exclude_hv: bool = ..., + exclude_idle: bool = ..., + mmap: bool = ..., + context_switch: bool = ..., + comm: bool = ..., + freq: bool = ..., + inherit_stat: bool = ..., + enable_on_exec: bool = ..., + task: bool = ..., + watermark: int = ..., + precise_ip: int = ..., + mmap_data: bool = ..., + sample_id_all: bool = ..., + wakeup_events: int = ..., + bp_type: int = ..., + bp_addr: int = ..., + bp_len: int = ..., + idx: int = ..., + ) -> None: ... + def __str__(self) -> str: + """Return string representation of the event.""" + ... + def open(self) -> None: + """Open the event selector file descriptor table.""" + ... + def read(self, cpu: int, thread: int) -> counts_values: + """Read counter values for a specific CPU and thread.""" + ... + ids: List[int] + def cpus(self) -> cpu_map: + """Get CPU map for this event.""" + ... + def threads(self) -> thread_map: + """Get thread map for this event.""" + ... + tracking: bool + config: int + read_format: int + sample_period: int + sample_type: int + size: int + type: int + wakeup_events: int + + +class _sample_members: + sample_pid: int + sample_tid: int + sample_time: int + sample_id: int + sample_stream_id: int + sample_period: int + sample_cpu: int + +class sample_event(_sample_members): + """Represents a sample event from perf.""" + evsel: evsel + sample_ip: int + sample_addr: int + sample_phys_addr: int + sample_weight: int + sample_data_src: int + sample_insn_count: int + sample_cyc_count: int + type: int + raw_buf: bytes + dso: str + dso_long_name: str + dso_bid: Optional[bytes] + map_start: int + map_end: int + map_pgoff: int + symbol: str + sym_start: int + sym_end: int + brstack: Optional['branch_stack'] + callchain: Optional['callchain'] + def srccode(self) -> str: ... + def insn(self) -> str: ... + def __getattr__(self, name: str) -> Any: ... + +class mmap_event(_sample_members): + """Represents a mmap event from perf.""" + type: int + misc: int + pid: int + tid: int + start: int + len: int + pgoff: int + filename: str + evsel: Optional['evsel'] + +class mmap2_event(_sample_members): + """Represents a mmap2 event from perf.""" + type: int + misc: int + pid: int + tid: int + start: int + len: int + pgoff: int + prot: int + flags: int + filename: str + maj: Optional[int] + min: Optional[int] + ino: Optional[int] + ino_generation: Optional[int] + build_id: Optional[bytes] + evsel: Optional['evsel'] + +class lost_event(_sample_members): + """Represents a lost events record.""" + type: int + id: int + lost: int + evsel: Optional['evsel'] + +class comm_event(_sample_members): + """Represents a COMM record.""" + type: int + pid: int + tid: int + comm: str + evsel: Optional['evsel'] + +class task_event(_sample_members): + """Represents an EXIT or FORK record.""" + type: int + pid: int + ppid: int + tid: int + ptid: int + time: int + evsel: Optional['evsel'] + +class throttle_event(_sample_members): + """Represents a THROTTLE or UNTHROTTLE record.""" + type: int + time: int + id: int + stream_id: int + evsel: Optional['evsel'] + +class read_event(_sample_members): + """Represents a READ record.""" + type: int + pid: int + tid: int + evsel: Optional['evsel'] + +class switch_event(_sample_members): + """Represents a SWITCH or SWITCH_CPU_WIDE record.""" + type: int + next_prev_pid: int + next_prev_tid: int + evsel: Optional['evsel'] + +class branch_entry: + """Represents a branch entry in the branch stack. + + Attributes: + from_ip: Source address of the branch (corresponds to 'from' keyword in C). + to_ip: Destination address of the branch. + mispred: True if the branch was mispredicted. + predicted: True if the branch was predicted. + in_tx: True if the branch was in a transaction. + abort: True if the branch was an abort. + cycles: Number of cycles since the last branch. + type: Type of branch. + """ + from_ip: int + to_ip: int + mispred: bool + predicted: bool + in_tx: bool + abort: bool + cycles: int + type: int + +class branch_stack: + """Sequence of branch entries in the branch stack.""" + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> branch_entry: ... + +class callchain_node: + """Represents a frame in the callchain.""" + ip: int + symbol: Optional[str] + dso: Optional[str] + +class callchain: + """Sequence of callchain frames.""" + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> callchain_node: ... + +class stat_event(_sample_members): + """Represents a stat event from perf.""" + type: int + id: int + cpu: int + thread: int + val: int + ena: int + run: int + evsel: Optional['evsel'] + +class stat_round_event(_sample_members): + """Represents a stat round event from perf.""" + type: int + time: int + stat_round_type: int + evsel: Optional['evsel'] + +class cpu_map: + """Map of CPUs being monitored.""" + def __init__(self, cpustr: Optional[str] = None) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> int: ... + def __iter__(self) -> Iterator[int]: ... + + +class evlist: + def __init__(self, cpus: cpu_map, threads: thread_map) -> None: ... + def open(self) -> None: + """Open the events in the list.""" + ... + def close(self) -> None: + """Close the events in the list.""" + ... + def mmap(self) -> None: + """Memory map the event buffers.""" + ... + def poll(self, timeout: int) -> int: + """Poll for events. + + Args: + timeout: Timeout in milliseconds. + + Returns: + Number of events ready. + """ + ... + def read_on_cpu(self, cpu: int) -> Optional[Any]: + """Read a sample event from a specific CPU. + + Args: + cpu: The CPU number. + + Returns: + A sample_event or other event type if available, or None. + """ + ... + def all_cpus(self) -> cpu_map: + """Get a cpu_map of all CPUs in the system.""" + ... + def metrics(self) -> List[str]: + """Get a list of metric names within the evlist.""" + ... + def compute_metric(self, metric: str, cpu: int, thread: int) -> float: + """Compute metric for given name, cpu and thread. + + Args: + metric: The metric name. + cpu: The CPU number. + thread: The thread ID. + + Returns: + The computed metric value. + """ + ... + def config(self) -> None: + """Configure the events in the list.""" + ... + def disable(self) -> None: + """Disable all events in the list.""" + ... + def enable(self) -> None: + """Enable all events in the list.""" + ... + def get_pollfd(self) -> List[int]: + """Get a list of file descriptors for polling.""" + ... + def add(self, evsel: evsel) -> int: + """Add an event to the list.""" + ... + def __iter__(self) -> Iterator[evsel]: + """Iterate over the events (evsel) in the list.""" + ... + + +class session: + def __init__( + self, + data: data, + sample: Optional[Callable[[sample_event], None]] = None, + stat: Optional[Callable[[Any, Optional[str]], None]] = None + ) -> None: + """Initialize a perf session. + + Args: + data: The perf data file to read. + sample: Callback for sample events. + stat: Callback for stat events. + """ + ... + def process_events(self) -> None: + """Process all events in the session.""" + ... + def find_thread(self, pid: int) -> thread: + """Returns the thread associated with a pid.""" + ... + +# Event Types +TYPE_HARDWARE: int +"""Hardware event.""" + +TYPE_SOFTWARE: int +"""Software event.""" + +TYPE_TRACEPOINT: int +"""Tracepoint event.""" + +TYPE_HW_CACHE: int +"""Hardware cache event.""" + +TYPE_RAW: int +"""Raw hardware event.""" + +TYPE_BREAKPOINT: int +"""Breakpoint event.""" + + +# Hardware Counters +COUNT_HW_CPU_CYCLES: int +"""Total cycles. Be wary of what happens during CPU frequency scaling.""" + +COUNT_HW_INSTRUCTIONS: int +"""Retired instructions. Be careful, these can be affected by various issues, +most notably hardware interrupt counts.""" + +COUNT_HW_CACHE_REFERENCES: int +"""Cache accesses. Usually this indicates Last Level Cache accesses but this +may vary depending on your CPU.""" + +COUNT_HW_CACHE_MISSES: int +"""Cache misses. Usually this indicates Last Level Cache misses.""" + +COUNT_HW_BRANCH_INSTRUCTIONS: int +"""Retired branch instructions.""" + +COUNT_HW_BRANCH_MISSES: int +"""Mispredicted branch instructions.""" + +COUNT_HW_BUS_CYCLES: int +"""Bus cycles, which can be different from total cycles.""" + +COUNT_HW_STALLED_CYCLES_FRONTEND: int +"""Stalled cycles during issue [This event is an alias of idle-cycles-frontend].""" + +COUNT_HW_STALLED_CYCLES_BACKEND: int +"""Stalled cycles during retirement [This event is an alias of idle-cycles-backend].""" + +COUNT_HW_REF_CPU_CYCLES: int +"""Total cycles; not affected by CPU frequency scaling.""" + + +# Cache Counters +COUNT_HW_CACHE_L1D: int +"""Level 1 data cache.""" + +COUNT_HW_CACHE_L1I: int +"""Level 1 instruction cache.""" + +COUNT_HW_CACHE_LL: int +"""Last Level Cache.""" + +COUNT_HW_CACHE_DTLB: int +"""Data TLB.""" + +COUNT_HW_CACHE_ITLB: int +"""Instruction TLB.""" + +COUNT_HW_CACHE_BPU: int +"""Branch Processing Unit.""" + +COUNT_HW_CACHE_OP_READ: int +"""Read accesses.""" + +COUNT_HW_CACHE_OP_WRITE: int +"""Write accesses.""" + +COUNT_HW_CACHE_OP_PREFETCH: int +"""Prefetch accesses.""" + +COUNT_HW_CACHE_RESULT_ACCESS: int +"""Accesses.""" + +COUNT_HW_CACHE_RESULT_MISS: int +"""Misses.""" + + +# Software Counters +COUNT_SW_CPU_CLOCK: int +"""CPU clock event.""" + +COUNT_SW_TASK_CLOCK: int +"""Task clock event.""" + +COUNT_SW_PAGE_FAULTS: int +"""Page faults.""" + +COUNT_SW_CONTEXT_SWITCHES: int +"""Context switches.""" + +COUNT_SW_CPU_MIGRATIONS: int +"""CPU migrations.""" + +COUNT_SW_PAGE_FAULTS_MIN: int +"""Minor page faults.""" + +COUNT_SW_PAGE_FAULTS_MAJ: int +"""Major page faults.""" + +COUNT_SW_ALIGNMENT_FAULTS: int +"""Alignment faults.""" + +COUNT_SW_EMULATION_FAULTS: int +"""Emulation faults.""" + +COUNT_SW_DUMMY: int +"""Dummy event.""" + + +# Sample Fields +SAMPLE_IP: int +"""Instruction pointer.""" + +SAMPLE_TID: int +"""Process and thread ID.""" + +SAMPLE_TIME: int +"""Timestamp.""" + +SAMPLE_ADDR: int +"""Sampled address.""" + +SAMPLE_READ: int +"""Read barcode.""" + +SAMPLE_CALLCHAIN: int +"""Call chain.""" + +SAMPLE_ID: int +"""Unique ID.""" + +SAMPLE_CPU: int +"""CPU number.""" + +SAMPLE_PERIOD: int +"""Sample period.""" + +SAMPLE_STREAM_ID: int +"""Stream ID.""" + +SAMPLE_RAW: int +"""Raw sample.""" + + +# Format Fields +FORMAT_TOTAL_TIME_ENABLED: int +"""Total time enabled.""" + +FORMAT_TOTAL_TIME_RUNNING: int +"""Total time running.""" + +FORMAT_ID: int +"""Event ID.""" + +FORMAT_GROUP: int +"""Event group.""" + + +# Record Types +RECORD_MMAP: int +"""MMAP record. Contains header, pid, tid, addr, len, pgoff, filename, and sample_id.""" + +RECORD_LOST: int +"""Lost events record. Contains header, id, lost count, and sample_id.""" + +RECORD_COMM: int +"""COMM record. Contains header, pid, tid, comm, and sample_id.""" + +RECORD_EXIT: int +"""EXIT record. Contains header, pid, ppid, tid, ptid, time, and sample_id.""" + +RECORD_THROTTLE: int +"""THROTTLE record. Contains header, time, id, stream_id, and sample_id.""" + +RECORD_UNTHROTTLE: int +"""UNTHROTTLE record. Contains header, time, id, stream_id, and sample_id.""" + +RECORD_FORK: int +"""FORK record. Contains header, pid, ppid, tid, ptid, time, and sample_id.""" + +RECORD_READ: int +"""READ record. Contains header, and read values.""" + +RECORD_SAMPLE: int +"""SAMPLE record. Contains header, and sample data requested by sample_type.""" + +RECORD_MMAP2: int +"""MMAP2 record. Contains header, pid, tid, addr, len, pgoff, maj, min, ino, +ino_generation, prot, flags, filename, and sample_id.""" + +RECORD_AUX: int +"""AUX record. Contains header, aux_offset, aux_size, flags, and sample_id.""" + +RECORD_ITRACE_START: int +"""ITRACE_START record. Contains header, pid, tid, and sample_id.""" + +RECORD_LOST_SAMPLES: int +"""LOST_SAMPLES record. Contains header, lost count, and sample_id.""" + +RECORD_SWITCH: int +"""SWITCH record. Contains header, and sample_id.""" + +RECORD_SWITCH_CPU_WIDE: int +"""SWITCH_CPU_WIDE record. Contains header, and sample_id.""" + +RECORD_STAT: int +"""STAT record.""" + +RECORD_STAT_ROUND: int +"""STAT_ROUND record.""" + +RECORD_MISC_SWITCH_OUT: int +"""MISC_SWITCH_OUT record.""" diff --git a/tools/perf/python/perf_live.py b/tools/perf/python/perf_live.py new file mode 100755 index 000000000000..616b5f657463 --- /dev/null +++ b/tools/perf/python/perf_live.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: GPL-2.0 +""" +Live event session helper using perf.evlist. + +This module provides a LiveSession class that allows running a callback +for each event collected live from the system, similar to perf.session +but without requiring a perf.data file. +""" + +import perf + + +class LiveSession: + """Represents a live event collection session.""" + + def __init__(self, event_string: str, sample_callback): + self.event_string = event_string + self.sample_callback = sample_callback + # Create a cpu map for all online CPUs + self.cpus = perf.cpu_map() + # Parse events and set maps + self.evlist = perf.parse_events(self.event_string, self.cpus) + self.evlist.config() + + def run(self): + """Run the live session.""" + try: + self.evlist.open() + self.evlist.mmap() + self.evlist.enable() + + while True: + # Poll for events with 100ms timeout + try: + self.evlist.poll(100) + except InterruptedError: + continue + for cpu in self.cpus: + for _ in range(1000): # Limit to 1000 events per CPU per poll to prevent starvation + try: + event = self.evlist.read_on_cpu(cpu) + except TypeError as e: + if "Unknown CPU" in str(e): + # CPU might be unmapped or offline, wait for mmap event + break + if "Unexpected header type" in str(e): + # Ignore valid but unsupported event types + continue + raise + + if event is None: + break + + if event.type == perf.RECORD_SAMPLE: + self.sample_callback(event) + except KeyboardInterrupt: + pass + finally: + self.evlist.close() diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py index ba208c90d631..42579f858684 100755 --- a/tools/perf/scripts/python/arm-cs-trace-disasm.py +++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py @@ -18,31 +18,32 @@ # Below are some example commands for using this script. # Note a --kcore recording is required for accurate decode -# due to the alternatives patching mechanism. However this -# script only supports reading vmlinux for disassembly dump, -# meaning that any patched instructions will appear -# as unpatched, but the instruction ranges themselves will -# be correct. In addition to this, source line info comes -# from Perf, and when using kcore there is no debug info. The -# following lists the supported features in each mode: +# due to the alternatives patching mechanism. In addition to this, +# source line info comes from Perf, and when using kcore there is +# no debug info. The following lists the supported features in each mode: # # +-----------+-----------------+------------------+------------------+ # | Recording | Accurate decode | Source line dump | Disassembly dump | # +-----------+-----------------+------------------+------------------+ # | --kcore | yes | no | yes | -# | normal | no | yes | yes | +# | normal | no | yes (inaccurate) | yes (inaccurate) | # +-----------+-----------------+------------------+------------------+ # # Output disassembly with objdump and auto detect vmlinux -# (when running on same machine.) -# perf script -s scripts/python/arm-cs-trace-disasm.py -d +# (when running on same machine.): +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ +# -- -d # # Output disassembly with llvm-objdump: -# perf script -s scripts/python/arm-cs-trace-disasm.py \ +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ # -- -d llvm-objdump-11 -k path/to/vmlinux # +# Output accurate disassembly by passing kcore to script: +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \ +# -- -d -k perf.data/kcore_dir/kcore +# # Output only source line and symbols: -# perf script -s scripts/python/arm-cs-trace-disasm.py +# perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py def default_objdump(): config = perf_config_get("annotate.objdump") @@ -57,7 +58,7 @@ def int_arg(v): args = argparse.ArgumentParser() args.add_argument("-k", "--vmlinux", - help="Set path to vmlinux file. Omit to autodetect if running on same machine") + help="Set path to vmlinux or kcore file. Omit to autodetect if running on same machine") args.add_argument("-d", "--objdump", nargs="?", const=default_objdump(), help="Show disassembly. Can also be used to change the objdump path"), args.add_argument("-v", "--verbose", action="store_true", help="Enable debugging log") diff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py index 8b78dc790adb..fedce1b68cad 100644 --- a/tools/perf/scripts/python/powerpc-hcalls.py +++ b/tools/perf/scripts/python/powerpc-hcalls.py @@ -25,6 +25,7 @@ # } ... # } output = {} +sort_key = 'count' # d_enter: { # cpu: { @@ -112,15 +113,17 @@ 420: 'H_CREATE_RPT', 424: 'H_REMOVE_RPT', 428: 'H_REGISTER_RPAGES', - 432: 'H_DISABLE_AND_GETC', + 432: 'H_DISABLE_AND_GET', 436: 'H_ERROR_DATA', 440: 'H_GET_HCA_INFO', 444: 'H_GET_PERF_COUNT', 448: 'H_MANAGE_TRACE', + 456: 'H_GET_CPU_CHARACTERISTICS', 468: 'H_FREE_LOGICAL_LAN_BUFFER', 472: 'H_POLL_PENDING', 484: 'H_QUERY_INT_STATE', 580: 'H_ILLAN_ATTRIBUTES', + 584: 'H_ADD_LOGICAL_LAN_BUFFERS', 592: 'H_MODIFY_HEA_QP', 596: 'H_QUERY_HEA_QP', 600: 'H_QUERY_HEA', @@ -135,11 +138,17 @@ 644: 'H_ADD_CONN', 648: 'H_DEL_CONN', 664: 'H_JOIN', + 672: 'H_VASI_SIGNAL', 676: 'H_VASI_STATE', + 680: 'H_VIOCTL', 688: 'H_ENABLE_CRQ', 696: 'H_GET_EM_PARMS', 720: 'H_SET_MPP', 724: 'H_GET_MPP', + 732: 'H_REG_SUB_CRQ', + 736: 'H_FREE_SUB_CRQ', + 740: 'H_SEND_SUB_CRQ', + 744: 'H_SEND_SUB_CRQ_INDIRECT', 748: 'H_HOME_NODE_ASSOCIATIVITY', 756: 'H_BEST_ENERGY', 764: 'H_XIRR_X', @@ -147,7 +156,88 @@ 772: 'H_COP', 788: 'H_GET_MPP_X', 796: 'H_SET_MODE', + 808: 'H_BLOCK_REMOVE', + 856: 'H_CLEAR_HPT', + 864: 'H_REQUEST_VMC', + 876: 'H_RESIZE_HPT_PREPARE', + 880: 'H_RESIZE_HPT_COMMIT', + 892: 'H_REGISTER_PROC_TBL', + 896: 'H_SIGNAL_SYS_RESET', + 904: 'H_ALLOCATE_VAS_WINDOW', + 908: 'H_MODIFY_VAS_WINDOW', + 912: 'H_DEALLOCATE_VAS_WINDOW', + 916: 'H_QUERY_VAS_WINDOW', + 920: 'H_QUERY_VAS_CAPABILITIES', + 924: 'H_QUERY_NX_CAPABILITIES', + 928: 'H_GET_NX_FAULT', + 936: 'H_INT_GET_SOURCE_INFO', + 940: 'H_INT_SET_SOURCE_CONFIG', + 944: 'H_INT_GET_SOURCE_CONFIG', + 948: 'H_INT_GET_QUEUE_INFO', + 952: 'H_INT_SET_QUEUE_CONFIG', + 956: 'H_INT_GET_QUEUE_CONFIG', + 960: 'H_INT_SET_OS_REPORTING_LINE', + 964: 'H_INT_GET_OS_REPORTING_LINE', + 968: 'H_INT_ESB', + 972: 'H_INT_SYNC', + 976: 'H_INT_RESET', + 996: 'H_SCM_READ_METADATA', + 1000: 'H_SCM_WRITE_METADATA', + 1004: 'H_SCM_BIND_MEM', + 1008: 'H_SCM_UNBIND_MEM', + 1012: 'H_SCM_QUERY_BLOCK_MEM_BINDING', + 1016: 'H_SCM_QUERY_LOGICAL_MEM_BINDING', + 1020: 'H_SCM_UNBIND_ALL', + 1024: 'H_SCM_HEALTH', + 1048: 'H_SCM_PERFORMANCE_STATS', + 1052: 'H_PKS_GET_CONFIG', + 1056: 'H_PKS_SET_PASSWORD', + 1060: 'H_PKS_GEN_PASSWORD', + 1068: 'H_PKS_WRITE_OBJECT', + 1072: 'H_PKS_GEN_KEY', + 1076: 'H_PKS_READ_OBJECT', + 1080: 'H_PKS_REMOVE_OBJECT', + 1084: 'H_PKS_CONFIRM_OBJECT_FLUSHED', + 1096: 'H_RPT_INVALIDATE', + 1100: 'H_SCM_FLUSH', + 1104: 'H_GET_ENERGY_SCALE_INFO', + 1108: 'H_PKS_SIGNED_UPDATE', + 1112: 'H_HTM', + 1116: 'H_WATCHDOG', + # Platform specific hcalls used by KVM on PowerVM + 1120: 'H_GUEST_GET_CAPABILITIES', + 1124: 'H_GUEST_SET_CAPABILITIES', + 1136: 'H_GUEST_CREATE', + 1140: 'H_GUEST_CREATE_VCPU', + 1144: 'H_GUEST_GET_STATE', + 1148: 'H_GUEST_SET_STATE', + 1152: 'H_GUEST_RUN_VCPU', + 1156: 'H_GUEST_COPY_MEMORY', + 1160: 'H_GUEST_DELETE', + # Key wrapping hcalls + 1168: 'H_PKS_WRAP_OBJECT', + 1172: 'H_PKS_UNWRAP_OBJECT', + # Platform-specific hcalls used by the Ultravisor + 61184: 'H_SVM_PAGE_IN', + 61188: 'H_SVM_PAGE_OUT', + 61192: 'H_SVM_INIT_START', + 61196: 'H_SVM_INIT_DONE', + 61204: 'H_SVM_INIT_ABORT', + # Platform specific hcalls used by KVM 61440: 'H_RTAS', + # Platform specific hcalls used by QEMU/SLOF + 61441: 'H_LOGICAL_MEMOP', + 61442: 'H_CAS', + 61443: 'H_UPDATE_DT', + # Platform specific hcalls provided by PHYP + 61560: 'H_GET_24X7_CATALOG_PAGE', + 61564: 'H_GET_24X7_DATA', + 61568: 'H_GET_PERF_COUNTER_INFO', + # Platform-specific hcalls used for nested HV KVM + 63488: 'H_SET_PARTITION_TABLE', + 63492: 'H_ENTER_NESTED', + 63496: 'H_TLB_INVALIDATE', + 63500: 'H_COPY_TOFROM_GUEST', } def hcall_table_lookup(opcode): @@ -158,10 +248,53 @@ def hcall_table_lookup(opcode): print_ptrn = '%-28s%10s%10s%10s%10s' +def sort_output(opcode): + stats = output[opcode] + + if sort_key == 'min': + return stats['min'] + if sort_key == 'max': + return stats['max'] + if sort_key == 'avg': + return stats['time'] // stats['cnt'] + + return stats['cnt'] + +def trace_begin(): + global sort_key + + valid_sort_keys = ['count', 'min', 'max', 'avg'] + + i = 1 + while i < len(sys.argv): + arg = sys.argv[i] + + if arg == '-s' or arg == '--sort': + if i + 1 >= len(sys.argv): + print("Error: -s/--sort requires a sort key argument") + sys.exit(1) + sort_key = sys.argv[i + 1] + i += 2 + continue + + if arg.startswith('--sort='): + sort_key = arg.split('=', 1)[1] + i += 1 + continue + + i += 1 + + if sort_key not in valid_sort_keys: + print(f"Error: Invalid sort key '{sort_key}'. Valid options are: {', '.join(valid_sort_keys)}") + sys.exit(1) + + print("SORT KEY =", sort_key) + def trace_end(): print(print_ptrn % ('hcall', 'count', 'min(ns)', 'max(ns)', 'avg(ns)')) print('-' * 68) - for opcode in output: + for opcode in sorted(output, key = sort_output, + reverse=True): h_name = hcall_table_lookup(opcode) time = output[opcode]['time'] cnt = output[opcode]['cnt'] diff --git a/tools/perf/tests/Build b/tools/perf/tests/Build index c2a67ce45941..66944a4f4968 100644 --- a/tools/perf/tests/Build +++ b/tools/perf/tests/Build @@ -3,6 +3,7 @@ perf-test-y += builtin-test.o perf-test-y += tests-scripts.o perf-test-y += parse-events.o +perf-test-y += uncore-event-sorting.o perf-test-y += dso-data.o perf-test-y += vmlinux-kallsyms.o perf-test-y += openat-syscall.o diff --git a/tools/perf/tests/backward-ring-buffer.c b/tools/perf/tests/backward-ring-buffer.c index c5e7999f2817..2735cc26d7ee 100644 --- a/tools/perf/tests/backward-ring-buffer.c +++ b/tools/perf/tests/backward-ring-buffer.c @@ -34,8 +34,8 @@ static int count_samples(struct evlist *evlist, int *sample_count, { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->overwrite_mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__overwrite_mmap(evlist)[i]; union perf_event *event; perf_mmap__read_init(&map->core); @@ -65,7 +65,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, int err; char sbuf[STRERR_BUFSIZE]; - err = evlist__mmap(evlist, mmap_pages); + err = evlist__do_mmap(evlist, mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); @@ -77,7 +77,7 @@ static int do_test(struct evlist *evlist, int mmap_pages, evlist__disable(evlist); err = count_samples(evlist, sample_count, comm_count); - evlist__munmap(evlist); + evlist__do_munmap(evlist); return err; } @@ -111,7 +111,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } parse_events_error__init(&parse_error); @@ -124,7 +124,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err) { pr_debug("Failed to parse tracepoint event, try use root\n"); ret = TEST_SKIP; - goto out_delete_evlist; + goto out_put_evlist; } evlist__config(evlist, &opts, NULL); @@ -133,19 +133,19 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } ret = TEST_FAIL; err = do_test(evlist, opts.mmap_pages, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; if ((sample_count != NR_ITERS) || (comm_count != NR_ITERS)) { pr_err("Unexpected counter: sample_count=%d, comm_count=%d\n", sample_count, comm_count); - goto out_delete_evlist; + goto out_put_evlist; } evlist__close(evlist); @@ -154,16 +154,16 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } err = do_test(evlist, 1, &sample_count, &comm_count); if (err != TEST_OK) - goto out_delete_evlist; + goto out_put_evlist; ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 06507066213b..7d18cb6021d3 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -10,34 +10,43 @@ #ifdef HAVE_BACKTRACE_SUPPORT #include #endif -#include -#include #include -#include #include -#include +#include + #include -#include +#include "util/term.h" +#include +#include +#include +#include +#include #include +#include +#include +#include +#include + +#include +#include +#include + #include "builtin.h" +#include "color.h" #include "config.h" +#include "debug.h" #include "hist.h" #include "intlist.h" -#include "tests.h" -#include "debug.h" -#include "color.h" -#include -#include #include "string2.h" #include "symbol.h" +#include "tests-scripts.h" +#include "tests.h" #include "util/rlimit.h" #include "util/strbuf.h" -#include -#include -#include -#include +#include "util/term.h" -#include "tests-scripts.h" +static const char *junit_filename; +static struct strbuf junit_xml_buf = STRBUF_INIT; /* * Command line option to not fork the test running in the same process and @@ -48,8 +57,11 @@ static bool dont_fork; static bool sequential; /* Number of times each test is run. */ static unsigned int runs_per_test = 1; +/* Number of lines to include in failure snippet. */ +static unsigned int failure_snippet_lines = 10; const char *dso_to_test; const char *test_objdump_path = "objdump"; +static const char *workload_control; /* * List of architecture specific tests. Not a weak symbol as the array length is @@ -71,6 +83,7 @@ static struct test_suite *generic_tests[] = { &suite__basic_mmap, &suite__mem, &suite__parse_events, + &suite__uncore_event_sorting, &suite__expr, &suite__PERF_RECORD, &suite__pmu, @@ -147,6 +160,7 @@ static struct test_suite *generic_tests[] = { static struct test_workload *workloads[] = { &workload__noploop, &workload__thloop, + &workload__named_threads, &workload__leafloop, &workload__sqrtloop, &workload__brstack, @@ -154,12 +168,21 @@ static struct test_workload *workloads[] = { &workload__landlock, &workload__traploop, &workload__inlineloop, + &workload__jitdump, + &workload__context_switch_loop, + &workload__deterministic, + &workload__callchain, #ifdef HAVE_RUST_SUPPORT &workload__code_with_type, #endif }; +struct workload_control { + int ctl_fd; + int ack_fd; +}; + #define workloads__for_each(workload) \ for (unsigned i = 0; i < ARRAY_SIZE(workloads) && ({ workload = workloads[i]; 1; }); i++) @@ -301,6 +324,11 @@ struct child_test { struct test_suite *test; int suite_num; int test_case_num; + struct strbuf err_output; + int result; + bool done; + struct timespec start_time; + struct timespec end_time; }; static jmp_buf run_test_jmp_buf; @@ -340,7 +368,7 @@ static int run_test_child(struct child_process *process) for (size_t i = 0; i < ARRAY_SIZE(signals); i++) signal(signals[i], child_test_sig_handler); - pr_debug("--- start ---\n"); + pr_debug("---- start ----\n"); pr_debug("test child forked, pid %d\n", getpid()); err = test_function(child->test, child->test_case_num)(child->test, child->test_case_num); pr_debug("---- end(%d) ----\n", err); @@ -355,42 +383,368 @@ static int run_test_child(struct child_process *process) #define TEST_RUNNING -3 +static struct pollfd *global_pfds; +static size_t *global_pfd_indices; +static unsigned int summary_tests_passed; +static unsigned int summary_subtests_passed; +static unsigned int summary_tests_skipped; +static unsigned int summary_tests_failed; +static struct strbuf summary_failed_tests_buf = STRBUF_INIT; + +static int strbuf_addstr_safe(struct strbuf *sb, const char *s); +static int __printf(2, 3) strbuf_addf_safe(struct strbuf *sb, const char *fmt, ...); + +static char *xml_escape(const char *str) +{ + struct strbuf buf = STRBUF_INIT; + const char *p; + char *res; + + if (!str) + return strdup(""); + + for (p = str; *p; p++) { + if (*p == '&') + strbuf_addstr(&buf, "&"); + else if (*p == '<') + strbuf_addstr(&buf, "<"); + else if (*p == '>') + strbuf_addstr(&buf, ">"); + else if (*p == '"') + strbuf_addstr(&buf, """); + else if ((unsigned char)*p >= 32 || *p == '\n' || *p == '\t') + strbuf_addch(&buf, *p); + } + res = strbuf_detach(&buf, NULL); + return res ? res : strdup(""); +} + +static int get_term_width(void) +{ + struct winsize ws; + int cols = 80; + int term_width; + + /* + * If output is redirected to a file or piped, we don't need to wrap + * or truncate at all. Use a massive virtually infinite terminal width + * so descriptions are printed in full. + */ + if (!isatty(fileno(debug_file()))) + return 10000; + + get_term_dimensions(&ws); + if (ws.ws_col > 0) + cols = ws.ws_col; + + /* + * Limit description width to fit on a single line. We subtract 35 + * columns of headroom to allocate space for: + * - The suite index prefix: e.g. " 10.100:" (8 characters) plus 1 space separator. + * - The trailing colon (1 character) and space before status (1 character). + * - The longest status results: e.g. "Skip (some metrics failed)" (26 characters) + * or "Running (XX active)" (20 characters). + * + * A minimum description width of 10 is enforced to ensure names are + * legible even on very narrow consoles. + */ + term_width = cols - 35; + if (term_width < 10) + term_width = 10; + + return term_width; +} + +static int get_max_desc_width(int width) +{ + int term_width = get_term_width(); + + return width > term_width ? term_width : width; +} + static int print_test_result(struct test_suite *t, int curr_suite, int curr_test_case, - int result, int width, int running) + int result, int width, int running, + const char *err_output, double elapsed) { + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + if (test_suite__num_test_cases(t) > 1) { - int subw = width > 2 ? width - 2 : width; + char prefix[32]; + int len = snprintf(prefix, sizeof(prefix), "%3d.%1d:", + curr_suite + 1, curr_test_case + 1); + int pad = len >= 4 ? pad_width + 4 - len : pad_width; + int trunc = len >= 4 ? term_width + 4 - len : term_width; - pr_info("%3d.%1d: %-*s:", curr_suite + 1, curr_test_case + 1, subw, + pr_info("%s %-*.*s:", prefix, pad, trunc, test_description(t, curr_test_case)); - } else - pr_info("%3d: %-*s:", curr_suite + 1, width, test_description(t, curr_test_case)); + } else { + pr_info("%3d: %-*.*s:", curr_suite + 1, pad_width, term_width, + test_description(t, curr_test_case)); + } switch (result) { case TEST_RUNNING: - color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Running (%d active)\n", running); break; case TEST_OK: + if (test_suite__num_test_cases(t) > 1) + summary_subtests_passed++; + else + summary_tests_passed++; pr_info(" Ok\n"); break; case TEST_SKIP: { const char *reason = skip_reason(t, curr_test_case); + summary_tests_skipped++; if (reason) - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (%s)\n", reason); else - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n"); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip\n"); } break; case TEST_FAIL: default: - color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n"); + summary_tests_failed++; + if (test_suite__num_test_cases(t) > 1) + strbuf_addf_safe(&summary_failed_tests_buf, " %3d.%1d: %s\n", + curr_suite + 1, curr_test_case + 1, + test_description(t, curr_test_case)); + else + strbuf_addf_safe(&summary_failed_tests_buf, " %3d: %s\n", + curr_suite + 1, + test_description(t, curr_test_case)); + color_fprintf(debug_file(), PERF_COLOR_RED, " FAILED!\n"); break; } + if (junit_filename && result != TEST_RUNNING) { + const char *classname = t->desc; + const char *testname = test_description(t, curr_test_case); + char *escaped_err = xml_escape(err_output); + char *escaped_class = xml_escape(classname); + char *escaped_test = xml_escape(testname); + + strbuf_addf(&junit_xml_buf, + " \n", + escaped_class, escaped_test, elapsed); + if (result != TEST_OK && result != TEST_SKIP) { + strbuf_addf(&junit_xml_buf, + " \n%s\n \n", + escaped_err); + } else if (result == TEST_SKIP) { + const char *reason = skip_reason(t, curr_test_case); + char *escaped_reason = xml_escape(reason ? reason : "Skip"); + + if (err_output && *err_output) { + strbuf_addf(&junit_xml_buf, + " \n%s\n \n", + escaped_reason, escaped_err); + } else { + strbuf_addf(&junit_xml_buf, " \n", + escaped_reason); + } + free(escaped_reason); + } + strbuf_addstr(&junit_xml_buf, " \n"); + free(escaped_err); + free(escaped_class); + free(escaped_test); + } + return 0; } +static const char * const fail_keywords[] = { + "error", "fail", "segv", "abort", + "signal", "fatal", "panic", "corrupt", NULL +}; + +static const char *find_next_keyword(const char *str, size_t max_len, size_t *kw_len) +{ + const char *best = NULL; + size_t best_len = 0; + int k; + + for (k = 0; fail_keywords[k]; k++) { + const char *s = str; + size_t len = strlen(fail_keywords[k]); + + while ((size_t)(s - str) + len <= max_len) { + size_t i; + + if (best && s >= best) + break; + + for (i = 0; i < len; i++) { + if (tolower(s[i]) != fail_keywords[k][i]) + break; + } + if (i == len) { + if (!best || s < best) { + best = s; + best_len = len; + } + break; + } + s++; + } + } + if (best) { + *kw_len = best_len; + return best; + } + return NULL; +} + +static void print_line_highlighted(FILE *fp, const char *line, size_t len) +{ + const char *s = line; + + while (len > 0) { + size_t kw_len = 0; + const char *match = find_next_keyword(s, len, &kw_len); + + if (!match) { + fwrite(s, 1, len, fp); + break; + } + if (match > s) + fwrite(s, 1, match - s, fp); + if (perf_use_color_default) + fprintf(fp, "%s", PERF_COLOR_RED); + fwrite(match, 1, kw_len, fp); + if (perf_use_color_default) + fprintf(fp, "%s", PERF_COLOR_RESET); + + len -= (match + kw_len) - s; + s = match + kw_len; + } +} + + +static void print_test_failure_snippet(FILE *fp, const char *buf) +{ + size_t num_lines = 0; + size_t max_lines = 128; + const char **lines = calloc(max_lines, sizeof(const char *)); + size_t *line_lens = calloc(max_lines, sizeof(size_t)); + const char *s = buf; + size_t i; + unsigned int picked_count = 0; + bool *pick; + int last_printed = -1; + + if (!lines || !line_lens) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + + while (*s) { + const char *eol = strchr(s, '\n'); + size_t len; + + if (eol) + len = eol - s + 1; + else + len = strlen(s); + + if (num_lines == max_lines) { + const char **new_lines; + size_t *new_lens; + + max_lines *= 2; + new_lines = realloc(lines, max_lines * sizeof(const char *)); + if (!new_lines) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + lines = new_lines; + + new_lens = realloc(line_lens, max_lines * sizeof(size_t)); + if (!new_lens) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + line_lens = new_lens; + } + lines[num_lines] = s; + line_lens[num_lines] = len; + num_lines++; + s += len; + } + + if (num_lines <= failure_snippet_lines) { + for (i = 0; i < num_lines; i++) + print_line_highlighted(fp, lines[i], line_lens[i]); + free(lines); free(line_lens); + return; + } + + pick = calloc(num_lines, sizeof(bool)); + if (!pick) { + for (i = 0; i < num_lines; i++) + print_line_highlighted(fp, lines[i], line_lens[i]); + free(lines); free(line_lens); + return; + } + + /* Pass 0: Always pick the very first line */ + if (num_lines > 0 && picked_count < failure_snippet_lines) { + pick[0] = true; + picked_count++; + } + + /* Pass 1: Pick lines with failure keywords from start (Highest Priority) */ + for (i = 0; i < num_lines && picked_count < failure_snippet_lines; i++) { + size_t dummy; + + if (find_next_keyword(lines[i], line_lens[i], &dummy)) { + if (!pick[i]) { + pick[i] = true; + picked_count++; + } + /* Prioritize getting the immediate next line for context */ + if (i + 1 < num_lines && !pick[i + 1] && + picked_count < failure_snippet_lines) { + pick[i + 1] = true; + picked_count++; + } + } + } + + /* Pass 2: Fill remaining quota from the end backwards */ + i = num_lines; + while (i > 0 && picked_count < failure_snippet_lines) { + i--; + if (!pick[i]) { + pick[i] = true; + picked_count++; + } + } + + for (i = 0; i < num_lines; i++) { + if (!pick[i]) + continue; + if (last_printed != -1 && (int)i > last_printed + 1) { + if (perf_use_color_default) + fprintf(fp, "%s...%s\n", PERF_COLOR_BLUE, PERF_COLOR_RESET); + else + fprintf(fp, "...\n"); + } + print_line_highlighted(fp, lines[i], line_lens[i]); + last_printed = i; + } + + free(pick); + free(lines); + free(line_lens); +} + static void finish_test(struct child_test **child_tests, int running_test, int child_test_num, int width) { @@ -401,6 +755,9 @@ static void finish_test(struct child_test **child_tests, int running_test, int c struct strbuf err_output = STRBUF_INIT; int last_running = -1; int ret; + struct timespec end_time; + double elapsed; + width = get_max_desc_width(width); if (child_test == NULL) { /* Test wasn't started. */ @@ -415,13 +772,14 @@ static void finish_test(struct child_test **child_tests, int running_test, int c * sub test names. */ if (test_suite__num_test_cases(t) > 1 && curr_test_case == 0) - pr_info("%3d: %-*s:\n", curr_suite + 1, width, test_description(t, -1)); + pr_info("%3d: %-*.*s:\n", curr_suite + 1, width, width, + test_description(t, -1)); /* * Busy loop reading from the child's stdout/stderr that are set to be * non-blocking until EOF. */ - if (err > 0) + if (err >= 0) fcntl(err, F_SETFL, O_NONBLOCK); if (verbose > 1) { if (test_suite__num_test_cases(t) > 1) @@ -454,7 +812,7 @@ static void finish_test(struct child_test **child_tests, int running_test, int c fprintf(debug_file(), PERF_COLOR_DELETE_LINE); } print_test_result(t, curr_suite, curr_test_case, TEST_RUNNING, - width, running); + width, running, NULL, 0.0); last_running = running; } } @@ -475,29 +833,338 @@ static void finish_test(struct child_test **child_tests, int running_test, int c if (len > 0) { err_done = false; buf[len] = '\0'; - strbuf_addstr(&err_output, buf); + strbuf_addstr_safe(&err_output, buf); } } } if (err_done) err_done = check_if_command_finished(&child_test->process); } + /* Drain any remaining data from the pipe. */ + if (err >= 0) { + char buf[512]; + ssize_t len; + + while ((len = read(err, buf, sizeof(buf) - 1)) > 0) { + buf[len] = '\0'; + strbuf_addstr_safe(&err_output, buf); + } + } if (perf_use_color_default && last_running != -1) { /* Erase "Running (.. active)" line printed before poll/sleep. */ fprintf(debug_file(), PERF_COLOR_DELETE_LINE); } /* Clean up child process. */ ret = finish_command(&child_test->process); - if (verbose > 1 || (verbose == 1 && ret == TEST_FAIL)) + child_test->process.pid = 0; + if (child_test->err_output.len > 0) { + struct strbuf merged = STRBUF_INIT; + + if (child_test->err_output.buf) + strbuf_addstr_safe(&merged, child_test->err_output.buf); + if (err_output.buf) + strbuf_addstr_safe(&merged, err_output.buf); + strbuf_release(&err_output); + err_output = merged; + } + if (verbose > 1) fprintf(stderr, "%s", err_output.buf); + else if (verbose == 1 && ret == TEST_FAIL) + print_test_failure_snippet(stderr, err_output.buf); + + clock_gettime(CLOCK_MONOTONIC, &end_time); + elapsed = (end_time.tv_sec - child_test->start_time.tv_sec) + + (end_time.tv_nsec - child_test->start_time.tv_nsec) / 1000000000.0; + print_test_result(t, curr_suite, curr_test_case, ret, width, /*running=*/0, + err_output.buf, elapsed); strbuf_release(&err_output); - print_test_result(t, curr_suite, curr_test_case, ret, width, /*running=*/0); + strbuf_release(&child_test->err_output); if (err > 0) close(err); zfree(&child_tests[running_test]); } +static int strbuf_addstr_safe(struct strbuf *sb, const char *s) +{ + sigset_t set, oldset; + int ret; + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + pthread_sigmask(SIG_BLOCK, &set, &oldset); + ret = strbuf_addstr(sb, s); + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return ret; +} + +static int __printf(2, 3) strbuf_addf_safe(struct strbuf *sb, const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + int len; + sigset_t set, oldset; + int ret; + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigprocmask(SIG_BLOCK, &set, &oldset); + + va_start(ap, fmt); + len = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + + if (len < 0) { + sigprocmask(SIG_SETMASK, &oldset, NULL); + return len; + } + if ((size_t)len >= sizeof(buf)) { + char *dynamic_buf = malloc(len + 1); + + if (!dynamic_buf) { + sigprocmask(SIG_SETMASK, &oldset, NULL); + return -ENOMEM; + } + va_start(ap, fmt); + vsnprintf(dynamic_buf, len + 1, fmt, ap); + va_end(ap); + ret = strbuf_addstr(sb, dynamic_buf); + free(dynamic_buf); + } else { + ret = strbuf_addstr(sb, buf); + } + + sigprocmask(SIG_SETMASK, &oldset, NULL); + return ret; +} + +static void drain_child_process_err(struct child_test *child) +{ + char buf[512]; + ssize_t len; + + while ((len = read(child->process.err, buf, sizeof(buf) - 1)) > 0) { + buf[len] = '\0'; + strbuf_addstr_safe(&child->err_output, buf); + } +} + +static void handle_child_pipe_activity(struct child_test *child, short revents) +{ + if (!revents) + return; + + drain_child_process_err(child); + /* + * If the child closed its end of the pipe (EOF) or encountered + * an error, close the file descriptor immediately and set it + * to -1. This removes it from the pfds array for subsequent + * iterations, preventing a tight CPU busy-loop while waiting + * for the process itself to exit. + */ + if (revents & (POLLHUP | POLLERR | POLLNVAL)) { + close(child->process.err); + child->process.err = -1; + } +} + +static int finish_tests_parallel(struct child_test **child_tests, size_t num_tests, int width) +{ + size_t next_to_print = 0; + struct pollfd *pfds; + size_t *pfd_indices; + size_t num_pfds = 0; + int last_running = -1; + size_t i; + int last_suite_printed = -1; + sigset_t set, oldset; + + width = get_max_desc_width(width); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + + pthread_sigmask(SIG_BLOCK, &set, &oldset); + global_pfds = calloc(num_tests, sizeof(*pfds)); + global_pfd_indices = calloc(num_tests, sizeof(*pfd_indices)); + pfds = global_pfds; + pfd_indices = global_pfd_indices; + if (!pfds || !pfd_indices) { + free(pfds); + free(pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return -ENOMEM; + } + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + + for (i = 0; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child) + continue; + strbuf_init(&child->err_output, 0); + if (child->process.err >= 0) + fcntl(child->process.err, F_SETFL, O_NONBLOCK); + } + + while (next_to_print < num_tests) { + size_t running_count = 0; + size_t p; + + while (next_to_print < num_tests && + (!child_tests[next_to_print] || child_tests[next_to_print]->done)) + next_to_print++; + + if (next_to_print >= num_tests) + break; + + num_pfds = 0; + + for (i = next_to_print; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child || child->done) + continue; + + if (!check_if_command_finished(&child->process)) + running_count++; + + if (child->process.err >= 0) { + pfds[num_pfds].fd = child->process.err; + pfds[num_pfds].events = POLLIN | POLLERR | POLLHUP | POLLNVAL; + pfd_indices[num_pfds] = i; + num_pfds++; + } + } + + if (perf_use_color_default && running_count != (size_t)last_running) { + struct child_test *next_child = child_tests[next_to_print]; + + if (last_running != -1) + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); + + if (next_child) { + if (test_suite__num_test_cases(next_child->test) > 1 && + last_suite_printed != next_child->suite_num) { + pr_info("%3d: %-*.*s:\n", + next_child->suite_num + 1, + width, width, + test_description( + next_child->test, -1)); + last_suite_printed = next_child->suite_num; + } + print_test_result(next_child->test, next_child->suite_num, + next_child->test_case_num, TEST_RUNNING, width, + running_count, NULL, 0.0); + } + last_running = running_count; + } + + if (num_pfds == 0) { + if (running_count > 0) + usleep(10 * 1000); + } else { + int pret = poll(pfds, num_pfds, 100); + + if (pret > 0) { + for (p = 0; p < num_pfds; p++) { + size_t idx = pfd_indices[p]; + + handle_child_pipe_activity(child_tests[idx], + pfds[p].revents); + } + } + } + + for (i = next_to_print; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child || child->done) + continue; + + if (check_if_command_finished(&child->process)) { + if (child->process.err >= 0) { + drain_child_process_err(child); + close(child->process.err); + child->process.err = -1; + } + child->result = finish_command(&child->process); + child->process.pid = 0; + clock_gettime(CLOCK_MONOTONIC, &child->end_time); + child->done = true; + } + } + + while (next_to_print < num_tests) { + struct child_test *child = child_tests[next_to_print]; + double elapsed; + + if (!child) { + next_to_print++; + continue; + } + if (!child->done) + break; + + if (perf_use_color_default && last_running != -1) { + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); + last_running = -1; + } + + if (test_suite__num_test_cases(child->test) > 1 && + last_suite_printed != child->suite_num) { + pr_info("%3d: %-*.*s:\n", child->suite_num + 1, + width, width, + test_description(child->test, -1)); + last_suite_printed = child->suite_num; + } + + if (verbose > 1) { + if (test_suite__num_test_cases(child->test) > 1) { + pr_info("%3d.%1d: %s:\n", child->suite_num + 1, + child->test_case_num + 1, + test_description(child->test, + child->test_case_num)); + } else { + pr_info("%3d: %s:\n", child->suite_num + 1, + test_description(child->test, -1)); + } + } + + if (verbose > 1) + fprintf(stderr, "%s", child->err_output.buf); + else if (verbose == 1 && child->result == TEST_FAIL) + print_test_failure_snippet(stderr, child->err_output.buf); + + elapsed = (child->end_time.tv_sec - child->start_time.tv_sec) + + (child->end_time.tv_nsec - + child->start_time.tv_nsec) / 1000000000.0; + + print_test_result(child->test, child->suite_num, child->test_case_num, + child->result, width, 0, child->err_output.buf, elapsed); + pthread_sigmask(SIG_BLOCK, &set, &oldset); + strbuf_release(&child->err_output); + child_tests[next_to_print] = NULL; + zfree(&child); + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + next_to_print++; + } + } + + pthread_sigmask(SIG_BLOCK, &set, &oldset); + free(global_pfds); + free(global_pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return 0; +} + static int start_test(struct test_suite *test, int curr_suite, int curr_test_case, struct child_test **child, int width, int pass) { @@ -506,11 +1173,18 @@ static int start_test(struct test_suite *test, int curr_suite, int curr_test_cas *child = NULL; if (dont_fork) { if (pass == 1) { + struct timespec start_time, end_time; + double elapsed; + + clock_gettime(CLOCK_MONOTONIC, &start_time); pr_debug("--- start ---\n"); err = test_function(test, curr_test_case)(test, curr_test_case); pr_debug("---- end ----\n"); + clock_gettime(CLOCK_MONOTONIC, &end_time); + elapsed = (end_time.tv_sec - start_time.tv_sec) + + (end_time.tv_nsec - start_time.tv_nsec) / 1000000000.0; print_test_result(test, curr_suite, curr_test_case, err, width, - /*running=*/0); + /*running=*/0, NULL, elapsed); } return 0; } @@ -531,13 +1205,14 @@ static int start_test(struct test_suite *test, int curr_suite, int curr_test_cas (*child)->test_case_num = curr_test_case; (*child)->process.pid = -1; (*child)->process.no_stdin = 1; + (*child)->process.in = -1; + (*child)->process.out = -1; + (*child)->process.err = -1; if (verbose <= 0) { (*child)->process.no_stdout = 1; (*child)->process.no_stderr = 1; } else { (*child)->process.stdout_to_stderr = 1; - (*child)->process.out = -1; - (*child)->process.err = -1; } (*child)->process.no_exec_cmd = run_test_child; if (sequential || pass == 2) { @@ -561,6 +1236,58 @@ static void cmd_test_sig_handler(int sig) siglongjmp(cmd_test_jmp_buf, sig); } +static void print_tests_summary(void) +{ + pr_info("\n=== Test Summary ===\n"); + pr_info("Passed main tests : %u\n", summary_tests_passed); + pr_info("Passed subtests : %u\n", summary_subtests_passed); + pr_info("Skipped tests : %u\n", summary_tests_skipped); + if (summary_tests_failed > 0) { + color_fprintf(debug_file(), PERF_COLOR_RED, "Failed tests : %u\n", + summary_tests_failed); + pr_info("List of failed tests:\n"); + pr_info("%s", summary_failed_tests_buf.buf); + } else { + color_fprintf(debug_file(), PERF_COLOR_GREEN, "Failed tests : 0\n"); + } + + if (junit_filename) { + int fd; + FILE *fp; + + fd = open(junit_filename, O_CREAT | O_TRUNC | O_WRONLY | O_NOFOLLOW, 0644); + if (fd >= 0) { + fp = fdopen(fd, "w"); + if (fp) { + unsigned int total = summary_tests_passed + + summary_subtests_passed + + summary_tests_skipped + + summary_tests_failed; + fprintf(fp, "\n"); + fprintf(fp, "\n"); + fprintf(fp, + " \n", + total, summary_tests_failed, + summary_tests_skipped); + fprintf(fp, "%s", junit_xml_buf.buf); + fprintf(fp, " \n"); + fprintf(fp, "\n"); + fclose(fp); + pr_info("Wrote junit XML output to %s\n", junit_filename); + } else { + close(fd); + pr_err("Failed to associate stream with fd for %s: %s\n", + junit_filename, strerror(errno)); + } + } else { + pr_err("Failed to open %s for writing junit XML output: %s\n", + junit_filename, strerror(errno)); + } + } + strbuf_release(&junit_xml_buf); + strbuf_release(&summary_failed_tests_buf); +} + static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], struct intlist *skiplist) { @@ -638,9 +1365,36 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], } if (intlist__find(skiplist, curr_suite + 1)) { - pr_info("%3d: %-*s:", curr_suite + 1, width, - test_description(*t, -1)); - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n"); + if (pass == 1) { + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + + pr_info("%3d: %-*.*s:", curr_suite + 1, + pad_width, term_width, + test_description(*t, -1)); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, + " Skip (user override)\n"); + summary_tests_skipped++; + if (junit_filename) { + char *escaped_class = + xml_escape((const char *) + test_description(*t, -1)); + char *escaped_test = xml_escape("override"); + char *escaped_reason = + xml_escape("user override"); + + strbuf_addf(&junit_xml_buf, + " \n", + escaped_class, escaped_test); + strbuf_addf(&junit_xml_buf, + " \n", + escaped_reason); + strbuf_addstr(&junit_xml_buf, " \n"); + free(escaped_reason); + free(escaped_test); + free(escaped_class); + } + } continue; } @@ -660,8 +1414,9 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], } if (!sequential) { /* Parallel mode starts tests but doesn't finish them. Do that now. */ - for (size_t x = 0; x < num_tests; x++) - finish_test(child_tests, x, num_tests, width); + err = finish_tests_parallel(child_tests, num_tests, width); + if (err) + goto err_out; } } err_out: @@ -672,6 +1427,11 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], for (size_t x = 0; x < num_tests; x++) finish_test(child_tests, x, num_tests, width); } + print_tests_summary(); + free(global_pfds); + free(global_pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; free(child_tests); return err; } @@ -710,13 +1470,185 @@ static int workloads__fprintf_list(FILE *fp) return printed; } +static int perf_control_open_fifo(struct workload_control *ctl, const char *str) +{ + char *s, *p; + int ret; + + if (strncmp(str, "fifo:", 5)) + return -EINVAL; + + str += 5; + if (!*str || *str == ',') + return -EINVAL; + + s = strdup(str); + if (!s) + return -ENOMEM; + + p = strchr(s, ','); + if (p) + *p = '\0'; + + ctl->ctl_fd = open(s, O_WRONLY | O_CLOEXEC); + if (ctl->ctl_fd < 0) { + ret = -errno; + pr_err("Failed to open workload control FIFO '%s': %m\n", s); + free(s); + return ret; + } + + if (p && *++p) { + ctl->ack_fd = open(p, O_RDONLY | O_CLOEXEC); + if (ctl->ack_fd < 0) { + ret = -errno; + pr_err("Failed to open workload control ack FIFO '%s': %m\n", p); + close(ctl->ctl_fd); + ctl->ctl_fd = -1; + free(s); + return ret; + } + } + + free(s); + return 0; +} + +static int perf_control_open(struct workload_control *ctl) +{ + int ret; + + if (!workload_control) + return 0; + + ret = perf_control_open_fifo(ctl, workload_control); + + if (ret == -EINVAL) { + pr_err("Unsupported workload control spec '%s', expected fifo:ctl-fifo[,ack-fifo]\n", + workload_control); + } + + return ret; +} + +static void perf_control_close(struct workload_control *ctl) +{ + if (ctl->ctl_fd >= 0) { + close(ctl->ctl_fd); + ctl->ctl_fd = -1; + } + if (ctl->ack_fd >= 0) { + close(ctl->ack_fd); + ctl->ack_fd = -1; + } +} + +static int perf_control_write_cmd(int fd, const char *cmd) +{ + size_t len = strlen(cmd); + ssize_t ret; + + while (len) { + ret = write(fd, cmd, len); + if (ret < 0) { + if (errno == EINTR) + continue; + pr_err("Failed to write perf control command: %m\n"); + return -1; + } + + if (!ret) { + pr_err("Failed to write perf control command: short write\n"); + return -1; + } + + cmd += ret; + len -= ret; + } + + return 0; +} + +static int perf_control_read_ack(int fd) +{ + char buf[16]; + ssize_t ret; + + do { + ret = read(fd, buf, sizeof(buf) - 1); + } while (ret < 0 && errno == EINTR); + + if (ret < 0) { + pr_err("Failed to read perf control ack: %m\n"); + return -1; + } + + if (!ret) { + pr_err("Unexpected EOF while reading perf control ack\n"); + return -1; + } + + buf[ret] = '\0'; + for (ssize_t i = 0; i < ret; i++) { + if (buf[i] == '\n' || buf[i] == '\0') { + buf[i] = '\0'; + break; + } + } + + if (strcmp(buf, "ack")) { + pr_err("Unexpected perf control ack: %s\n", buf); + return -1; + } + + return 0; +} + +static int perf_control_send(struct workload_control *ctl, const char *cmd) +{ + if (ctl->ctl_fd < 0) + return 0; + + if (perf_control_write_cmd(ctl->ctl_fd, cmd)) + return -1; + + if (ctl->ack_fd >= 0 && perf_control_read_ack(ctl->ack_fd)) + return -1; + + return 0; +} + static int run_workload(const char *work, int argc, const char **argv) { struct test_workload *twl; workloads__for_each(twl) { - if (!strcmp(twl->name, work)) - return twl->func(argc, argv); + struct workload_control ctl = { + .ctl_fd = -1, + .ack_fd = -1, + }; + int control_ret, ret; + + if (strcmp(twl->name, work)) + continue; + + ret = perf_control_open(&ctl); + if (ret) + return ret; + + if (perf_control_send(&ctl, "enable\n")) { + perf_control_close(&ctl); + return -1; + } + + ret = twl->func(argc, argv); + + control_ret = perf_control_send(&ctl, "disable\n"); + perf_control_close(&ctl); + if (control_ret) + return -1; + + return ret; } pr_info("No workload found: %s\n", work); @@ -754,10 +1686,21 @@ static struct test_suite **build_suites(void) for (size_t i = 0, j = 0; i < ARRAY_SIZE(suites); i++, j = 0) \ while ((suite = suites[i][j++]) != NULL) - for_each_suite(t) + for_each_suite(t) { + if (t->setup) { + int ret = t->setup(t); + + if (ret < 0) { + errno = -ret; + return NULL; + } + } num_suites++; + } result = calloc(num_suites + 1, sizeof(struct test_suite *)); + if (!result) + return NULL; for (int pass = 1; pass <= 2; pass++) { for_each_suite(t) { @@ -798,10 +1741,16 @@ int cmd_test(int argc, const char **argv) OPT_UINTEGER('r', "runs-per-test", &runs_per_test, "Run each test the given number of times, default 1"), OPT_STRING('w', "workload", &workload, "work", "workload to run for testing, use '--list-workloads' to list the available ones."), + OPT_STRING(0, "record-ctl", &workload_control, "fifo:ctl-fifo[,ack-fifo]", + "Write enable to the fifo just before running the workload and disable after, with optional ack from ack-fifo"), OPT_BOOLEAN(0, "list-workloads", &list_workloads, "List the available builtin workloads to use with -w/--workload"), OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"), OPT_STRING(0, "objdump", &test_objdump_path, "path", "objdump binary to use for disassembly and annotations"), + OPT_UINTEGER(0, "failure-snippet-lines", &failure_snippet_lines, + "Number of lines to include in failure snippet, default 10"), + OPT_STRING_OPTARG('j', "junit", &junit_filename, "file", + "Generate junit XML output, default test.xml", "test.xml"), OPT_END() }; const char * const test_subcommands[] = { "list", NULL }; @@ -820,6 +1769,8 @@ int cmd_test(int argc, const char **argv) argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0); if (argc >= 1 && !strcmp(argv[0], "list")) { suites = build_suites(); + if (!suites) + return errno ? -errno : -ENOMEM; ret = perf_test__list(stdout, suites, argc - 1, argv + 1); free(suites); return ret; @@ -852,6 +1803,8 @@ int cmd_test(int argc, const char **argv) rlimit__bump_memlock(); suites = build_suites(); + if (!suites) + return errno ? -errno : -ENOMEM; ret = __cmd_test(suites, argc, argv, skiplist); free(suites); return ret; diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c index 47043a3a2fb4..f0e8ea8754ef 100644 --- a/tools/perf/tests/code-reading.c +++ b/tools/perf/tests/code-reading.c @@ -471,8 +471,11 @@ static int read_object_code(u64 addr, size_t len, u8 cpumode, goto out; } - decomp = true; - objdump_name = decomp_name; + /* empty pathname means file wasn't actually compressed */ + if (decomp_name[0] != '\0') { + decomp = true; + objdump_name = decomp_name; + } } /* Read the object code using objdump */ @@ -589,8 +592,8 @@ static int process_events(struct machine *machine, struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -778,7 +781,7 @@ static int do_test_code_reading(bool try_kcore) goto out_put; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); str = events[evidx]; pr_debug("Parsing event '%s'\n", str); @@ -806,8 +809,8 @@ static int do_test_code_reading(bool try_kcore) pr_debug("perf_evlist__open() failed!\n%s\n", errbuf); } - perf_evlist__set_maps(&evlist->core, NULL, NULL); - evlist__delete(evlist); + perf_evlist__set_maps(evlist__core(evlist), NULL, NULL); + evlist__put(evlist); evlist = NULL; continue; } @@ -817,7 +820,7 @@ static int do_test_code_reading(bool try_kcore) if (events[evidx] == NULL) goto out_put; - ret = evlist__mmap(evlist, UINT_MAX); + ret = evlist__do_mmap(evlist, UINT_MAX); if (ret < 0) { pr_debug("evlist__mmap failed\n"); goto out_put; @@ -844,7 +847,7 @@ static int do_test_code_reading(bool try_kcore) out_put: thread__put(thread); out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); machine__delete(machine); diff --git a/tools/perf/tests/dlfilter-test.c b/tools/perf/tests/dlfilter-test.c index e63790c61d53..204663571943 100644 --- a/tools/perf/tests/dlfilter-test.c +++ b/tools/perf/tests/dlfilter-test.c @@ -188,8 +188,12 @@ static int write_sample(struct test_data *td, u64 sample_type, u64 id, pid_t pid event->header.type = PERF_RECORD_SAMPLE; event->header.misc = PERF_RECORD_MISC_USER; - event->header.size = perf_event__sample_event_size(&sample, sample_type, 0); - err = perf_event__synthesize_sample(event, sample_type, 0, &sample); + event->header.size = perf_event__sample_event_size(&sample, sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0); + err = perf_event__synthesize_sample(event, sample_type, + /*read_format=*/0, + /*branch_sample_type=*/0, &sample); if (err) return test_result("perf_event__synthesize_sample() failed", TEST_FAIL); diff --git a/tools/perf/tests/event-times.c b/tools/perf/tests/event-times.c index ae3b98bb42cf..56dd37ca760e 100644 --- a/tools/perf/tests/event-times.c +++ b/tools/perf/tests/event-times.c @@ -50,7 +50,7 @@ static int attach__enable_on_exec(struct evlist *evlist) static int detach__enable_on_exec(struct evlist *evlist) { - waitpid(evlist->workload.pid, NULL, 0); + waitpid(evlist__workload_pid(evlist), NULL, 0); return 0; } @@ -186,7 +186,7 @@ static int test_times(int (attach)(struct evlist *), err = attach(evlist); if (err == TEST_SKIP) { pr_debug(" SKIP : not enough rights\n"); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -205,7 +205,7 @@ static int test_times(int (attach)(struct evlist *), count.ena, count.run); out_err: - evlist__delete(evlist); + evlist__put(evlist); return !err ? TEST_OK : TEST_FAIL; } diff --git a/tools/perf/tests/event_update.c b/tools/perf/tests/event_update.c index facc65e29f20..220cc0347747 100644 --- a/tools/perf/tests/event_update.c +++ b/tools/perf/tests/event_update.c @@ -92,7 +92,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to allocate ids", !perf_evsel__alloc_id(&evsel->core, 1, 1)); - perf_evlist__id_add(&evlist->core, &evsel->core, 0, 0, 123); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, 0, 123); free((char *)evsel->unit); evsel->unit = strdup("KRAVA"); @@ -117,7 +117,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes TEST_ASSERT_VAL("failed to synthesize attr update cpus", !perf_event__synthesize_event_update_cpus(&tmp.tool, evsel, process_event_cpus)); - evlist__delete(evlist); + evlist__put(evlist); return 0; } diff --git a/tools/perf/tests/evsel-roundtrip-name.c b/tools/perf/tests/evsel-roundtrip-name.c index 1922cac13a24..6a220634c52f 100644 --- a/tools/perf/tests/evsel-roundtrip-name.c +++ b/tools/perf/tests/evsel-roundtrip-name.c @@ -33,7 +33,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) if (err) { pr_debug("Failure to parse cache event '%s' possibly as PMUs don't support it", name); - evlist__delete(evlist); + evlist__put(evlist); continue; } evlist__for_each_entry(evlist, evsel) { @@ -42,7 +42,7 @@ static int perf_evsel__roundtrip_cache_name_test(void) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } } } @@ -66,7 +66,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) if (err) { pr_debug("failed to parse event '%s', err %d\n", names[i], err); - evlist__delete(evlist); + evlist__put(evlist); ret = TEST_FAIL; continue; } @@ -76,7 +76,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names) ret = TEST_FAIL; } } - evlist__delete(evlist); + evlist__put(evlist); } return ret; } diff --git a/tools/perf/tests/evsel-tp-sched.c b/tools/perf/tests/evsel-tp-sched.c index 226196fb9677..9e456f88a13a 100644 --- a/tools/perf/tests/evsel-tp-sched.c +++ b/tools/perf/tests/evsel-tp-sched.c @@ -64,7 +64,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "next_prio", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); evsel = evsel__newtp("sched", "sched_wakeup"); @@ -85,7 +85,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse if (evsel__test_field(evsel, "target_cpu", 4, true)) ret = TEST_FAIL; - evsel__delete(evsel); + evsel__put(evsel); return ret; } diff --git a/tools/perf/tests/expand-cgroup.c b/tools/perf/tests/expand-cgroup.c index dd547f2f77cc..04d62611766a 100644 --- a/tools/perf/tests/expand-cgroup.c +++ b/tools/perf/tests/expand-cgroup.c @@ -28,7 +28,7 @@ static int test_expand_events(struct evlist *evlist) TEST_ASSERT_VAL("evlist is empty", !evlist__empty(evlist)); - nr_events = evlist->core.nr_entries; + nr_events = evlist__nr_entries(evlist); ev_name = calloc(nr_events, sizeof(*ev_name)); if (ev_name == NULL) { pr_debug("memory allocation failure\n"); @@ -54,7 +54,7 @@ static int test_expand_events(struct evlist *evlist) } ret = TEST_FAIL; - if (evlist->core.nr_entries != nr_events * nr_cgrps) { + if (evlist__nr_entries(evlist) != nr_events * nr_cgrps) { pr_debug("event count doesn't match\n"); goto out; } @@ -106,7 +106,7 @@ static int expand_default_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); ret = test_expand_events(evlist); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -133,7 +133,7 @@ static int expand_group_events(void) ret = test_expand_events(evlist); out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -164,7 +164,7 @@ static int expand_libpfm_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -179,7 +179,8 @@ static int expand_metric_events(void) TEST_ASSERT_VAL("failed to get evlist", evlist); pme_test = find_core_metrics_table("testarch", "testcpu"); - ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str); + ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str, + /*cputype_filter=*/false); if (ret < 0) { pr_debug("failed to parse '%s' metric\n", metric_str); goto out; @@ -188,7 +189,7 @@ static int expand_metric_events(void) ret = test_expand_events(evlist); out: - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/hists_cumulate.c b/tools/perf/tests/hists_cumulate.c index 606aa926a8fc..9356451a172e 100644 --- a/tools/perf/tests/hists_cumulate.c +++ b/tools/perf/tests/hists_cumulate.c @@ -87,7 +87,6 @@ static int add_hist_entries(struct hists *hists, struct machine *machine) addr_location__init(&al); for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { struct hist_entry_iter iter = { - .evsel = evsel, .sample = &sample, .hide_unresolved = false, }; @@ -705,7 +704,7 @@ static int test4(struct evsel *evsel, struct machine *machine) static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { int err = TEST_FAIL; - struct machines machines; + struct machines machines = { 0 }; struct machine *machine; struct evsel *evsel; struct evlist *evlist = evlist__new(); @@ -724,7 +723,8 @@ static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subt goto out; err = TEST_FAIL; - machines__init(&machines); + if (machines__init(&machines)) + goto out; /* setup threads/dso/map/symbols also */ machine = setup_fake_machine(&machines); @@ -744,7 +744,7 @@ static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subt out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_filter.c b/tools/perf/tests/hists_filter.c index cc6b26e373d1..f9eaa511487b 100644 --- a/tools/perf/tests/hists_filter.c +++ b/tools/perf/tests/hists_filter.c @@ -63,7 +63,6 @@ static int add_hist_entries(struct evlist *evlist, evlist__for_each_entry(evlist, evsel) { for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { struct hist_entry_iter iter = { - .evsel = evsel, .sample = &sample, .ops = &hist_iter_normal, .hide_unresolved = false, @@ -117,7 +116,7 @@ static void put_fake_samples(void) static int test__hists_filter(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { int err = TEST_FAIL; - struct machines machines; + struct machines machines = { 0 }; struct machine *machine; struct evsel *evsel; struct evlist *evlist = evlist__new(); @@ -132,7 +131,8 @@ static int test__hists_filter(struct test_suite *test __maybe_unused, int subtes goto out; err = TEST_FAIL; - machines__init(&machines); + if (machines__init(&machines)) + goto out; /* setup threads/dso/map/symbols also */ machine = setup_fake_machine(&machines); @@ -332,7 +332,7 @@ static int test__hists_filter(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_link.c b/tools/perf/tests/hists_link.c index 996f5f0b3bd1..d88591bcbe50 100644 --- a/tools/perf/tests/hists_link.c +++ b/tools/perf/tests/hists_link.c @@ -287,7 +287,7 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest { int err = -1; struct hists *hists, *first_hists; - struct machines machines; + struct machines machines = { 0 }; struct machine *machine = NULL; struct evsel *evsel, *first; struct evlist *evlist = evlist__new(); @@ -303,7 +303,8 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest goto out; err = TEST_FAIL; - machines__init(&machines); + if (machines__init(&machines)) + goto out; /* setup threads/dso/map/symbols also */ machine = setup_fake_machine(&machines); @@ -352,7 +353,7 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); reset_output_field(); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hists_output.c b/tools/perf/tests/hists_output.c index 7818950d786e..f58c8d18fe33 100644 --- a/tools/perf/tests/hists_output.c +++ b/tools/perf/tests/hists_output.c @@ -57,7 +57,6 @@ static int add_hist_entries(struct hists *hists, struct machine *machine) addr_location__init(&al); for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { struct hist_entry_iter iter = { - .evsel = evsel, .sample = &sample, .ops = &hist_iter_normal, .hide_unresolved = false, @@ -591,7 +590,7 @@ static int test5(struct evsel *evsel, struct machine *machine) static int test__hists_output(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { int err = TEST_FAIL; - struct machines machines; + struct machines machines = { 0 }; struct machine *machine; struct evsel *evsel; struct evlist *evlist = evlist__new(); @@ -611,7 +610,8 @@ static int test__hists_output(struct test_suite *test __maybe_unused, int subtes goto out; err = TEST_FAIL; - machines__init(&machines); + if (machines__init(&machines)) + goto out; /* setup threads/dso/map/symbols also */ machine = setup_fake_machine(&machines); @@ -631,7 +631,7 @@ static int test__hists_output(struct test_suite *test __maybe_unused, int subtes out: /* tear down everything */ - evlist__delete(evlist); + evlist__put(evlist); machines__exit(&machines); put_fake_samples(); diff --git a/tools/perf/tests/hwmon_pmu.c b/tools/perf/tests/hwmon_pmu.c index 4aa4aac94f09..e26b3fe3fab1 100644 --- a/tools/perf/tests/hwmon_pmu.c +++ b/tools/perf/tests/hwmon_pmu.c @@ -1,15 +1,20 @@ // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) -#include "debug.h" -#include "evlist.h" #include "hwmon_pmu.h" -#include "parse-events.h" -#include "tests.h" + #include +#include + #include -#include #include #include #include +#include + +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "pmus.h" +#include "tests.h" static const struct test_event { const char *name; @@ -179,9 +184,10 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -192,9 +198,9 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) continue; if (evsel->core.attr.config != (u64)test_events[i].key.type_and_num) { - pr_debug("FAILED %s:%d Unexpected config for '%s', %lld != %ld\n", + pr_debug("FAILED %s:%d Unexpected config for '%s', %" PRIu64 " != %ld\n", __FILE__, __LINE__, str, - evsel->core.attr.config, + (uint64_t)evsel->core.attr.config, test_events[i].key.type_and_num); ret = TEST_FAIL; goto out; @@ -210,7 +216,7 @@ static int do_test(size_t i, bool with_pmu, bool with_alias) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/kallsyms-split.c b/tools/perf/tests/kallsyms-split.c index 117ed3b70f63..244daa01bd5d 100644 --- a/tools/perf/tests/kallsyms-split.c +++ b/tools/perf/tests/kallsyms-split.c @@ -97,7 +97,7 @@ static int create_proc_dir(void) static int test__kallsyms_split(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { - struct machine m; + struct machine m = { 0 }; struct map *map = NULL; int ret = TEST_FAIL; @@ -113,7 +113,10 @@ static int test__kallsyms_split(struct test_suite *test __maybe_unused, signal(SIGTERM, remove_proc_dir); pr_debug("create kernel maps from the fake root directory\n"); - machine__init(&m, root_dir, HOST_KERNEL_ID); + if (machine__init(&m, root_dir, HOST_KERNEL_ID)) { + pr_debug("FAIL: failed to init machine\n"); + goto out; + } if (machine__create_kernel_maps(&m) < 0) { pr_debug("FAIL: failed to create kernel maps\n"); goto out; diff --git a/tools/perf/tests/keep-tracking.c b/tools/perf/tests/keep-tracking.c index 729cc9cc1cb7..b760041bed30 100644 --- a/tools/perf/tests/keep-tracking.c +++ b/tools/perf/tests/keep-tracking.c @@ -37,8 +37,8 @@ static int find_comm(struct evlist *evlist, const char *comm) int i, found; found = 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; while ((event = perf_mmap__read_event(&md->core)) != NULL) { @@ -87,7 +87,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "dummy:u")); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -106,7 +106,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); /* * First, test that a 'comm' event can be found when the event is @@ -153,7 +153,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte out_err: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/make b/tools/perf/tests/make index 6587dc326d1b..d2c2f526e1db 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -78,8 +78,6 @@ make_libperl := LIBPERL=1 make_no_libpython := NO_LIBPYTHON=1 make_no_scripts := NO_LIBPYTHON=1 make_no_slang := NO_SLANG=1 -make_no_gtk2 := NO_GTK2=1 -make_no_ui := NO_SLANG=1 NO_GTK2=1 make_no_demangle := NO_DEMANGLE=1 make_no_libelf := NO_LIBELF=1 make_no_libdw := NO_LIBDW=1 @@ -93,7 +91,7 @@ make_no_libbpf := NO_LIBBPF=1 make_libbpf_dynamic := LIBBPF_DYNAMIC=1 make_no_libbpf_DEBUG := NO_LIBBPF=1 DEBUG=1 make_no_libllvm := NO_LIBLLVM=1 -make_with_babeltrace:= LIBBABELTRACE=1 +make_no_babeltrace2 := NO_BABELTRACE2=1 make_with_coresight := CORESIGHT=1 make_no_sdt := NO_SDT=1 make_no_libpfm4 := NO_LIBPFM4=1 @@ -118,7 +116,7 @@ make_install_prefix_slash := install prefix=/tmp/krava/ make_static := LDFLAGS=-static NO_PERF_READ_VDSO32=1 NO_PERF_READ_VDSOX32=1 NO_JVMTI=1 NO_LIBTRACEEVENT=1 NO_LIBELF=1 # all the NO_* variable combined -make_minimal := NO_LIBPYTHON=1 NO_GTK2=1 +make_minimal := NO_LIBPYTHON=1 make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_BACKTRACE=1 make_minimal += NO_LIBNUMA=1 NO_LIBBIONIC=1 NO_LIBDW=1 make_minimal += NO_LIBBPF=1 @@ -153,8 +151,6 @@ run += make_libperl run += make_no_libpython run += make_no_scripts run += make_no_slang -run += make_no_gtk2 -run += make_no_ui run += make_no_demangle run += make_no_libelf run += make_no_libdw @@ -170,7 +166,7 @@ run += make_no_libbpf_DEBUG run += make_no_libllvm run += make_no_sdt run += make_no_syscall_tbl -run += make_with_babeltrace +run += make_no_babeltrace2 run += make_with_coresight run += make_with_clangllvm run += make_no_libpfm4 diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c index 3313c236104e..5cec7644952c 100644 --- a/tools/perf/tests/mmap-basic.c +++ b/tools/perf/tests/mmap-basic.c @@ -1,25 +1,29 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include #include #include + +#include +#include +#include +#include + #include +#include +#include #include "cpumap.h" #include "debug.h" #include "event.h" #include "evlist.h" #include "evsel.h" -#include "thread_map.h" +#include "pmu.h" +#include "pmus.h" #include "tests.h" +#include "thread_map.h" #include "util/affinity.h" #include "util/mmap.h" #include "util/sample.h" -#include -#include -#include -#include -#include /* * This test will generate random numbers of calls to some getpid syscalls, @@ -77,7 +81,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest goto out_free_cpus; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); for (i = 0; i < nsyscalls; ++i) { char name[64]; @@ -90,7 +94,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest /* Permissions failure, flag the failure as a skip. */ err = TEST_SKIP; } - goto out_delete_evlist; + goto out_put_evlist; } evsels[i]->core.attr.wakeup_events = 1; @@ -102,17 +106,17 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest pr_debug("failed to open counter: %s, " "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[i] = 0; expected_nr_events[i] = 1 + rand() % 127; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } for (i = 0; i < nsyscalls; ++i) @@ -120,7 +124,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest syscalls[i](); } - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -130,7 +134,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (event->header.type != PERF_RECORD_SAMPLE) { pr_debug("unexpected %s event\n", perf_event__name(event->header.type)); - goto out_delete_evlist; + goto out_put_evlist; } perf_sample__init(&sample, /*all=*/false); @@ -138,16 +142,18 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest if (err) { pr_err("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } err = -1; - evsel = evlist__id2evsel(evlist, sample.id); + evsel = sample.evsel; + if (!evsel) + evsel = evlist__id2evsel(evlist, sample.id); perf_sample__exit(&sample); if (evsel == NULL) { pr_debug("event with id %" PRIu64 " doesn't map to an evsel\n", sample.id); - goto out_delete_evlist; + goto out_put_evlist; } nr_events[evsel->core.idx]++; perf_mmap__consume(&md->core); @@ -162,12 +168,12 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest expected_nr_events[evsel->core.idx], evsel__name(evsel), nr_events[evsel->core.idx]); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } } -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out_free_cpus: perf_cpu_map__put(cpus); out_free_threads: diff --git a/tools/perf/tests/openat-syscall-all-cpus.c b/tools/perf/tests/openat-syscall-all-cpus.c index 0be43f8db3bd..cc63df2b3bc5 100644 --- a/tools/perf/tests/openat-syscall-all-cpus.c +++ b/tools/perf/tests/openat-syscall-all-cpus.c @@ -59,7 +59,7 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } perf_cpu_map__for_each_cpu(cpu, idx, cpus) { @@ -116,8 +116,8 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb evsel__free_counts(evsel); out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_cpu_map_delete: perf_cpu_map__put(cpus); out_thread_map_delete: diff --git a/tools/perf/tests/openat-syscall-tp-fields.c b/tools/perf/tests/openat-syscall-tp-fields.c index 2a139d2781a8..5365889d326f 100644 --- a/tools/perf/tests/openat-syscall-tp-fields.c +++ b/tools/perf/tests/openat-syscall-tp-fields.c @@ -51,7 +51,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (IS_ERR(evsel)) { pr_debug("%s: evsel__newtp\n", __func__); ret = PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL; - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -59,25 +59,25 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("%s: evlist__create_maps\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } evsel__config(evsel, &opts, NULL); - perf_thread_map__set_pid(evlist->core.threads, 0, getpid()); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, getpid()); err = evlist__open(evlist); if (err < 0) { pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -90,11 +90,11 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused while (1) { int before = nr_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -115,15 +115,18 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (err) { pr_debug("Can't parse sample, err = %d\n", err); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } - tp_flags = evsel__intval(evsel, &sample, "flags"); + tp_flags = perf_sample__intval(&sample, "flags"); perf_sample__exit(&sample); - if (flags != tp_flags) { + /* C library wrapper may set additional flags, + access mode must be unchanged */ + if ((tp_flags & O_ACCMODE) != (flags & O_ACCMODE) || + (tp_flags & flags) != flags) { pr_debug("%s: Expected flags=%#x, got %#x\n", __func__, flags, tp_flags); - goto out_delete_evlist; + goto out_put_evlist; } goto out_ok; @@ -136,13 +139,13 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused if (++nr_polls > 5) { pr_debug("%s: no events!\n", __func__); - goto out_delete_evlist; + goto out_put_evlist; } } out_ok: ret = TEST_OK; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); out: return ret; } diff --git a/tools/perf/tests/openat-syscall.c b/tools/perf/tests/openat-syscall.c index b54cbe5f1808..9f16f0dd3a29 100644 --- a/tools/perf/tests/openat-syscall.c +++ b/tools/perf/tests/openat-syscall.c @@ -42,7 +42,7 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, "tweak /proc/sys/kernel/perf_event_paranoid?\n", str_error_r(errno, sbuf, sizeof(sbuf))); err = TEST_SKIP; - goto out_evsel_delete; + goto out_evsel_put; } for (i = 0; i < nr_openat_calls; ++i) { @@ -64,8 +64,8 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused, err = TEST_OK; out_close_fd: perf_evsel__close_fd(&evsel->core); -out_evsel_delete: - evsel__delete(evsel); +out_evsel_put: + evsel__put(evsel); out_thread_map_delete: perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 05c3e899b425..5f6f95c8a5b4 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -109,7 +109,7 @@ static int test__checkevent_tracepoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong sample_type", @@ -122,7 +122,7 @@ static int test__checkevent_tracepoint_multi(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -144,7 +144,7 @@ static int test__checkevent_raw(struct evlist *evlist) struct evsel *evsel; bool raw_type_match = false; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { struct perf_pmu *pmu __maybe_unused = NULL; @@ -182,7 +182,7 @@ static int test__checkevent_numeric(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", 1 == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); return TEST_OK; @@ -193,7 +193,7 @@ static int test__checkevent_symbolic_name(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -207,7 +207,7 @@ static int test__checkevent_symbolic_name_config(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("unexpected event", @@ -228,7 +228,7 @@ static int test__checkevent_symbolic_alias(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_PAGE_FAULTS), evsel); return TEST_OK; @@ -238,7 +238,7 @@ static int test__checkevent_genhw(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type, evsel); @@ -251,7 +251,7 @@ static int test__checkevent_breakpoint(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -265,7 +265,7 @@ static int test__checkevent_breakpoint_x(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_X == evsel->core.attr.bp_type, evsel); @@ -278,7 +278,7 @@ static int test__checkevent_breakpoint_r(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_R == evsel->core.attr.bp_type, evsel); @@ -290,7 +290,7 @@ static int test__checkevent_breakpoint_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -302,7 +302,7 @@ static int test__checkevent_breakpoint_rw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -316,7 +316,7 @@ static int test__checkevent_tracepoint_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -330,7 +330,7 @@ test__checkevent_tracepoint_multi_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -346,7 +346,7 @@ static int test__checkevent_raw_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -361,7 +361,7 @@ static int test__checkevent_numeric_modifier(struct evlist *evlist) { struct evsel *evsel; - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -377,7 +377,7 @@ static int test__checkevent_symbolic_name_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -394,7 +394,7 @@ static int test__checkevent_exclude_host_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -409,7 +409,7 @@ static int test__checkevent_exclude_guest_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -423,7 +423,8 @@ static int test__checkevent_symbolic_alias_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -437,7 +438,7 @@ static int test__checkevent_genhw_modifier(struct evlist *evlist) struct evsel *evsel; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { @@ -454,7 +455,7 @@ static int test__checkevent_exclude_idle_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -473,7 +474,7 @@ static int test__checkevent_exclude_idle_modifier_1(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel); @@ -622,7 +623,7 @@ static int test__checkevent_breakpoint_2_events(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "breakpoint1"), evsel); @@ -641,7 +642,7 @@ static int test__checkevent_pmu(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", test_hw_config(evsel, 10), evsel); TEST_ASSERT_EVSEL("wrong config1", 1 == evsel->core.attr.config1, evsel); @@ -661,7 +662,7 @@ static int test__checkevent_list(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist__nr_entries(evlist), evsel); /* r1 */ TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT != evsel->core.attr.type, evsel); @@ -707,14 +708,15 @@ static int test__checkevent_pmu_name(struct evlist *evlist) char buf[256]; /* default_core/config=1,name=krava/u */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "krava"), evsel); /* default_core/config=2/u" */ evsel = evsel__next(evsel); - TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel); + TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 2 == evsel->core.attr.config, evsel); snprintf(buf, sizeof(buf), "%s/config=2/u", core_pmu->name); @@ -729,7 +731,8 @@ static int test__checkevent_pmu_partial_time_callgraph(struct evlist *evlist) struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); /* default_core/config=1,call-graph=fp,time,period=100000/ */ - TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", + 2 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel); /* @@ -760,7 +763,7 @@ static int test__checkevent_pmu_events(struct evlist *evlist) struct evsel *evsel; struct perf_pmu *core_pmu = perf_pmus__find_core_pmu(); - TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist__nr_entries(evlist), evlist); evlist__for_each_entry(evlist, evsel) { TEST_ASSERT_EVSEL("wrong type", @@ -787,8 +790,9 @@ static int test__checkevent_pmu_events_mix(struct evlist *evlist) * The wild card event will be opened at least once, but it may be * opened on each core PMU. */ - TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries >= 2, evlist); - for (int i = 0; i < evlist->core.nr_entries - 1; i++) { + TEST_ASSERT_EVLIST("wrong number of entries", + evlist__nr_entries(evlist) >= 2, evlist); + for (int i = 0; i < evlist__nr_entries(evlist) - 1; i++) { evsel = (i == 0 ? evlist__first(evlist) : evsel__next(evsel)); /* pmu-event:u */ TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); @@ -905,7 +909,7 @@ static int test__group1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -950,7 +954,7 @@ static int test__group2(struct evlist *evlist) struct evsel *evsel, *leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist) + 1), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist) + 1), evlist); /* * TODO: Currently the software event won't be grouped with the hardware @@ -1018,7 +1022,7 @@ static int test__group3(struct evlist *evlist __maybe_unused) struct evsel *evsel, *group1_leader = NULL, *group2_leader = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * perf_pmus__num_core_pmus() + 2), + evlist__nr_entries(evlist) == (3 * perf_pmus__num_core_pmus() + 2), evlist); /* * Currently the software event won't be grouped with the hardware event @@ -1144,7 +1148,7 @@ static int test__group4(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (num_core_entries(evlist) * 2), + evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2), evlist); TEST_ASSERT_EVLIST("wrong number of groups", num_core_entries(evlist) == evlist__nr_groups(evlist), @@ -1191,7 +1195,7 @@ static int test__group5(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (5 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (5 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == (2 * num_core_entries(evlist)), @@ -1284,7 +1288,7 @@ static int test__group_gh1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1329,7 +1333,7 @@ static int test__group_gh2(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1374,7 +1378,7 @@ static int test__group_gh3(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1419,7 +1423,7 @@ static int test__group_gh4(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); TEST_ASSERT_EVLIST("wrong number of groups", evlist__nr_groups(evlist) == num_core_entries(evlist), @@ -1464,7 +1468,7 @@ static int test__leader_sample1(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1520,7 +1524,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (2 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1562,7 +1566,7 @@ static int test__checkevent_pinned_modifier(struct evlist *evlist) struct evsel *evsel = NULL; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1581,7 +1585,7 @@ static int test__pinned_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == (3 * num_core_entries(evlist)), + evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1618,7 +1622,7 @@ static int test__checkevent_exclusive_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1634,7 +1638,7 @@ static int test__exclusive_group(struct evlist *evlist) struct evsel *evsel = NULL, *leader; TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 3 * num_core_entries(evlist), + evlist__nr_entries(evlist) == 3 * num_core_entries(evlist), evlist); for (int i = 0; i < num_core_entries(evlist); i++) { @@ -1669,7 +1673,7 @@ static int test__checkevent_breakpoint_len(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", @@ -1684,7 +1688,7 @@ static int test__checkevent_breakpoint_len_w(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel); TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel); @@ -1698,7 +1702,7 @@ test__checkevent_breakpoint_len_rw_modifier(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel); @@ -1712,7 +1716,7 @@ static int test__checkevent_precise_max_modifier(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == 1 + num_core_entries(evlist), + evlist__nr_entries(evlist) == 1 + num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK), evsel); return TEST_OK; @@ -1723,7 +1727,7 @@ static int test__checkevent_config_symbol(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "insn"), evsel); return TEST_OK; @@ -1733,7 +1737,7 @@ static int test__checkevent_config_raw(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "rawpmu"), evsel); return TEST_OK; } @@ -1742,7 +1746,7 @@ static int test__checkevent_config_num(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "numpmu"), evsel); return TEST_OK; } @@ -1752,7 +1756,7 @@ static int test__checkevent_config_cache(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "cachepmu"), evsel); return test__checkevent_genhw(evlist); @@ -1777,7 +1781,7 @@ static int test__intel_pt(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "intel_pt//u"), evsel); return TEST_OK; } @@ -1798,7 +1802,8 @@ static int test__ratio_to_prev(struct evlist *evlist) { struct evsel *evsel, *leader; - TEST_ASSERT_VAL("wrong number of entries", 2 * perf_pmus__num_core_pmus() == evlist->core.nr_entries); + TEST_ASSERT_VAL("wrong number of entries", + 2 * perf_pmus__num_core_pmus() == evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, evsel) { if (evsel != evsel__leader(evsel) || @@ -1842,7 +1847,7 @@ static int test__checkevent_complex_name(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong complex name parsing", evsel__name_is(evsel, @@ -1855,7 +1860,7 @@ static int test__checkevent_raw_pmu(struct evlist *evlist) { struct evsel *evsel = evlist__first(evlist); - TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist); + TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist); TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type, evsel); TEST_ASSERT_EVSEL("wrong config", 0x1a == evsel->core.attr.config, evsel); return TEST_OK; @@ -1866,7 +1871,7 @@ static int test__sym_event_slash(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel); @@ -1878,7 +1883,7 @@ static int test__sym_event_dc(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel); @@ -1890,7 +1895,7 @@ static int test__term_equal_term(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "name") == 0, evsel); @@ -1902,7 +1907,7 @@ static int test__term_equal_legacy(struct evlist *evlist) struct evsel *evsel = evlist__first(evlist); TEST_ASSERT_EVLIST("wrong number of entries", - evlist->core.nr_entries == num_core_entries(evlist), + evlist__nr_entries(evlist) == num_core_entries(evlist), evlist); TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel); TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "l1d") == 0, evsel); @@ -1958,7 +1963,7 @@ static int count_tracepoints(void) static int test__all_tracepoints(struct evlist *evlist) { TEST_ASSERT_VAL("wrong events count", - count_tracepoints() == evlist->core.nr_entries); + count_tracepoints() == evlist__nr_entries(evlist)); return test__checkevent_tracepoint_multi(evlist); } @@ -2556,8 +2561,10 @@ static int test_event(const struct evlist_test *e) return TEST_FAIL; } parse_events_error__init(&err); - ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, &err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/true); + ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", e->name, ret); parse_events_error__print(&err, e->name); @@ -2568,7 +2575,7 @@ static int test_event(const struct evlist_test *e) ret = e->check(evlist); } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -2584,8 +2591,9 @@ static int test_event_fake_pmu(const char *str) return -ENOMEM; parse_events_error__init(&err); - ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, &err, - /*fake_pmu=*/true, /*warn_if_reordered=*/true, + ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, &err, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, /*fake_tp=*/true); if (ret) { pr_debug("failed to parse event '%s', err %d\n", @@ -2594,7 +2602,7 @@ static int test_event_fake_pmu(const char *str) } parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/parse-metric.c b/tools/perf/tests/parse-metric.c index 7c7f489a5eb0..872d9a7aa72f 100644 --- a/tools/perf/tests/parse-metric.c +++ b/tools/perf/tests/parse-metric.c @@ -53,7 +53,7 @@ static double compute_single(struct evlist *evlist, const char *name) struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { - me = metricgroup__lookup(&evlist->metric_events, evsel, false); + me = metricgroup__lookup(evlist__metric_events(evlist), evsel, false); if (me != NULL) { list_for_each_entry (mexp, &me->head, nd) { if (strcmp(mexp->metric_name, name)) @@ -84,15 +84,16 @@ static int __compute_metric(const char *name, struct value *vals, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); /* Parse the metric into metric_events list. */ pme_test = find_core_metrics_table("testarch", "testcpu"); - err = metricgroup__parse_groups_test(evlist, pme_test, name); + err = metricgroup__parse_groups_test(evlist, pme_test, name, + /*cputype_filter=*/false); if (err) goto out; @@ -113,7 +114,7 @@ static int __compute_metric(const char *name, struct value *vals, /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/parse-no-sample-id-all.c b/tools/perf/tests/parse-no-sample-id-all.c index 50e68b7d43aa..78bb8db19240 100644 --- a/tools/perf/tests/parse-no-sample-id-all.c +++ b/tools/perf/tests/parse-no-sample-id-all.c @@ -49,7 +49,7 @@ static int process_events(union perf_event **events, size_t count) for (i = 0; i < count && !err; i++) err = process_event(&evlist, events[i]); - evlist__delete(evlist); + evlist__put(evlist); return err; } @@ -82,6 +82,9 @@ static int test__parse_no_sample_id_all(struct test_suite *test __maybe_unused, .type = PERF_RECORD_HEADER_ATTR, .size = sizeof(struct test_attr_event), }, + .attr = { + .size = sizeof(struct perf_event_attr), + }, .id = 1, }; struct test_attr_event event2 = { @@ -89,6 +92,9 @@ static int test__parse_no_sample_id_all(struct test_suite *test __maybe_unused, .type = PERF_RECORD_HEADER_ATTR, .size = sizeof(struct test_attr_event), }, + .attr = { + .size = sizeof(struct perf_event_attr), + }, .id = 2, }; struct perf_record_mmap event3 = { diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c index ad44cc68820b..0cac6ae1a1fc 100644 --- a/tools/perf/tests/perf-record.c +++ b/tools/perf/tests/perf-record.c @@ -105,7 +105,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__create_maps(evlist, &opts.target); if (err < 0) { pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -117,7 +117,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest err = evlist__prepare_workload(evlist, &opts.target, argv, false, NULL); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -129,12 +129,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest evsel__set_sample_bit(evsel, TIME); evlist__config(evlist, &opts, NULL); - err = sched__get_first_possible_cpu(evlist->workload.pid, cpu_mask); + err = sched__get_first_possible_cpu(evlist__workload_pid(evlist), cpu_mask); if (err < 0) { pr_debug("sched__get_first_possible_cpu: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } cpu = err; @@ -142,11 +142,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest /* * So that we can check perf_sample.cpu on all the samples. */ - if (sched_setaffinity(evlist->workload.pid, cpu_mask_size, cpu_mask) < 0) { + if (sched_setaffinity(evlist__workload_pid(evlist), cpu_mask_size, cpu_mask) < 0) { pr_debug("sched_setaffinity: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -158,7 +158,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest pr_debug("perf_evlist__open: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -166,12 +166,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest * fds in the same CPU to be injected in the same mmap ring buffer * (using ioctl(PERF_EVENT_IOC_SET_OUTPUT)). */ - err = evlist__mmap(evlist, opts.mmap_pages); + err = evlist__do_mmap(evlist, opts.mmap_pages); if (err < 0) { pr_debug("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); evlist__cancel_workload(evlist); - goto out_delete_evlist; + goto out_put_evlist; } /* @@ -188,11 +188,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest while (1) { int before = total_events; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { union perf_event *event; struct mmap *md; - md = &evlist->mmap[i]; + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -209,7 +209,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest if (verbose > 0) perf_event__fprintf(event, NULL, stderr); pr_debug("Couldn't parse sample\n"); - goto out_delete_evlist; + goto out_put_evlist; } if (verbose > 0) { @@ -231,15 +231,15 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest ++errs; } - if ((pid_t)sample.pid != evlist->workload.pid) { + if ((pid_t)sample.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid, expected %d, got %d\n", - name, evlist->workload.pid, sample.pid); + name, evlist__workload_pid(evlist), sample.pid); ++errs; } - if ((pid_t)sample.tid != evlist->workload.pid) { + if ((pid_t)sample.tid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected tid, expected %d, got %d\n", - name, evlist->workload.pid, sample.tid); + name, evlist__workload_pid(evlist), sample.tid); ++errs; } @@ -248,7 +248,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest type == PERF_RECORD_MMAP2 || type == PERF_RECORD_FORK || type == PERF_RECORD_EXIT) && - (pid_t)event->comm.pid != evlist->workload.pid) { + (pid_t)event->comm.pid != evlist__workload_pid(evlist)) { pr_debug("%s with unexpected pid/tid\n", name); ++errs; } @@ -350,11 +350,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest pr_debug("PERF_RECORD_MMAP for %s missing!\n", "[vdso]"); ++errs; } -out_delete_evlist: +out_put_evlist: CPU_FREE(cpu_mask); - evlist__delete(evlist); out: perf_sample__exit(&sample); + evlist__put(evlist); if (err == -EACCES) return TEST_SKIP; if (err < 0 || errs != 0) diff --git a/tools/perf/tests/perf-time-to-tsc.c b/tools/perf/tests/perf-time-to-tsc.c index cca41bd37ae3..f8f71fdd32b1 100644 --- a/tools/perf/tests/perf-time-to-tsc.c +++ b/tools/perf/tests/perf-time-to-tsc.c @@ -99,7 +99,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist = evlist__new(); CHECK_NOT_NULL__(evlist); - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); CHECK__(parse_event(evlist, "cpu-cycles:u")); @@ -121,9 +121,9 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su goto out_err; } - CHECK__(evlist__mmap(evlist, UINT_MAX)); + CHECK__(evlist__do_mmap(evlist, UINT_MAX)); - pc = evlist->mmap[0].core.base; + pc = evlist__mmap(evlist)[0].core.base; ret = perf_read_tsc_conversion(pc, &tc); if (ret) { if (ret == -EOPNOTSUPP) { @@ -145,8 +145,8 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su evlist__disable(evlist); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -201,7 +201,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su err = TEST_OK; out_err: - evlist__delete(evlist); + evlist__put(evlist); perf_cpu_map__put(cpus); perf_thread_map__put(threads); return err; diff --git a/tools/perf/tests/pfm.c b/tools/perf/tests/pfm.c index fca4a86452df..f7bf55be5e6e 100644 --- a/tools/perf/tests/pfm.c +++ b/tools/perf/tests/pfm.c @@ -69,18 +69,18 @@ static int test__pfm_events(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), 0); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } @@ -154,18 +154,18 @@ static int test__pfm_group(struct test_suite *test __maybe_unused, if (evlist == NULL) return -ENOMEM; - opt.value = evlist; + opt.value = &evlist; parse_libpfm_events_option(&opt, table[i].events, 0); TEST_ASSERT_EQUAL(table[i].events, - count_pfm_events(&evlist->core), + count_pfm_events(evlist__core(evlist)), table[i].nr_events); TEST_ASSERT_EQUAL(table[i].events, evlist__nr_groups(evlist), table[i].nr_groups); - evlist__delete(evlist); + evlist__put(evlist); } return 0; } diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index a99716862168..f507ce39439a 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -15,6 +15,7 @@ #include "util/expr.h" #include "util/hashmap.h" #include "util/parse-events.h" +#include "util/tool_pmu.h" #include "metricgroup.h" #include "stat.h" @@ -793,11 +794,13 @@ static int check_parse_id(const char *id, struct parse_events_error *error) for (cur = strchr(dup, '@') ; cur; cur = strchr(++cur, '@')) *cur = '/'; - ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, error, /*fake_pmu=*/true, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, error, /*fake_pmu=*/true, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); free(dup); - evlist__delete(evlist); + evlist__put(evlist); return ret; } @@ -817,6 +820,26 @@ struct metric { struct metric_ref metric_ref; }; +static bool is_expected_broken_metric(const struct pmu_metric *pm) +{ + if (!strcmp(pm->metric_name, "M1") || !strcmp(pm->metric_name, "M2") || + !strcmp(pm->metric_name, "M3")) + return true; + +#if defined(__aarch64__) + /* + * Arm64 platforms may return "#slots == 0", which is treated as a + * syntax error by the parser. Don't test these metrics when running + * on such platforms. + */ + if (strstr(pm->metric_expr, "#slots") && + !tool_pmu__cpu_slots_per_cycle()) + return true; +#endif + + return false; +} + static int test__parsing_callback(const struct pmu_metric *pm, const struct pmu_metrics_table *table, void *data) @@ -844,16 +867,17 @@ static int test__parsing_callback(const struct pmu_metric *pm, cpus = perf_cpu_map__new("0"); if (!cpus) { - evlist__delete(evlist); + evlist__put(evlist); return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, NULL); + perf_evlist__set_maps(evlist__core(evlist), cpus, NULL); - err = metricgroup__parse_groups_test(evlist, table, pm->metric_name); + err = metricgroup__parse_groups_test(evlist, table, + pm->metric_name, + /*cputype_filter=*/false); if (err) { - if (!strcmp(pm->metric_name, "M1") || !strcmp(pm->metric_name, "M2") || - !strcmp(pm->metric_name, "M3")) { + if (is_expected_broken_metric(pm)) { (*failures)--; pr_debug("Expected broken metric %s skipping\n", pm->metric_name); err = 0; @@ -875,7 +899,8 @@ static int test__parsing_callback(const struct pmu_metric *pm, k++; } evlist__for_each_entry(evlist, evsel) { - struct metric_event *me = metricgroup__lookup(&evlist->metric_events, evsel, false); + struct metric_event *me = metricgroup__lookup(evlist__metric_events(evlist), + evsel, false); if (me != NULL) { struct metric_expr *mexp; @@ -899,17 +924,24 @@ static int test__parsing_callback(const struct pmu_metric *pm, /* ... cleanup. */ evlist__free_stats(evlist); perf_cpu_map__put(cpus); - evlist__delete(evlist); + evlist__put(evlist); return err; } -static int test__parsing(struct test_suite *test __maybe_unused, - int subtest __maybe_unused) +static int test__parsing(struct test_suite *test, int subtest) { int failures = 0; + const struct pmu_metrics_table *table = NULL; + + if (test->test_cases) + table = test->test_cases[subtest].priv; - pmu_for_each_core_metric(test__parsing_callback, &failures); - pmu_for_each_sys_metric(test__parsing_callback, &failures); + if (table) { + pmu_metrics_table__for_each_metric(table, test__parsing_callback, &failures); + } else { + pmu_for_each_core_metric(test__parsing_callback, &failures); + pmu_for_each_sys_metric(test__parsing_callback, &failures); + } return failures == 0 ? TEST_OK : TEST_FAIL; } @@ -1000,8 +1032,8 @@ static int test__parsing_fake_callback(const struct pmu_metric *pm, * Parse all the metrics for current architecture, or all defined cpus via the * 'fake_pmu' in parse_events. */ -static int test__parsing_fake(struct test_suite *test __maybe_unused, - int subtest __maybe_unused) +static int test__parsing_fake_static(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) { int err = 0; @@ -1011,6 +1043,26 @@ static int test__parsing_fake(struct test_suite *test __maybe_unused, return err; } + return 0; +} + +static int test__parsing_fake(struct test_suite *test, int subtest) +{ + int err = 0; + const struct pmu_metrics_table *table = NULL; + + if (test->test_cases) + table = test->test_cases[subtest].priv; + + if (table) + return pmu_metrics_table__for_each_metric(table, test__parsing_fake_callback, NULL); + + for (size_t i = 0; i < ARRAY_SIZE(metrics); i++) { + err = metric_parse_fake("", metrics[i].str); + if (err) + return err; + } + err = pmu_for_each_core_metric(test__parsing_fake_callback, NULL); if (err) return err; @@ -1039,17 +1091,130 @@ static int test__parsing_threshold(struct test_suite *test __maybe_unused, return pmu_for_each_sys_metric(test__parsing_threshold_callback, NULL); } +struct populate_cb_data { + struct test_case *test_cases; + size_t curr; +}; + +static int count_metrics_tables_cb(const struct pmu_metrics_table *table __maybe_unused, void *data) +{ + size_t *count = data; + (*count)++; + return 0; +} + +static int populate_metrics_tables_cb(const struct pmu_metrics_table *table, void *data) +{ + struct populate_cb_data *cb_data = data; + const char *table_name = pmu_metrics_table__name(table); + char *desc_real, *desc_fake; + + if (!table_name) + table_name = "unknown"; + + if (asprintf(&desc_real, "PMU metric parsing: %s", table_name) < 0) + return -ENOMEM; + if (asprintf(&desc_fake, "PMU metric parsing with fake PMU: %s", table_name) < 0) { + free(desc_real); + return -ENOMEM; + } + + cb_data->test_cases[cb_data->curr++] = (struct test_case){ + .name = "parsing", + .desc = desc_real, + .run_case = test__parsing, + .priv = (void *)table, + .skip_reason = "some metrics failed", + }; + + cb_data->test_cases[cb_data->curr++] = (struct test_case){ + .name = "parsing_fake", + .desc = desc_fake, + .run_case = test__parsing_fake, + .priv = (void *)table, + }; + + return 0; +} + +static struct test_case pmu_events_tests[]; + +static int setup_pmu_events_suite(struct test_suite *suite) +{ + size_t num_tables = 0; + size_t num_fixed_tests = 4; + size_t tests_per_table = 2; + size_t total_tests; + struct test_case *test_cases; + size_t curr = 0; + struct populate_cb_data cb_data; + int ret; + + if (suite->test_cases != pmu_events_tests) + return 0; + + ret = pmu_metrics_table__iterate_tables(count_metrics_tables_cb, &num_tables); + if (ret) + return ret; + + total_tests = num_fixed_tests + (num_tables * tests_per_table) + 1; + + test_cases = calloc(total_tests, sizeof(*test_cases)); + if (!test_cases) + return -ENOMEM; + + test_cases[curr++] = (struct test_case){ + .name = "pmu_event_table", + .desc = "PMU event table sanity", + .run_case = test__pmu_event_table, + }; + test_cases[curr++] = (struct test_case){ + .name = "aliases", + .desc = "PMU event map aliases", + .run_case = test__aliases, + }; + test_cases[curr++] = (struct test_case){ + .name = "parsing_fake_static", + .desc = "Parsing of static metrics with fake PMU", + .run_case = test__parsing_fake_static, + }; + test_cases[curr++] = (struct test_case){ + .name = "parsing_threshold", + .desc = "Parsing of metric thresholds with fake PMU", + .run_case = test__parsing_threshold, + }; + + cb_data = (struct populate_cb_data){ + .test_cases = test_cases, + .curr = curr, + }; + + ret = pmu_metrics_table__iterate_tables(populate_metrics_tables_cb, &cb_data); + if (ret) { + size_t i; + + for (i = num_fixed_tests; i < cb_data.curr; i++) + free((char *)test_cases[i].desc); + free(test_cases); + return ret; + } + + suite->test_cases = test_cases; + return 0; +} + static struct test_case pmu_events_tests[] = { TEST_CASE("PMU event table sanity", pmu_event_table), TEST_CASE("PMU event map aliases", aliases), TEST_CASE_REASON("Parsing of PMU event table metrics", parsing, "some metrics failed"), - TEST_CASE("Parsing of PMU event table metrics with fake PMUs", parsing_fake), - TEST_CASE("Parsing of metric thresholds with fake PMUs", parsing_threshold), + TEST_CASE("Parsing of PMU event table metrics with fake PMU", parsing_fake), + TEST_CASE("Parsing of metric thresholds with fake PMU", parsing_threshold), { .name = NULL, } }; struct test_suite suite__pmu_events = { .desc = "PMU JSON event tests", .test_cases = pmu_events_tests, + .setup = setup_pmu_events_suite, }; diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index 0ebf2d7b2cb4..13e8d7fa80af 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -201,7 +201,8 @@ static int test__pmu_format(struct test_suite *test __maybe_unused, int subtest return ret; } -static int test__pmu_usr_chgs(struct test_suite *test __maybe_unused, int subtest __maybe_unused) +static int test__pmu_config_helpers(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) { const char *event = "perf-pmu-test/config=15,config1=4,krava02=170," "krava03=1,krava11=27,krava12=1/"; @@ -236,6 +237,12 @@ static int test__pmu_usr_chgs(struct test_suite *test __maybe_unused, int subtes } evsel = evlist__first(evlist); + /* Test evsel__config_exists() */ + TEST_ASSERT_EQUAL("krava01 should exist", + evsel__config_exists(evsel, "krava01"), true); + TEST_ASSERT_EQUAL("krava99 should not exist", + evsel__config_exists(evsel, "krava99"), false); + /* * Set via config=15, krava01 bits 0-1 * Set via config1=4, krava11 bit 1 @@ -287,7 +294,7 @@ static int test__pmu_usr_chgs(struct test_suite *test __maybe_unused, int subtes ret = TEST_OK; err_out: parse_events_terms__exit(&terms); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } @@ -339,7 +346,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest ret = TEST_OK; err_out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); test_pmu_put(dir, pmu); return ret; } @@ -629,7 +636,7 @@ static struct test_case tests__pmu[] = { TEST_CASE("PMU name combining", name_len), TEST_CASE("PMU name comparison", name_cmp), TEST_CASE("PMU cmdline match", pmu_match), - TEST_CASE("PMU user config changes", pmu_usr_chgs), + TEST_CASE("PMU config helpers", pmu_config_helpers), { .name = NULL, } }; diff --git a/tools/perf/tests/sample-parsing.c b/tools/perf/tests/sample-parsing.c index a7327c942ca2..20cab91ceaeb 100644 --- a/tools/perf/tests/sample-parsing.c +++ b/tools/perf/tests/sample-parsing.c @@ -205,15 +205,11 @@ static bool samples_same(struct perf_sample *s1, static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) { - struct evsel evsel = { - .needs_swap = false, - .core = { - . attr = { - .sample_type = sample_type, - .read_format = read_format, - }, - }, + struct perf_event_attr attr = { + .sample_type = sample_type, + .read_format = read_format, }; + struct evsel *evsel; union perf_event *event; union { struct ip_callchain callchain; @@ -287,16 +283,21 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) size_t i, sz, bufsz; int err, ret = -1; + evsel = evsel__new(&attr); + if (!evsel) { + pr_debug("evsel__new failed\n"); + return -1; + } perf_sample__init(&sample_out, /*all=*/false); perf_sample__init(&sample_out_endian, /*all=*/false); if (sample_type & PERF_SAMPLE_REGS_USER) - evsel.core.attr.sample_regs_user = sample_regs; + evsel->core.attr.sample_regs_user = sample_regs; if (sample_type & PERF_SAMPLE_REGS_INTR) - evsel.core.attr.sample_regs_intr = sample_regs; + evsel->core.attr.sample_regs_intr = sample_regs; if (sample_type & PERF_SAMPLE_BRANCH_STACK) - evsel.core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; for (i = 0; i < sizeof(regs); i++) *(i + (u8 *)regs) = i & 0xfe; @@ -310,12 +311,13 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) sample.read.one.lost = 1; } - sz = perf_event__sample_event_size(&sample, sample_type, read_format); + sz = perf_event__sample_event_size(&sample, sample_type, read_format, + evsel->core.attr.branch_sample_type); bufsz = sz + 4096; /* Add a bit for overrun checking */ event = malloc(bufsz); if (!event) { pr_debug("malloc failed\n"); - return -1; + goto out_free; } memset(event, 0xff, bufsz); @@ -324,7 +326,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) event->header.size = sz; err = perf_event__synthesize_sample(event, sample_type, read_format, - &sample); + evsel->core.attr.branch_sample_type, &sample); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "perf_event__synthesize_sample", sample_type, err); @@ -342,32 +344,33 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) goto out_free; } - evsel.sample_size = __evsel__sample_size(sample_type); + evsel->sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out); + err = evsel__parse_sample(evsel, event, &sample_out); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; } if (sample_type == PERF_SAMPLE_BRANCH_STACK) { - evsel.needs_swap = true; - evsel.sample_size = __evsel__sample_size(sample_type); - err = evsel__parse_sample(&evsel, event, &sample_out_endian); + evsel->needs_swap = true; + evsel->sample_size = __evsel__sample_size(sample_type); + err = evsel__parse_sample(evsel, event, &sample_out_endian); if (err) { pr_debug("%s failed for sample_type %#"PRIx64", error %d\n", "evsel__parse_sample", sample_type, err); goto out_free; } - if (!samples_same(&sample, &sample_out_endian, sample_type, read_format, evsel.needs_swap)) { + if (!samples_same(&sample, &sample_out_endian, sample_type, + read_format, evsel->needs_swap)) { pr_debug("parsing failed for sample_type %#"PRIx64"\n", sample_type); goto out_free; @@ -379,6 +382,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format) free(event); perf_sample__exit(&sample_out_endian); perf_sample__exit(&sample_out); + evsel__put(evsel); if (ret && read_format) pr_debug("read_format %#"PRIx64"\n", read_format); return ret; diff --git a/tools/perf/tests/shell/amd-ibs-swfilt.sh b/tools/perf/tests/shell/amd-ibs-swfilt.sh index e7f66df05c4b..5d17dc624cbe 100755 --- a/tools/perf/tests/shell/amd-ibs-swfilt.sh +++ b/tools/perf/tests/shell/amd-ibs-swfilt.sh @@ -1,6 +1,33 @@ #!/bin/bash # AMD IBS software filtering +cpu_family() { + grep -m1 '^cpu family[[:space:]]*:' /proc/cpuinfo \ + | awk -F: '{gsub(/^[ \t]+/, "", $2); print $2}' +} + +cpu_model() { + grep -m1 '^model[[:space:]]*:' /proc/cpuinfo \ + | awk -F: '{gsub(/^[ \t]+/, "", $2); print $2}' +} + +# IBS PMUs does not advertize privilege filtering capability. Rely +# on Family / Model check. +hw_priv_filter_supported() { + family=$(cpu_family) + model=$(cpu_model) + + if (( family > 0x1a )) || + { (( family == 0x1a )) && + { { (( model >= 0x50 && model <= 0x5f )) || + (( model >= 0x80 && model <= 0xaf )) || + (( model >= 0xc0 && model <= 0xcf )); }; }; }; then + return 0 # True + else + return 1 # False + fi +} + ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } @@ -23,10 +50,12 @@ echo "run perf record with modifier and swfilt" err=0 # setting any modifiers should fail -perf record -B -e ibs_op//u -o /dev/null true 2> /dev/null -if [ $? -eq 0 ]; then - echo "[FAIL] IBS PMU should not accept exclude_kernel" - exit 1 +if ! hw_priv_filter_supported; then + perf record -B -e ibs_op//u -o /dev/null true 2> /dev/null + if [ $? -eq 0 ]; then + echo "[FAIL] IBS PMU should not accept exclude_kernel" + exit 1 + fi fi # setting it with swfilt should be fine diff --git a/tools/perf/tests/shell/coresight/Makefile b/tools/perf/tests/shell/coresight/Makefile deleted file mode 100644 index fa08fd9a5991..000000000000 --- a/tools/perf/tests/shell/coresight/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Carsten Haitzler , 2021 -include ../../../../../tools/scripts/Makefile.include -include ../../../../../tools/scripts/Makefile.arch -include ../../../../../tools/scripts/utilities.mak - -SUBDIRS = \ - asm_pure_loop \ - memcpy_thread \ - thread_loop \ - unroll_loop_thread - -all: $(SUBDIRS) -$(SUBDIRS): - @$(MAKE) -C $@ >/dev/null - -INSTALLDIRS = $(SUBDIRS:%=install-%) - -install-tests: $(INSTALLDIRS) -$(INSTALLDIRS): - @$(MAKE) -C $(@:install-%=%) install-tests >/dev/null - -CLEANDIRS = $(SUBDIRS:%=clean-%) - -clean: $(CLEANDIRS) -$(CLEANDIRS): - $(call QUIET_CLEAN, test-$(@:clean-%=%)) $(MAKE) -C $(@:clean-%=%) clean >/dev/null - -.PHONY: all clean $(SUBDIRS) $(CLEANDIRS) $(INSTALLDIRS) diff --git a/tools/perf/tests/shell/coresight/Makefile.miniconfig b/tools/perf/tests/shell/coresight/Makefile.miniconfig deleted file mode 100644 index 5f72a9cb43f3..000000000000 --- a/tools/perf/tests/shell/coresight/Makefile.miniconfig +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Carsten Haitzler , 2021 - -ifndef DESTDIR -prefix ?= $(HOME) -endif - -DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) -INSTALL = install -INSTDIR_SUB = tests/shell/coresight - -include ../../../../../scripts/Makefile.include -include ../../../../../scripts/Makefile.arch -include ../../../../../scripts/utilities.mak diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop.sh b/tools/perf/tests/shell/coresight/asm_pure_loop.sh deleted file mode 100755 index 0301904b9637..000000000000 --- a/tools/perf/tests/shell/coresight/asm_pure_loop.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# CoreSight / ASM Pure Loop (exclusive) - -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -TEST="asm_pure_loop" - -# shellcheck source=../lib/coresight.sh -. "$(dirname $0)"/../lib/coresight.sh - -ARGS="" -DATV="out" -# shellcheck disable=SC2153 -DATA="$DATD/perf-$TEST-$DATV.data" - -perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS - -perf_dump_aux_verify "$DATA" 10 10 10 - -err=$? -exit $err diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore b/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore deleted file mode 100644 index 468673ac32e8..000000000000 --- a/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore +++ /dev/null @@ -1 +0,0 @@ -asm_pure_loop diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile b/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile deleted file mode 100644 index 206849e92bc9..000000000000 --- a/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -include ../Makefile.miniconfig - -# Binary to produce -BIN=asm_pure_loop -# Any linking/libraries needed for the binary - empty if none needed -LIB= - -all: $(BIN) - -$(BIN): $(BIN).S -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Build line - this is raw asm with no libc to have an always exact binary - $(Q)$(CC) $(BIN).S -nostdlib -static -o $(BIN) $(LIB) -endif -endif - -install-tests: all -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Install the test tool in the right place - $(call QUIET_INSTALL, tests) \ - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ - $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' -endif -endif - -clean: - $(Q)$(RM) -f $(BIN) - -.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S b/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S deleted file mode 100644 index 577760046772..000000000000 --- a/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Tamas Zsoldos , 2021 */ - -.globl _start -_start: - mov x0, 0x0000ffff - mov x1, xzr -loop: - nop - nop - cbnz x1, noskip - nop - nop - adrp x2, skip - add x2, x2, :lo12:skip - br x2 - nop - nop -noskip: - nop - nop -skip: - sub x0, x0, 1 - cbnz x0, loop - - mov x0, #0 - mov x8, #93 // __NR_exit syscall - svc #0 - -.section .note.GNU-stack, "", @progbits diff --git a/tools/perf/tests/shell/coresight/callchain.sh b/tools/perf/tests/shell/coresight/callchain.sh new file mode 100755 index 000000000000..13cca7dc1118 --- /dev/null +++ b/tools/perf/tests/shell/coresight/callchain.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# CoreSight synthesized callchain (exclusive) +# SPDX-License-Identifier: GPL-2.0 + +glb_err=1 + +if ! tmpdir=$(mktemp -d /tmp/perf-cs-callchain-test.XXXXXX); then + echo "mktemp failed" + exit 1 +fi + +cleanup_files() +{ + rm -rf "$tmpdir" +} + +trap cleanup_files EXIT +trap 'cleanup_files; exit $glb_err' TERM INT + +skip_if_system_is_not_ready() +{ + perf list | grep -Pzq 'cs_etm//' || { + echo "[Skip] cs_etm event is not available" >&2 + return 2 + } + + # Requires root for trace in kernel + [ "$(id -u)" = 0 ] || { + echo "[Skip] No root permission" >&2 + return 2 + } + + return 0 +} + +record_trace() +{ + local data=$1 + local script=$2 + + local cf="$tmpdir/ctl" + local af="$tmpdir/ack" + + mkfifo "$cf" "$af" + + perf record -o "$data" -e cs_etm// --per-thread -D -1 --control fifo:"$cf","$af" -- \ + perf test --record-ctl fifo:"$cf","$af" -w callchain >/dev/null 2>&1 && + + # It is safe to use 'i3i' with a three-instruction interval, since the + # workload is compiled with -O0. + perf script --itrace=g16i3il64 -i "$data" > "$script" +} + +callchain_regex_1() +{ + printf '%s' \ +'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'([[:space:]]+[[:xdigit:]]+ .*\n)*' +} + +callchain_regex_2() +{ + printf '%s' \ +'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'([[:space:]]+[[:xdigit:]]+ .*\n)*' +} + +callchain_regex_3() +{ + printf '%s' \ +'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\ +'[[:space:]]+[[:xdigit:]]+ syscall(@plt)?\+0x[[:xdigit:]]+ \(.*\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'([[:space:]]+[[:xdigit:]]+ .*\n)*' +} + +callchain_regex_4() +{ + printf '%s' \ +'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\ +'[[:space:]]+[[:xdigit:]]+ .*\+0x[[:xdigit:]]+ \(\[kernel\.kallsyms\]\)\n'\ +'[[:space:]]+[[:xdigit:]]+ syscall(@plt)?\+0x[[:xdigit:]]+ \(.*\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\ +'([[:space:]]+[[:xdigit:]]+ .*\n)*' +} + +find_after_line() +{ + local regex="$1" + local file="$2" + local start="$3" + local offset + local line + + # Search in byte offset + offset=$( + tail -n +"$start" "$file" | + grep -Pzob -m1 "$regex" | + tr '\0' '\n' | + sed -n 's/^\([0-9][0-9]*\):.*/\1/p;q' + ) + + if [ -z "$offset" ]; then + echo "Failed to match regex after line $start" >&2 + echo "Regex:" >&2 + printf '%s\n' "$regex" >&2 + echo "Context from line $start:" >&2 + sed -n "${start},$((start + 100))p" "$file" >&2 + return 1 + fi + + # Convert from offset to line + line=$( + tail -n +"$start" "$file" | + head -c "$offset" | + wc -l + ) + + echo "$((start + line))" +} + +check_callchain_flow() +{ + local file="$1" + local l1 l2 l3 l4 l5 l6 l7 + + # Callchain push + l1=$(find_after_line "$(callchain_regex_1)" "$file" 1) || return 1 + l2=$(find_after_line "$(callchain_regex_2)" "$file" "$((l1 + 1))") || return 1 + l3=$(find_after_line "$(callchain_regex_3)" "$file" "$((l2 + 1))") || return 1 + l4=$(find_after_line "$(callchain_regex_4)" "$file" "$((l3 + 1))") || return 1 + + # Callchain pop + l5=$(find_after_line "$(callchain_regex_3)" "$file" "$((l4 + 1))") || return 1 + l6=$(find_after_line "$(callchain_regex_2)" "$file" "$((l5 + 1))") || return 1 + l7=$(find_after_line "$(callchain_regex_1)" "$file" "$((l6 + 1))") || return 1 + + echo "Callchain flow matched:" + echo " l1=$l1 l2=$l2 l3=$l3 l4=$l4 l5=$l5 l6=$l6 l7=$l7" + + return 0 +} + +run_test() +{ + local data=$tmpdir/perf.data + local script=$tmpdir/perf.script + + if ! record_trace "$data" "$script"; then + echo "perf record/script failed" + return + fi + + check_callchain_flow "$script" || return + + glb_err=0 +} + +skip_if_system_is_not_ready || exit 2 + +run_test + +exit $glb_err diff --git a/tools/perf/tests/shell/coresight/concurrent_threads.sh b/tools/perf/tests/shell/coresight/concurrent_threads.sh new file mode 100755 index 000000000000..3349fff8c767 --- /dev/null +++ b/tools/perf/tests/shell/coresight/concurrent_threads.sh @@ -0,0 +1,45 @@ +#!/bin/bash -e +# CoreSight concurrent threads (exclusive) + +# SPDX-License-Identifier: GPL-2.0 + +# If CoreSight is not available, skip the test +perf list pmu | grep -q cs_etm || exit 2 + +tmpdir=$(mktemp -d /tmp/__perf_test.coresight_concurrent_threads.XXXXX) + +cleanup() { + rm -rf "${tmpdir}" + trap - EXIT TERM INT +} + +trap_cleanup() { + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +cf="$tmpdir/ctl" +af="$tmpdir/ack" +mkfifo "$cf" "$af" + +nthreads=10 + +# Timestamps off to reduce trace size, start disabled and use the control FIFO +# to only record the workload and not startup. +perf record -o "$tmpdir/data" -e cs_etm/timestamp=0/u -D -1 --control fifo:"$cf","$af" \ + -- perf test --record-ctl fifo:"$cf","$af" -w named_threads $nthreads 1 > /dev/null 2>&1 + +perf script -i "$tmpdir/data" > "$tmpdir/script" 2>/dev/null + +# Check all threads were traced and they have the correct thread name and symbol +for i in $(seq 1 $nthreads); do + if ! grep -q "thread${i} .* named_threads_thread${i}" "$tmpdir/script"; then + echo "Error: thread${i} missing" >&2 + cleanup + exit 1 + fi +done + +cleanup +exit 0 diff --git a/tools/perf/tests/shell/coresight/context_switch_thread.sh b/tools/perf/tests/shell/coresight/context_switch_thread.sh new file mode 100755 index 000000000000..2b9c44b86c59 --- /dev/null +++ b/tools/perf/tests/shell/coresight/context_switch_thread.sh @@ -0,0 +1,69 @@ +#!/bin/bash -e +# CoreSight context switch thread attribution (exclusive) + +# SPDX-License-Identifier: GPL-2.0 + +# If CoreSight is not available, skip the test +perf list pmu | grep -q cs_etm || exit 2 + +if [ "$(id -u)" != 0 ]; then + # Requires root for "-C 0" in record command + echo "[Skip] No root permission" + exit 2 +fi + +tmpdir=$(mktemp -d /tmp/__perf_test.coresight_context_switch.XXXXX) + +cleanup() { + rm -rf "${tmpdir}" + trap - EXIT TERM INT +} + +trap_cleanup() { + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +check_samples() { + owner_samples=$(grep -c "proc1.*context_switch_loop_proc1" "$tmpdir/script" || true) + next_samples=$(grep -c "proc2.*context_switch_loop_proc2" "$tmpdir/script" || true) + + if [ "$owner_samples" -eq 0 ] || [ "$next_samples" -eq 0 ]; then + echo "No samples found" + cleanup + exit 1 + fi + + if grep "proc2.*context_switch_loop_proc1" "$tmpdir/script"; then + echo "Thread1 symbol was attributed to proc2" + cleanup + exit 1 + fi + + if grep "proc1.*context_switch_loop_proc2" "$tmpdir/script"; then + echo "Thread2 symbol was attributed to proc1" + cleanup + exit 1 + fi +} + +cf="$tmpdir/ctl" +af="$tmpdir/ack" +mkfifo "$cf" "$af" + +# Pin to one CPU so the two threads alternate running but record into the same +# trace buffer. Start disabled and use the control FIFO to only record the +# workload and not startup. +perf record -o "$tmpdir/data" -e cs_etm/timestamp=0/u -C 0 -D -1 --control fifo:"$cf","$af" -- \ + taskset --cpu-list 0 perf test --record-ctl fifo:"$cf","$af" \ + -w context_switch_loop > /dev/null 2>&1 + +# Test both instruction and branch sample generation modes. +perf script -i "$tmpdir/data" --itrace=i4 -F comm,pid,tid,ip,sym > "$tmpdir/script" 2>/dev/null +check_samples +perf script -i "$tmpdir/data" --itrace=b -F comm,pid,tid,ip,sym > "$tmpdir/script" 2>/dev/null +check_samples + +cleanup +exit 0 diff --git a/tools/perf/tests/shell/coresight/deterministic.sh b/tools/perf/tests/shell/coresight/deterministic.sh new file mode 100755 index 000000000000..75d4973056f0 --- /dev/null +++ b/tools/perf/tests/shell/coresight/deterministic.sh @@ -0,0 +1,72 @@ +#!/bin/bash -e +# CoreSight deterministic workload decode (exclusive) + +# SPDX-License-Identifier: GPL-2.0 + +# If CoreSight is not available, skip the test +perf list pmu | grep -q cs_etm || exit 2 + +tmpdir=$(mktemp -d /tmp/__perf_test.coresight_deterministic.XXXXX) + +cleanup() { + rm -rf "${tmpdir}" + trap - EXIT TERM INT +} + +trap_cleanup() { + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +cf="$tmpdir/ctl" +af="$tmpdir/ack" +mkfifo "$cf" "$af" + +# Start disabled and use the control FIFO to only record the workload and not +# startup. +perf record -o "$tmpdir/data" -e cs_etm//u -D -1 --control fifo:"$cf","$af" -- \ + perf test --record-ctl fifo:"$cf","$af" -w deterministic > /dev/null 2>&1 + +perf script -i "$tmpdir/data" --itrace=i1i -F ip,srcline | \ + grep "deterministic.c" | uniq > "$tmpdir/script" 2>/dev/null + + +# Remove brace lines and call sites as they may not be hit or may have +# extra hits after returning, depending on the compiler. +sed -i \ + -e '/deterministic.c:8$/d' \ + -e '/deterministic.c:12$/d' \ + -e '/deterministic.c:15$/d' \ + -e '/deterministic.c:19$/d' \ + -e '/deterministic.c:23$/d' \ + -e '/deterministic.c:28$/d' \ + -e '/deterministic.c:34$/d' \ + -e '/deterministic.c:36$/d' \ + -e '/deterministic.c:37$/d' \ + "$tmpdir/script" + +cat > "$tmpdir/expected" << EOF + deterministic.c:24 + deterministic.c:25 + deterministic.c:26 + deterministic.c:9 + deterministic.c:10 + deterministic.c:11 + deterministic.c:30 + deterministic.c:31 + deterministic.c:32 + deterministic.c:16 + deterministic.c:17 + deterministic.c:18 +EOF + +if ! diff -q "$tmpdir/script" "$tmpdir/expected"; then + echo "FAIL: line numbers don't match expected: " + head -n 100 "$tmpdir/script" + cleanup + exit 1 +fi + +cleanup +exit 0 diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore b/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore deleted file mode 100644 index f8217e56091e..000000000000 --- a/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore +++ /dev/null @@ -1 +0,0 @@ -memcpy_thread diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/Makefile b/tools/perf/tests/shell/coresight/memcpy_thread/Makefile deleted file mode 100644 index 2db637eb2c26..000000000000 --- a/tools/perf/tests/shell/coresight/memcpy_thread/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 -include ../Makefile.miniconfig - -# Binary to produce -BIN=memcpy_thread -# Any linking/libraries needed for the binary - empty if none needed -LIB=-pthread - -all: $(BIN) - -$(BIN): $(BIN).c -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Build line - $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) -endif -endif - -install-tests: all -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Install the test tool in the right place - $(call QUIET_INSTALL, tests) \ - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ - $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' -endif -endif - -clean: - $(Q)$(RM) -f $(BIN) - -.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c b/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c deleted file mode 100644 index 7e879217be30..000000000000 --- a/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Carsten Haitzler , 2021 -#include -#include -#include -#include -#include - -struct args { - unsigned long loops; - unsigned long size; - pthread_t th; - void *ret; -}; - -static void *thrfn(void *arg) -{ - struct args *a = arg; - unsigned long i, len = a->loops; - unsigned char *src, *dst; - - src = malloc(a->size * 1024); - dst = malloc(a->size * 1024); - if ((!src) || (!dst)) { - printf("ERR: Can't allocate memory\n"); - exit(1); - } - for (i = 0; i < len; i++) - memcpy(dst, src, a->size * 1024); - - return NULL; -} - -static pthread_t new_thr(void *(*fn) (void *arg), void *arg) -{ - pthread_t t; - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_create(&t, &attr, fn, arg); - return t; -} - -int main(int argc, char **argv) -{ - unsigned long i, len, size, thr; - struct args args[256]; - long long v; - - if (argc < 4) { - printf("ERR: %s [copysize Kb] [numthreads] [numloops (hundreds)]\n", argv[0]); - exit(1); - } - - v = atoll(argv[1]); - if ((v < 1) || (v > (1024 * 1024))) { - printf("ERR: max memory 1GB (1048576 KB)\n"); - exit(1); - } - size = v; - thr = atol(argv[2]); - if ((thr < 1) || (thr > 256)) { - printf("ERR: threads 1-256\n"); - exit(1); - } - v = atoll(argv[3]); - if ((v < 1) || (v > 40000000000ll)) { - printf("ERR: loops 1-40000000000 (hundreds)\n"); - exit(1); - } - len = v * 100; - for (i = 0; i < thr; i++) { - args[i].loops = len; - args[i].size = size; - args[i].th = new_thr(thrfn, &(args[i])); - } - for (i = 0; i < thr; i++) - pthread_join(args[i].th, &(args[i].ret)); - return 0; -} diff --git a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh deleted file mode 100755 index 1f765d69acc3..000000000000 --- a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# CoreSight / Memcpy 16k 10 Threads (exclusive) - -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -TEST="memcpy_thread" - -# shellcheck source=../lib/coresight.sh -. "$(dirname $0)"/../lib/coresight.sh - -ARGS="16 10 1" -DATV="16k_10" -# shellcheck disable=SC2153 -DATA="$DATD/perf-$TEST-$DATV.data" - -perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS - -perf_dump_aux_verify "$DATA" 10 10 10 - -err=$? -exit $err diff --git a/tools/perf/tests/shell/coresight/raw_dump_stress.sh b/tools/perf/tests/shell/coresight/raw_dump_stress.sh new file mode 100755 index 000000000000..bea70d825596 --- /dev/null +++ b/tools/perf/tests/shell/coresight/raw_dump_stress.sh @@ -0,0 +1,65 @@ +#!/bin/bash -e +# CoreSight raw dump stress (exclusive) + +# SPDX-License-Identifier: GPL-2.0 + +if [ "$(id -u)" != 0 ]; then + # Requires root for larger buffer size + echo "[Skip] No root permission" + exit 2 +fi + +# If CoreSight is not available, skip the test +perf list pmu | grep -q cs_etm || exit 2 + +tmpdir=$(mktemp -d /tmp/__perf_test.coresight_raw_dump_stress.XXXXX) + +cleanup() { + rm -r "${tmpdir}" + trap - EXIT TERM INT +} + +trap_cleanup() { + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +# Use exit snapshot to record 2M of trace to make about 80MB of raw dump data. +echo "Recording..." +perf record -e cs_etm/timestamp=0/u -m,2M -Se -o "$tmpdir/data" -- \ + perf test -w brstack 20000 > /dev/null 2>&1 + +# Test raw dump runs to completion but don't decode because that's too slow for +# a test +echo "Dumping raw trace..." +perf report --dump-raw-trace -i "$tmpdir/data" 2>/dev/null > "$tmpdir/rawdump" + +# Get the size and offset of the first AUXTRACE buffer and the index of the last +# packet in the raw dump. +read -r size offset last_idx <<< "$(awk ' + found && /PERF_RECORD_/ { exit } + /PERF_RECORD_AUXTRACE / { found = 1; size = $7; offset = $9; next } + found && /Idx:/ { last_idx = $1; gsub(/Idx:|;/, "", last_idx) } + END { if (last_idx) print size, offset, last_idx } +' "$tmpdir/rawdump")" + +# The last Idx minus start offset should equal the size of the buffer if +# everything was dumped. Allow 48 bytes difference to cover 3 frames: current +# frame length, a partial frame and a final empty one, all of which aren't +# dumped. +# +# TODO: for a single snapshot, offset should always be zero. However, we +# currently output AUX records in snapshot mode when we shouldn't, which +# increments the offset. Allow for that until it's fixed so we can test raw +# dumping. +decode_size=$((1 + last_idx - offset)) +if [ "$decode_size" -gt "$((size - 48))" ] && [ "$decode_size" -le "$((size))" ]; then + echo "PASS: AUXTRACE buffer length matches dumped packet index" + cleanup + exit 0 +fi + +echo "FAIL: AUXTRACE buffer length mismatch: size=$size offset=$offset last_idx=$last_idx decode_size=$decode_size" +cleanup +exit 1 diff --git a/tools/perf/tests/shell/test_arm_coresight.sh b/tools/perf/tests/shell/coresight/test_arm_coresight.sh similarity index 83% rename from tools/perf/tests/shell/test_arm_coresight.sh rename to tools/perf/tests/shell/coresight/test_arm_coresight.sh index bbf89e944e7b..83295a8fe179 100755 --- a/tools/perf/tests/shell/test_arm_coresight.sh +++ b/tools/perf/tests/shell/coresight/test_arm_coresight.sh @@ -20,6 +20,12 @@ skip_if_no_cs_etm_event() { skip_if_no_cs_etm_event || exit 2 +if [ "$(id -u)" != 0 ]; then + # Requires root for -C and system wide tests + echo "[Skip] No root permission" + exit 2 +fi + perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) file=$(mktemp /tmp/temporary_file.XXXXX) @@ -52,17 +58,6 @@ perf_script_branch_samples() { grep -E " +$1 +[0-9]+ .* +branches:(.*:)? +" > /dev/null 2>&1 } -perf_report_branch_samples() { - echo "Looking at perf.data file for reporting branch samples:" - - # Below is an example of the branch samples reporting: - # 73.04% 73.04% touch libc-2.27.so [.] _dl_addr - # 7.71% 7.71% touch libc-2.27.so [.] getenv - # 2.59% 2.59% touch ld-2.27.so [.] strcmp - perf report --stdio -i ${perfdata} 2>&1 | \ - grep -E " +[0-9]+\.[0-9]+% +[0-9]+\.[0-9]+% +$1 " > /dev/null 2>&1 -} - perf_report_instruction_samples() { echo "Looking at perf.data file for instruction samples:" @@ -123,7 +118,6 @@ arm_cs_iterate_devices() { record_touch_file $device_name $2 && perf_script_branch_samples touch && - perf_report_branch_samples touch && perf_report_instruction_samples touch err=$? @@ -154,9 +148,7 @@ arm_cs_etm_system_wide_test() { # System-wide mode should include perf samples so test for that # instead of ls - perf_script_branch_samples perf && - perf_report_branch_samples perf && - perf_report_instruction_samples perf + perf_script_branch_samples perf err=$? arm_cs_report "CoreSight system wide testing" $err @@ -164,7 +156,7 @@ arm_cs_etm_system_wide_test() { arm_cs_etm_snapshot_test() { echo "Recording trace with snapshot mode" - perf record -o ${perfdata} -e cs_etm// -S \ + perf record -o ${perfdata} -e cs_etm// -S -m,128K \ -- dd if=/dev/zero of=/dev/null > /dev/null 2>&1 & PERFPID=$! @@ -179,7 +171,6 @@ arm_cs_etm_snapshot_test() { wait $PERFPID perf_script_branch_samples dd && - perf_report_branch_samples dd && perf_report_instruction_samples dd err=$? @@ -188,11 +179,9 @@ arm_cs_etm_snapshot_test() { arm_cs_etm_basic_test() { echo "Recording trace with '$*'" - perf record -o ${perfdata} "$@" -m,8M -- ls > /dev/null 2>&1 + perf record -o ${perfdata} "$@" -- true > /dev/null 2>&1 - perf_script_branch_samples ls && - perf_report_branch_samples ls && - perf_report_instruction_samples ls + perf_script_branch_samples true err=$? arm_cs_report "CoreSight basic testing with '$*'" $err @@ -256,12 +245,12 @@ arm_cs_etm_snapshot_test # Test all combinations of per-thread, system-wide and normal mode with # and without timestamps -arm_cs_etm_basic_test -e cs_etm/timestamp=0/ --per-thread -arm_cs_etm_basic_test -e cs_etm/timestamp=1/ --per-thread -arm_cs_etm_basic_test -e cs_etm/timestamp=0/ -a -arm_cs_etm_basic_test -e cs_etm/timestamp=1/ -a -arm_cs_etm_basic_test -e cs_etm/timestamp=0/ -arm_cs_etm_basic_test -e cs_etm/timestamp=1/ +arm_cs_etm_basic_test -e cs_etm/timestamp=0/u --per-thread +arm_cs_etm_basic_test -e cs_etm/timestamp=1/u --per-thread +arm_cs_etm_basic_test -e cs_etm/timestamp=0/u -a +arm_cs_etm_basic_test -e cs_etm/timestamp=1/u -a +arm_cs_etm_basic_test -e cs_etm/timestamp=0/u +arm_cs_etm_basic_test -e cs_etm/timestamp=1/u arm_cs_etm_sparse_cpus_test diff --git a/tools/perf/tests/shell/test_arm_coresight_disasm.sh b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh similarity index 59% rename from tools/perf/tests/shell/test_arm_coresight_disasm.sh rename to tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh index 0dfb4fadf531..f3ebad596378 100755 --- a/tools/perf/tests/shell/test_arm_coresight_disasm.sh +++ b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh @@ -24,7 +24,7 @@ perfdata_dir=$(mktemp -d /tmp/__perf_test.perf.data.XXXXX) perfdata=${perfdata_dir}/perf.data file=$(mktemp /tmp/temporary_file.XXXXX) # Relative path works whether it's installed or running from repo -script_path=$(dirname "$0")/../../scripts/python/arm-cs-trace-disasm.py +script_path=$(dirname "$0")/../../../scripts/python/arm-cs-trace-disasm.py cleanup_files() { @@ -38,28 +38,27 @@ cleanup_files() trap cleanup_files EXIT TERM INT # Ranges start and end on branches, so check for some likely branch instructions -sep="\s\|\s" -branch_search="\sbl${sep}b${sep}b.ne${sep}b.eq${sep}cbz\s" +branch_search='[[:space:]](bl|b(\.(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al))?|br|blr|ret|cbz|cbnz|tbz|tbnz|svc|eret)([[:space:]]|$)' ## Test kernel ## -if [ -e /proc/kcore ]; then +if [ "$(id -u)" == 0 ] && [ -e /proc/kcore ]; then echo "Testing kernel disassembly" - perf record -o ${perfdata} -e cs_etm//k --kcore -- touch $file > /dev/null 2>&1 - perf script -i ${perfdata} -s python:${script_path} -- \ - -d --stop-sample=30 2> /dev/null > ${file} - grep -q -e ${branch_search} ${file} + perf record -o ${perfdata} -e cs_etm//k --kcore -Se -m,64K -- touch $file > /dev/null 2>&1 + perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ + -d --stop-sample=2 -k ${perfdata}/kcore_dir/kcore 2> /dev/null > ${file} + grep -q -E ${branch_search} ${file} echo "Found kernel branches" else - # kcore is required for correct kernel decode due to runtime code patching - echo "No kcore, skipping kernel test" + # Root and kcore are required for correct kernel decode due to runtime code patching + echo "No root or kcore, skipping kernel test" fi ## Test user ## echo "Testing userspace disassembly" -perf record -o ${perfdata} -e cs_etm//u -- touch $file > /dev/null 2>&1 -perf script -i ${perfdata} -s python:${script_path} -- \ - -d --stop-sample=30 2> /dev/null > ${file} -grep -q -e ${branch_search} ${file} +perf record -o ${perfdata} -e cs_etm//u -Se -m,64K -- touch $file > /dev/null 2>&1 +perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \ + -d --stop-sample=2 2> /dev/null > ${file} +grep -q -E ${branch_search} ${file} echo "Found userspace branches" glb_err=0 diff --git a/tools/perf/tests/shell/coresight/thread_loop/.gitignore b/tools/perf/tests/shell/coresight/thread_loop/.gitignore deleted file mode 100644 index 6d4c33eaa9e8..000000000000 --- a/tools/perf/tests/shell/coresight/thread_loop/.gitignore +++ /dev/null @@ -1 +0,0 @@ -thread_loop diff --git a/tools/perf/tests/shell/coresight/thread_loop/Makefile b/tools/perf/tests/shell/coresight/thread_loop/Makefile deleted file mode 100644 index ea846c038e7a..000000000000 --- a/tools/perf/tests/shell/coresight/thread_loop/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 -include ../Makefile.miniconfig - -# Binary to produce -BIN=thread_loop -# Any linking/libraries needed for the binary - empty if none needed -LIB=-pthread - -all: $(BIN) - -$(BIN): $(BIN).c -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Build line - $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) -endif -endif - -install-tests: all -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Install the test tool in the right place - $(call QUIET_INSTALL, tests) \ - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ - $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' -endif -endif - -clean: - $(Q)$(RM) -f $(BIN) - -.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c b/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c deleted file mode 100644 index 86f3f548b006..000000000000 --- a/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Carsten Haitzler , 2021 - -// define this for gettid() -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#ifndef SYS_gettid -// gettid is 178 on arm64 -# define SYS_gettid 178 -#endif -#define gettid() syscall(SYS_gettid) - -struct args { - unsigned int loops; - pthread_t th; - void *ret; -}; - -static void *thrfn(void *arg) -{ - struct args *a = arg; - int i = 0, len = a->loops; - - if (getenv("SHOW_TID")) { - unsigned long long tid = gettid(); - - printf("%llu\n", tid); - } - asm volatile( - "loop:\n" - "add %w[i], %w[i], #1\n" - "cmp %w[i], %w[len]\n" - "blt loop\n" - : /* out */ - : /* in */ [i] "r" (i), [len] "r" (len) - : /* clobber */ - ); - return (void *)(long)i; -} - -static pthread_t new_thr(void *(*fn) (void *arg), void *arg) -{ - pthread_t t; - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_create(&t, &attr, fn, arg); - return t; -} - -int main(int argc, char **argv) -{ - unsigned int i, len, thr; - struct args args[256]; - - if (argc < 3) { - printf("ERR: %s [numthreads] [numloops (millions)]\n", argv[0]); - exit(1); - } - - thr = atoi(argv[1]); - if ((thr < 1) || (thr > 256)) { - printf("ERR: threads 1-256\n"); - exit(1); - } - len = atoi(argv[2]); - if ((len < 1) || (len > 4000)) { - printf("ERR: max loops 4000 (millions)\n"); - exit(1); - } - len *= 1000000; - for (i = 0; i < thr; i++) { - args[i].loops = len; - args[i].th = new_thr(thrfn, &(args[i])); - } - for (i = 0; i < thr; i++) - pthread_join(args[i].th, &(args[i].ret)); - return 0; -} diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh deleted file mode 100755 index 7f43a93a2ac2..000000000000 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e -# CoreSight / Thread Loop 10 Threads - Check TID (exclusive) - -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -TEST="thread_loop" - -# shellcheck source=../lib/coresight.sh -. "$(dirname $0)"/../lib/coresight.sh - -ARGS="10 1" -DATV="check-tid-10th" -# shellcheck disable=SC2153 -DATA="$DATD/perf-$TEST-$DATV.data" -STDO="$DATD/perf-$TEST-$DATV.stdout" - -SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO - -perf_dump_aux_tid_verify "$DATA" "$STDO" - -err=$? -exit $err diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh deleted file mode 100755 index a94d2079ed06..000000000000 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -e -# CoreSight / Thread Loop 2 Threads - Check TID (exclusive) - -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -TEST="thread_loop" - -# shellcheck source=../lib/coresight.sh -. "$(dirname $0)"/../lib/coresight.sh - -ARGS="2 20" -DATV="check-tid-2th" -# shellcheck disable=SC2153 -DATA="$DATD/perf-$TEST-$DATV.data" -STDO="$DATD/perf-$TEST-$DATV.stdout" - -SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO - -perf_dump_aux_tid_verify "$DATA" "$STDO" - -err=$? -exit $err diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore b/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore deleted file mode 100644 index 2cb4e996dbf3..000000000000 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore +++ /dev/null @@ -1 +0,0 @@ -unroll_loop_thread diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile b/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile deleted file mode 100644 index 6264c4e3abd1..000000000000 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 -include ../Makefile.miniconfig - -# Binary to produce -BIN=unroll_loop_thread -# Any linking/libraries needed for the binary - empty if none needed -LIB=-pthread - -all: $(BIN) - -$(BIN): $(BIN).c -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Build line - $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) -endif -endif - -install-tests: all -ifdef CORESIGHT -ifeq ($(ARCH),arm64) -# Install the test tool in the right place - $(call QUIET_INSTALL, tests) \ - $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ - $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' -endif -endif - -clean: - $(Q)$(RM) -f $(BIN) - -.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c b/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c deleted file mode 100644 index 8f4e1c985ca3..000000000000 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Carsten Haitzler , 2021 -#include -#include -#include -#include -#include - -struct args { - pthread_t th; - unsigned int in; - void *ret; -}; - -static void *thrfn(void *arg) -{ - struct args *a = arg; - unsigned int i, in = a->in; - - for (i = 0; i < 10000; i++) { - asm volatile ( -// force an unroll of thia add instruction so we can test long runs of code -#define SNIP1 "add %w[in], %w[in], #1\n" -// 10 -#define SNIP2 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 -// 100 -#define SNIP3 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 -// 1000 -#define SNIP4 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 -// 10000 -#define SNIP5 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 -// 100000 - SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 - : /* out */ - : /* in */ [in] "r" (in) - : /* clobber */ - ); - } - - return NULL; -} - -static pthread_t new_thr(void *(*fn) (void *arg), void *arg) -{ - pthread_t t; - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_create(&t, &attr, fn, arg); - return t; -} - -int main(int argc, char **argv) -{ - unsigned int i, thr; - struct args args[256]; - - if (argc < 2) { - printf("ERR: %s [numthreads]\n", argv[0]); - exit(1); - } - - thr = atoi(argv[1]); - if ((thr > 256) || (thr < 1)) { - printf("ERR: threads 1-256\n"); - exit(1); - } - for (i = 0; i < thr; i++) { - args[i].in = rand(); - args[i].th = new_thr(thrfn, &(args[i])); - } - for (i = 0; i < thr; i++) - pthread_join(args[i].th, &(args[i].ret)); - return 0; -} diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh deleted file mode 100755 index cb3e97a0a89f..000000000000 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e -# CoreSight / Unroll Loop Thread 10 (exclusive) - -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -TEST="unroll_loop_thread" - -# shellcheck source=../lib/coresight.sh -. "$(dirname $0)"/../lib/coresight.sh - -ARGS="10" -DATV="10" -# shellcheck disable=SC2153 -DATA="$DATD/perf-$TEST-$DATV.data" - -perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS - -perf_dump_aux_verify "$DATA" 10 10 10 - -err=$? -exit $err diff --git a/tools/perf/tests/shell/data_file_offset_diagnostics.sh b/tools/perf/tests/shell/data_file_offset_diagnostics.sh new file mode 100755 index 000000000000..389b0b84752d --- /dev/null +++ b/tools/perf/tests/shell/data_file_offset_diagnostics.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Test that perf report includes file offsets and event type names in diagnostic messages. +# SPDX-License-Identifier: GPL-2.0 +# +# The file_offset diagnostic series adds "at offset 0x...: TYPE (N)" +# to all skip/stop/error messages. This test corrupts an event's size +# field in a perf.data file, then verifies the resulting warning +# includes the file offset and event type. + +err=0 + +cleanup() { + [ -n "${perfdata}" ] && rm -f "${perfdata}" "${perfdata}.old" + rm -f "${corrupted}" "${stderrfile}" + trap - EXIT TERM INT +} +trap 'cleanup; exit 1' TERM INT +trap cleanup EXIT + +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 +corrupted=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 +stderrfile=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 + +if ! perf record -o "${perfdata}" -- perf test -w noploop 2>/dev/null; then + echo "Skip: perf record failed" + cleanup + exit 2 +fi + +# Find the file offset of the first MMAP2 event via perf report -D. +# Format: "timestamp 0xOFFSET [0xSIZE]: PERF_RECORD_MMAP2 ..." +mmap2_line=$(perf report -D -i "${perfdata}" 2>/dev/null | grep "PERF_RECORD_MMAP2" | head -1) +if [ -z "${mmap2_line}" ]; then + echo "Skip: no MMAP2 events found in perf.data" + cleanup + exit 2 +fi + +mmap2_offset=$(echo "${mmap2_line}" | awk '{print $2}') +mmap2_offset_dec=$((mmap2_offset)) + +# Copy the file and corrupt the MMAP2 event's size field. +# perf_event_header layout: type(u32) misc(u16) size(u16) +# Set size to 16 — below the MMAP2 minimum, which triggers +# the "event size too small" warning. +# +# perf.data uses native byte order, so write the u16 in the +# host's endianness to work on both LE and BE architectures. +cp "${perfdata}" "${corrupted}" +if printf '\x01\x02' | od -A n -t x2 | grep -q "0201"; then + # Little-endian + printf '\x10\x00' +else + # Big-endian + printf '\x00\x10' +fi | dd of="${corrupted}" bs=1 seek=$((mmap2_offset_dec + 6)) conv=notrunc 2>/dev/null + +perf report -i "${corrupted}" --stdio > /dev/null 2> "${stderrfile}" + +# Check that warnings include "at offset 0x..." +if grep -q "at offset 0x" "${stderrfile}"; then + echo "File offset in diagnostics [Success]" +else + echo "File offset in diagnostics [Failed: no 'at offset 0x...' found]" + echo " stderr was:" + head -5 "${stderrfile}" + err=1 +fi + +# Check that the event type name and numeric id appear: "MMAP2 (10)" +if grep -q "MMAP2 (10)" "${stderrfile}"; then + echo "Event type name in diagnostics [Success]" +else + echo "Event type name in diagnostics [Failed: no 'MMAP2 (10)' found]" + echo " stderr was:" + head -5 "${stderrfile}" + err=1 +fi + +# Check that the reported offset matches the actual corruption point +if grep -q "at offset ${mmap2_offset}:" "${stderrfile}"; then + echo "Correct offset reported [Success]" +else + echo "Correct offset reported [Failed: expected offset ${mmap2_offset}]" + echo " stderr was:" + head -5 "${stderrfile}" + err=1 +fi + +cleanup +exit ${err} diff --git a/tools/perf/tests/shell/data_validation.sh b/tools/perf/tests/shell/data_validation.sh new file mode 100755 index 000000000000..bf16b2f2b911 --- /dev/null +++ b/tools/perf/tests/shell/data_validation.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Test that perf report handles truncated perf.data gracefully (no crash, no segfault — clean error exit). +# SPDX-License-Identifier: GPL-2.0 +# +# Exercises the bounds checking and minimum-size validation added +# by the perf-data-validation hardening series. + +err=0 + +cleanup() { + [ -n "${perfdata}" ] && rm -f "${perfdata}" "${perfdata}.old" + rm -f "${truncated}" "${stderrfile}" + trap - EXIT TERM INT +} +trap 'cleanup; exit 1' TERM INT +trap cleanup EXIT + +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 +truncated=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 +stderrfile=$(mktemp /tmp/__perf_test.perf.data.XXXXX) || exit 2 + +# Record a simple workload +if ! perf record -o "${perfdata}" -- perf test -w noploop 2>/dev/null; then + echo "Skip: perf record failed" + cleanup + exit 2 +fi + +file_size=$(wc -c < "${perfdata}") +if [ "${file_size}" -lt 512 ]; then + echo "Skip: perf.data too small (${file_size} bytes)" + cleanup + exit 2 +fi + +# Test truncation at various offsets that exercise different +# parsing stages: +# 8 — file header magic only, no attrs or data +# 64 — partial file header (attr section incomplete) +# 256 — into the first events (partial event headers) +# 75% — mid-stream truncation (partial event data) +for cut_at in 8 64 256 $((file_size * 3 / 4)); do + if [ "${cut_at}" -ge "${file_size}" ]; then + continue + fi + dd if="${perfdata}" of="${truncated}" bs="${cut_at}" count=1 2>/dev/null + + # perf report should exit with an error, not crash. + # Capture stderr to detect sanitizer violations. + perf report -i "${truncated}" --stdio > /dev/null 2> "${stderrfile}" + exit_code=$? + + # A truncated file should never parse successfully + if [ ${exit_code} -eq 0 ]; then + echo "FAIL: perf report exited 0 (success) on ${cut_at}-byte truncated file — expected an error" + err=1 + continue + fi + + # Detect sanitizer violations — ASAN/MSAN/TSAN/UBSAN exit + # with code 1 by default, which would otherwise look like a + # clean error exit. Check stderr for their markers. + if grep -qE "^(==[0-9]+==ERROR:|SUMMARY: [A-Za-z]*Sanitizer)" "${stderrfile}" 2>/dev/null; then + sanitizer=$(grep -oE "(Address|Memory|Thread|UndefinedBehavior)Sanitizer" "${stderrfile}" | head -1) + echo "FAIL: perf report triggered ${sanitizer:-sanitizer} on ${cut_at}-byte truncated file" + err=1 + continue + fi + + # Detect crash signals portably — signal numbers differ + # across architectures (e.g. SIGBUS is 7 on x86/ARM but + # 10 on MIPS/SPARC). Use kill -l to map the number to a + # name on the running system. + if [ ${exit_code} -gt 128 ] && [ ${exit_code} -lt 200 ]; then + sig_name=$(kill -l $((exit_code - 128)) 2>/dev/null) + case ${sig_name} in + KILL|ILL|ABRT|BUS|FPE|SEGV|TRAP|SYS) + echo "FAIL: perf report crashed (SIG${sig_name}) on ${cut_at}-byte truncated file" + err=1 + ;; + esac + fi +done + +cleanup +exit ${err} diff --git a/tools/perf/tests/shell/ftrace.sh b/tools/perf/tests/shell/ftrace.sh index 7f8aafcbb761..9f6e590f6437 100755 --- a/tools/perf/tests/shell/ftrace.sh +++ b/tools/perf/tests/shell/ftrace.sh @@ -71,9 +71,10 @@ test_ftrace_profile() { grep ^# "${output}" time_re="[[:space:]]+1[[:digit:]]{5}\.[[:digit:]]{3}" # 100283.000 100283.000 100283.000 1 __x64_sys_clock_nanosleep - # Check for one *clock_nanosleep line with a Count of just 1 that takes a bit more than 0.1 seconds - # Strip the _x64_sys part to work with other architectures - grep -E "^${time_re}${time_re}${time_re}[[:space:]]+1[[:space:]]+.*clock_nanosleep" "${output}" + # Check for one *sys_*nanosleep line with a Count of just 1 that takes a bit more than 0.1 seconds + # Strip the _x64_ part to work with other architectures, strip the clock part to support + # C libraries that use the nanosleep syscall instead of clock_nanosleep + grep -E "^${time_re}${time_re}${time_re}[[:space:]]+1[[:space:]]+.*sys_.*nanosleep" "${output}" echo "perf ftrace profile test [Success]" } diff --git a/tools/perf/tests/shell/inject_aslr.sh b/tools/perf/tests/shell/inject_aslr.sh new file mode 100755 index 000000000000..d83c2591db8f --- /dev/null +++ b/tools/perf/tests/shell/inject_aslr.sh @@ -0,0 +1,539 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# perf inject --aslr test + +set -e +set -o pipefail + +shelldir=$(dirname "$0") +# shellcheck source=lib/perf_has_symbol.sh +. "${shelldir}"/lib/perf_has_symbol.sh + +sym="noploop" + +skip_test_missing_symbol ${sym} + +# Create global temp directory +temp_dir=$(mktemp -d /tmp/perf-test-aslr.XXXXXXXXXX) + +prog="perf test -w noploop" +[ "$(uname -m)" = "s390x" ] && prog="$prog 3" +err=0 +kprog="dd if=/dev/urandom of=/dev/null bs=1M count=50" + +cleanup() { + local exit_code=${1:-$?} + trap - EXIT TERM INT + if [ "${exit_code}" -ne 0 ] || [ "${err}" -ne 0 ]; then + echo "Test failed! Preserving temp directory: ${temp_dir}" + return + fi + # Check if temp_dir is set and looks sane before removing + if [[ "${temp_dir}" =~ ^/tmp/perf-test-aslr\. ]]; then + rm -rf "${temp_dir}" + fi +} + +trap_cleanup() { + local exit_code=$? + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup ${exit_code} + exit ${exit_code} +} +trap trap_cleanup EXIT TERM INT + +get_noploop_addr() { + local file=$1 + perf script -i "$file" | awk ' + BEGIN { found=0 } + { + for (i=1; i<=NF; i++) { + if ($i ~ /noploop\+/) { + if (!found) { + print $(i-1) + found=1 + } + } + } + }' +} + +test_basic_aslr() { + echo "Test basic ASLR remapping" + local data + data=$(mktemp "${temp_dir}/perf.data.basic.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.basic.XXXXXX") + + perf record -e task-clock:u -o "${data}" ${prog} + perf inject -v --aslr -i "${data}" -o "${data2}" + + orig_addr=$(get_noploop_addr "${data}") + new_addr=$(get_noploop_addr "${data2}") + + echo "Basic ASLR: orig_addr=$orig_addr, new_addr=$new_addr" + + if [ -z "$orig_addr" ]; then + echo "Basic ASLR test [Failed - no noploop samples in original file]" + err=1 + elif [ -z "$new_addr" ]; then + echo "Basic ASLR test [Failed - could not find remapped address]" + err=1 + elif [ "$orig_addr" = "$new_addr" ]; then + echo "Basic ASLR test [Failed - addresses are not remapped]" + err=1 + else + echo "Basic ASLR test [Success]" + fi +} + +test_pipe_aslr() { + echo "Test pipe mode ASLR remapping" + local data + data=$(mktemp "${temp_dir}/perf.data.pipe.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.pipe.XXXXXX") + + # Use tee to save the original pipe data for comparison + perf record -e task-clock:u -o - ${prog} | tee "${data}" | perf inject --aslr -o "${data2}" + + orig_addr=$(get_noploop_addr "${data}") + new_addr=$(get_noploop_addr "${data2}") + + echo "Pipe ASLR: orig_addr=$orig_addr, new_addr=$new_addr" + + if [ -z "$orig_addr" ]; then + echo "Pipe ASLR test [Failed - no noploop samples in original file]" + err=1 + elif [ -z "$new_addr" ]; then + echo "Pipe ASLR test [Failed - could not find remapped address]" + err=1 + elif [ "$orig_addr" = "$new_addr" ]; then + echo "Pipe ASLR test [Failed - addresses are not remapped]" + err=1 + else + echo "Pipe ASLR test [Success]" + fi +} + +test_callchain_aslr() { + echo "Test Callchain ASLR remapping" + local data + data=$(mktemp "${temp_dir}/perf.data.callchain.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.callchain.XXXXXX") + + perf record -g -e task-clock:u -o "${data}" ${prog} + perf inject --aslr -i "${data}" -o "${data2}" + + orig_addr=$(get_noploop_addr "${data}") + new_addr=$(get_noploop_addr "${data2}") + + echo "Callchain ASLR: orig_addr=$orig_addr, new_addr=$new_addr" + + if [ -z "$orig_addr" ]; then + echo "Callchain ASLR test [Failed - no noploop samples in original file]" + err=1 + elif [ -z "$new_addr" ]; then + if perf evlist -v -i "${data}" | grep -q 'sample_type:.*STACK_USER'; then + echo "Dropping stack user sample as possible ASLR leak" + echo "Call-graph dwarf not supported with 'perf inject --aslr'" + echo "Callchain ASLR test [Skip]" + else + echo "Callchain ASLR test [Failed - could not find remapped address]" + err=1 + fi + elif [ "$orig_addr" = "$new_addr" ]; then + echo "Callchain ASLR test [Failed - addresses are not remapped]" + err=1 + else + # Extract callchain addresses (indented lines starting with hex addresses) + orig_callchain=$(perf script -i "${data}" | awk '/^[[:space:]]+[0-9a-f]+/ {print $1}') + new_callchain=$(perf script -i "${data2}" | awk '/^[[:space:]]+[0-9a-f]+/ {print $1}') + + if [ -z "$orig_callchain" ]; then + echo "Callchain ASLR test [Failed - no callchain samples in original file]" + err=1 + elif [ -z "$new_callchain" ]; then + echo "Callchain ASLR test [Failed - callchain data was dropped]" + err=1 + elif [ "$orig_callchain" = "$new_callchain" ]; then + echo "Callchain ASLR test [Failed - callchain addresses were not remapped]" + err=1 + else + echo "Callchain ASLR test [Success]" + fi + fi +} + +test_report_aslr() { + echo "Test perf report consistency" + local data + data=$(mktemp "${temp_dir}/perf.data.report.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.report.XXXXXX") + local data_clean + data_clean=$(mktemp "${temp_dir}/perf.data.clean.XXXXXX") + + perf record -e task-clock:u -o "${data}" ${prog} + # Use -b to inject build-ids and force ordered events processing in both + perf inject -b -i "${data}" -o "${data_clean}" + perf inject -v -b --aslr -i "${data}" -o "${data2}" + + local report1="${temp_dir}/report1_basic" + local report2="${temp_dir}/report2_basic" + local report1_clean="${temp_dir}/report1_basic.clean" + local report2_clean="${temp_dir}/report2_basic.clean" + local diff_file="${temp_dir}/diff_basic" + + perf report -i "${data_clean}" --stdio > "${report1}" + perf report -i "${data2}" --stdio > "${report2}" + + # Strip headers and compare lines with percentages + grep '%' "${report1}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report1_clean}" || true + grep '%' "${report2}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report2_clean}" || true + + diff -u -w "${report1_clean}" "${report2_clean}" > "${diff_file}" || true + + if [ ! -s "${report1_clean}" ]; then + echo "Report ASLR test [Failed - no samples captured]" + err=1 + elif [ -s "${diff_file}" ]; then + echo "Report ASLR test [Failed - reports differ]" + echo "Showing first 20 lines of diff:" + head -n 20 "${diff_file}" + err=1 + else + echo "Report ASLR test [Success]" + fi +} + +test_pipe_report_aslr() { + echo "Test pipe mode perf report consistency" + local data + data=$(mktemp "${temp_dir}/perf.data.pipe_report.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.pipe_report.XXXXXX") + local data_clean + data_clean=$(mktemp "${temp_dir}/perf.data.clean.XXXXXX") + + # Use tee to save the original pipe data, then process it with inject -b + perf record -e task-clock:u -o - ${prog} | \ + tee "${data}" | \ + perf inject -b --aslr -o "${data2}" + perf inject -b -i "${data}" -o "${data_clean}" + + local report1="${temp_dir}/report1_pipe" + local report2="${temp_dir}/report2_pipe" + local report1_clean="${temp_dir}/report1_pipe.clean" + local report2_clean="${temp_dir}/report2_pipe.clean" + local diff_file="${temp_dir}/diff_pipe" + + perf report -i "${data_clean}" --stdio > "${report1}" + perf report -i "${data2}" --stdio > "${report2}" + + # Strip headers and compare lines with percentages + grep '%' "${report1}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report1_clean}" || true + grep '%' "${report2}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report2_clean}" || true + + diff -u -w "${report1_clean}" "${report2_clean}" > "${diff_file}" || true + + if [ ! -s "${report1_clean}" ]; then + echo "Pipe Report ASLR test [Failed - no samples captured]" + err=1 + elif [ -s "${diff_file}" ]; then + echo "Pipe Report ASLR test [Failed - reports differ]" + echo "Showing first 20 lines of diff:" + head -n 20 "${diff_file}" + err=1 + else + echo "Pipe Report ASLR test [Success]" + fi +} + +test_pipe_out_report_aslr() { + echo "Test pipe output mode perf report consistency" + local data + data=$(mktemp "${temp_dir}/perf.data.pipe_out_report.XXXXXX") + local data_clean + data_clean=$(mktemp "${temp_dir}/perf.data.clean.XXXXXX") + + perf record -e task-clock:u -o "${data}" ${prog} + perf inject -b -i "${data}" -o "${data_clean}" + + local report1="${temp_dir}/report1_pipe_out" + local report2="${temp_dir}/report2_pipe_out" + local report1_clean="${temp_dir}/report1_pipe_out.clean" + local report2_clean="${temp_dir}/report2_pipe_out.clean" + local diff_file="${temp_dir}/diff_pipe_out" + + perf report -i "${data_clean}" --stdio > "${report1}" + perf inject -b --aslr -i "${data}" -o - | perf report -i - --stdio > "${report2}" + + # Strip headers and compare lines with percentages + grep '%' "${report1}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report1_clean}" || true + grep '%' "${report2}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | sort > "${report2_clean}" || true + + diff -u -w "${report1_clean}" "${report2_clean}" > "${diff_file}" || true + + if [ ! -s "${report1_clean}" ]; then + echo "Pipe Output Report ASLR test [Failed - no samples captured]" + err=1 + elif [ -s "${diff_file}" ]; then + echo "Pipe Output Report ASLR test [Failed - reports differ]" + echo "Showing first 20 lines of diff:" + head -n 20 "${diff_file}" + err=1 + else + echo "Pipe Output Report ASLR test [Success]" + fi +} + +test_dropped_samples() { + echo "Test dropped samples (phys-data)" + local data + data=$(mktemp "${temp_dir}/perf.data.dropped.XXXXXX") + local data2 + data2=$(mktemp "${temp_dir}/perf.data2.dropped.XXXXXX") + + # Check if --phys-data is supported by recording a short run + if ! perf record -e task-clock:u --phys-data -o "${data}" -- sleep 0.1 > /dev/null 2>&1; then + echo "Skipping dropped samples test as --phys-data is not supported" + return + fi + + perf record -e task-clock:u --phys-data -o "${data}" ${prog} + perf inject --aslr -i "${data}" -o "${data2}" + + # Verify that the original file actually contained samples! + orig_samples=$(perf script -i "${data}" | wc -l) + if [ "$orig_samples" -eq 0 ]; then + echo "Dropped samples test [Failed - no samples in original file]" + err=1 + else + # Verify that samples are dropped. + samples_count=$(perf script -i "${data2}" | wc -l) + + if [ "$samples_count" -gt 0 ]; then + echo "Dropped samples test [Failed - samples were not dropped]" + err=1 + else + echo "Dropped samples test [Success]" + fi + fi +} + +test_kernel_aslr() { + echo "Test kernel ASLR remapping" + local kdata + kdata=$(mktemp "${temp_dir}/perf.data.kernel.XXXXXX") + local kdata2 + kdata2=$(mktemp "${temp_dir}/perf.data2.kernel.XXXXXX") + local log_file + log_file=$(mktemp "${temp_dir}/kernel_record.log.XXXXXX") + + # Try to record kernel samples + if ! perf record -e task-clock:k -o "${kdata}" ${kprog} > "${log_file}" 2>&1; then + echo "Skipping kernel ASLR test as recording failed (maybe no permissions)" + return + fi + + # Check for warning about kernel map restriction + if grep -q "Couldn't record kernel reference relocation symbol" "${log_file}"; then + echo "Skipping kernel ASLR test as kernel map could not be recorded (permissions restricted)" + return + fi + + perf inject -v --aslr -i "${kdata}" -o "${kdata2}" + + # Check if kernel addresses are remapped. + # Find the field that ends with :k: (the event name) and take the next field! + orig_addr=$(perf script -i "${kdata}" | awk ' + BEGIN { found=0 } + { + for (i=1; i "${log_file}" 2>&1; then + echo "Skipping kernel report test as recording failed (maybe no permissions)" + return + fi + + # Check for warning about kernel map restriction + if grep -q "Couldn't record kernel reference relocation symbol" "${log_file}"; then + echo "Skipping kernel report test as kernel map could not be recorded (permissions restricted)" + return + fi + + # Use -b to inject build-ids and force ordered events processing in both + perf inject -b -i "${kdata}" -o "${data_clean}" + perf inject -v -b --aslr -i "${kdata}" -o "${kdata2}" + + local report1="${temp_dir}/report_kernel1" + local report2="${temp_dir}/report_kernel2" + local report1_clean="${temp_dir}/report_kernel1.clean" + local report2_clean="${temp_dir}/report_kernel2.clean" + + perf report -i "${data_clean}" --stdio > "${report1}" + perf report -i "${kdata2}" --stdio > "${report2}" + + # Strip headers and compare lines with percentages + grep '%' "${report1}" | grep -v '^#' > "${report1_clean}" || true + grep '%' "${report2}" | grep -v '^#' > "${report2_clean}" || true + + # Normalize kernel DSOs and addresses in clean reports + # This allows kernel modules to be either a module or kernel.kallsyms + local report1_norm="${temp_dir}/report_kernel1.norm" + local report2_norm="${temp_dir}/report_kernel2.norm" + local diff_file="${temp_dir}/diff_kernel" + + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' "${report1_clean}" | \ + awk '{gsub(/\[[a-zA-Z0-9_.-]{2,}\](\.[a-zA-Z0-9_]+)?/, "[kernel]", $0); print}' | \ + sort > "${report1_norm}" || true + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' "${report2_clean}" | \ + awk '{gsub(/\[[a-zA-Z0-9_.-]{2,}\](\.[a-zA-Z0-9_]+)?/, "[kernel]", $0); print}' | \ + sort > "${report2_norm}" || true + + diff -u -w "${report1_norm}" "${report2_norm}" > "${diff_file}" || true + + if [ ! -s "${report1_norm}" ]; then + echo "Kernel Report ASLR test [Failed - no samples captured]" + err=1 + elif [ -s "${diff_file}" ]; then + echo "Kernel Report ASLR test [Failed - reports differ]" + echo "Showing first 20 lines of diff:" + head -n 20 "${diff_file}" + err=1 + else + echo "Kernel Report ASLR test [Success]" + fi +} + +test_regs_stripping() { + echo "Test user register stripping" + local rdata="${temp_dir}/perf.data.regs" + local rdata2="${temp_dir}/perf.data.regs.injected" + local rdata_clean="${temp_dir}/perf.data.regs.clean" + + if ! perf record -e cycles:u --user-regs -o "${rdata}" ${prog} > /dev/null 2>&1; then + echo "Skipping user registers test as recording failed (unsupported flag/platform)" + return + fi + + perf inject -b -i "${rdata}" -o "${rdata_clean}" + perf inject -v -b --aslr -i "${rdata}" -o "${rdata2}" + + local report1="${temp_dir}/report_regs1" + local report2="${temp_dir}/report_regs2" + local report1_clean="${temp_dir}/report_regs1.clean" + local report2_clean="${temp_dir}/report_regs2.clean" + local diff_file="${temp_dir}/diff_regs" + + perf report -i "${rdata_clean}" --stdio > "${report1}" 2>/dev/null || true + perf report -i "${rdata2}" --stdio > "${report2}" 2>/dev/null || true + + grep '%' "${report1}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | \ + sort > "${report1_clean}" || true + grep '%' "${report2}" | grep -v '^#' | \ + grep -v -E '0x[0-9a-f]{8,}|0000000000000000' | \ + sort > "${report2_clean}" || true + + diff -u -w "${report1_clean}" "${report2_clean}" > "${diff_file}" || true + + if [ ! -s "${report1_clean}" ]; then + echo "User registers stripping test [Failed - profile trace starved/empty]" + err=1 + return + elif [ -s "${diff_file}" ]; then + echo "User registers stripping test [Failed - report parsing differs]" + echo "Showing first 20 lines of diff:" + head -n 20 "${diff_file}" + err=1 + return + fi + + local script_dump="${temp_dir}/script_regs_dump" + perf script -D -i "${rdata2}" > "${script_dump}" 2>/dev/null || true + if grep -q "user regs:" "${script_dump}"; then + echo "User registers stripping test [Failed - register dumps still present]" + err=1 + else + echo "User registers stripping test [Success]" + fi +} + +test_basic_aslr +test_pipe_aslr +test_callchain_aslr +test_report_aslr +test_pipe_report_aslr +test_pipe_out_report_aslr +test_dropped_samples +case "$(uname -m)" in + aarch64*|arm*) + echo "Skipping kernel ASLR tests on ARM" + ;; + *) + test_kernel_aslr + test_kernel_report_aslr + ;; +esac + +test_regs_stripping + +cleanup ${err} +exit $err diff --git a/tools/perf/tests/shell/jitdump-python.sh b/tools/perf/tests/shell/jitdump-python.sh index ae86203b14a2..05aaa3bd900b 100755 --- a/tools/perf/tests/shell/jitdump-python.sh +++ b/tools/perf/tests/shell/jitdump-python.sh @@ -16,11 +16,15 @@ if [ "${HAS_PERF_JIT}" != "True" ]; then exit 2 fi -PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXXX) +PERF_DATA_DIR=$(mktemp -d /tmp/__perf_test.perf.data.dir.XXXXXX) +PERF_DATA="${PERF_DATA_DIR}/perf.data" cleanup() { echo "Cleaning up files..." - rm -f ${PERF_DATA} ${PERF_DATA}.jit /tmp/jit-${PID}.dump /tmp/jitted-${PID}-*.so 2> /dev/null + rm -rf ${PERF_DATA_DIR} 2> /dev/null + for p in ${ALL_PIDS}; do + rm -f /tmp/jit-${p}.dump /tmp/jitted-${p}-*.so 2> /dev/null + done trap - EXIT TERM INT } @@ -33,9 +37,16 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT -echo "Run python with -Xperf_jit" -cat <') - -echo "Found ${NUM} matching lines" - -echo "Remove JIT-ed DSOs from the build-ID cache" -for F in /tmp/jitted-${PID}-*.so; do - perf buildid-cache -r "${F}" + if [ -f "${PERF_DATA}.pid" ]; then + REAL_PID=$(cat "${PERF_DATA}.pid") + ALL_PIDS="${ALL_PIDS} ${REAL_PID}" + fi + + # extract PID of the target process from the data + PID=$(perf report -i "${PERF_DATA}" --stdio -F pid -q -g none | \ + cut -d: -f1 -s | sort -u | head -n 1 | tr -d ' ') + if [ -z "${PID}" ]; then + echo "Failed to get PID, retrying..." + continue + fi + ALL_PIDS="${ALL_PIDS} ${PID}" + + echo "Generate JIT-ed DSOs using perf inject" + DEBUGINFOD_URLS='' perf inject -i "${PERF_DATA}" -j -o "${PERF_DATA}.jit" + + echo "Add JIT-ed DSOs to the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -a "${F}" + done + + echo "Check the symbol containing the function/module name" + NUM=$(perf report -i "${PERF_DATA}.jit" -s sym --stdio | grep -cE 'py::(foo|bar|baz):') + + echo "Remove JIT-ed DSOs from the build-ID cache" + for F in /tmp/jitted-${PID}-*.so; do + perf buildid-cache -r "${F}" + done + rm -f /tmp/jitted-${PID}-*.so /tmp/jit-${PID}.dump 2>/dev/null + + if [ "${NUM}" -gt 0 ]; then + echo "Success: found ${NUM} matching lines" + break + fi + echo "No matching lines found, retrying with more iterations..." done cleanup diff --git a/tools/perf/tests/shell/kvm.sh b/tools/perf/tests/shell/kvm.sh index f88e859025c4..7b26368051ff 100755 --- a/tools/perf/tests/shell/kvm.sh +++ b/tools/perf/tests/shell/kvm.sh @@ -39,17 +39,28 @@ skip() { test_kvm_stat() { echo "Testing perf kvm stat" - echo "Recording kvm events for pid ${qemu_pid}..." - if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" sleep 1; then - echo "Failed to record kvm events" - err=1 - return - fi + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm events for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" \ + sleep ${duration} >/dev/null 2>&1; then + echo "perf kvm stat record failed, retrying..." + continue + fi - echo "Reporting kvm events..." - if ! perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + success=true + break + fi + echo "No VM-EXIT events found, retrying..." + done + + if [ "$success" = false ]; then echo "Failed to find VM-EXIT in report" - perf kvm -i "${perfdata}" stat report 2>&1 + perf kvm -i "${perfdata}" stat report 2>&1 || true err=1 return fi @@ -60,22 +71,26 @@ test_kvm_stat() { test_kvm_record_report() { echo "Testing perf kvm record/report" - echo "Recording kvm profile for pid ${qemu_pid}..." - # Use --host to avoid needing guest symbols/mounts for this simple test - # We just want to verify the command runs and produces data - # We run in background and kill it because 'perf kvm record' appends options - # after the command, which breaks 'sleep' (e.g. it gets '-e cycles'). - perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" & - rec_pid=$! - sleep 1 - kill -INT "${rec_pid}" - wait "${rec_pid}" || true + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm profile for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old - echo "Reporting kvm profile..." - # Check for some standard output from report - if ! perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" \ + -e cpu-clock sleep ${duration} + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + success=true + break + fi + echo "No samples or report failed, retrying..." + done + + if [ "$success" = false ]; then echo "Failed to report kvm profile" - perf kvm -i "${perfdata}" report --stdio 2>&1 + perf kvm -i "${perfdata}" report --stdio 2>&1 || true err=1 return fi @@ -123,6 +138,15 @@ test_kvm_stat_live() { echo "perf kvm stat live test [Success]" } +test_kvm_default_event() { + echo "Testing perf kvm record default event with command line" + + # Check if kvm record with default events handle command line arguments + perf kvm record -p "${qemu_pid}" -o /dev/null sleep 1 + + echo "perf kvm record default event [Success]" +} + setup_qemu() { # Find qemu if [ "$(uname -m)" = "x86_64" ]; then @@ -176,6 +200,7 @@ if [ $err -eq 0 ]; then test_kvm_record_report test_kvm_buildid_list test_kvm_stat_live + test_kvm_default_event fi cleanup diff --git a/tools/perf/tests/shell/lib/coresight.sh b/tools/perf/tests/shell/lib/coresight.sh deleted file mode 100644 index 184d62e7e5bd..000000000000 --- a/tools/perf/tests/shell/lib/coresight.sh +++ /dev/null @@ -1,134 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Carsten Haitzler , 2021 - -# This is sourced from a driver script so no need for #!/bin... etc. at the -# top - the assumption below is that it runs as part of sourcing after the -# test sets up some basic env vars to say what it is. - -# This currently works with ETMv4 / ETF not any other packet types at thi -# point. This will need changes if that changes. - -# perf record options for the perf tests to use -PERFRECMEM="-m ,16M" -PERFRECOPT="$PERFRECMEM -e cs_etm//u" - -TOOLS=$(dirname $0) -DIR="$TOOLS/$TEST" -BIN="$DIR/$TEST" -# If the test tool/binary does not exist and is executable then skip the test -if ! test -x "$BIN"; then exit 2; fi -# If CoreSight is not available, skip the test -perf list pmu | grep -q cs_etm || exit 2 -DATD="." -# If the data dir env is set then make the data dir use that instead of ./ -if test -n "$PERF_TEST_CORESIGHT_DATADIR"; then - DATD="$PERF_TEST_CORESIGHT_DATADIR"; -fi -# If the stat dir env is set then make the data dir use that instead of ./ -STATD="." -if test -n "$PERF_TEST_CORESIGHT_STATDIR"; then - STATD="$PERF_TEST_CORESIGHT_STATDIR"; -fi - -# Called if the test fails - error code 1 -err() { - echo "$1" - exit 1 -} - -# Check that some statistics from our perf -check_val_min() { - STATF="$4" - if test "$2" -lt "$3"; then - echo ", FAILED" >> "$STATF" - err "Sanity check number of $1 is too low ($2 < $3)" - fi -} - -perf_dump_aux_verify() { - # Some basic checking that the AUX chunk contains some sensible data - # to see that we are recording something and at least a minimum - # amount of it. We should almost always see Fn packets in just about - # anything but certainly we will see some trace info and async - # packets - DUMP="$DATD/perf-tmp-aux-dump.txt" - perf report --stdio --dump -i "$1" | \ - grep -o -e I_ATOM_F -e I_ASYNC -e I_TRACE_INFO > "$DUMP" - # Simply count how many of these packets we find to see that we are - # producing a reasonable amount of data - exact checks are not sane - # as this is a lossy process where we may lose some blocks and the - # compiler may produce different code depending on the compiler and - # optimization options, so this is rough just to see if we're - # either missing almost all the data or all of it - ATOM_FX_NUM=$(grep -c I_ATOM_F "$DUMP") - ASYNC_NUM=$(grep -c I_ASYNC "$DUMP") - TRACE_INFO_NUM=$(grep -c I_TRACE_INFO "$DUMP") - rm -f "$DUMP" - - # Arguments provide minimums for a pass - CHECK_FX_MIN="$2" - CHECK_ASYNC_MIN="$3" - CHECK_TRACE_INFO_MIN="$4" - - # Write out statistics, so over time you can track results to see if - # there is a pattern - for example we have less "noisy" results that - # produce more consistent amounts of data each run, to see if over - # time any techinques to minimize data loss are having an effect or - # not - STATF="$STATD/stats-$TEST-$DATV.csv" - if ! test -f "$STATF"; then - echo "ATOM Fx Count, Minimum, ASYNC Count, Minimum, TRACE INFO Count, Minimum" > "$STATF" - fi - echo -n "$ATOM_FX_NUM, $CHECK_FX_MIN, $ASYNC_NUM, $CHECK_ASYNC_MIN, $TRACE_INFO_NUM, $CHECK_TRACE_INFO_MIN" >> "$STATF" - - # Actually check to see if we passed or failed. - check_val_min "ATOM_FX" "$ATOM_FX_NUM" "$CHECK_FX_MIN" "$STATF" - check_val_min "ASYNC" "$ASYNC_NUM" "$CHECK_ASYNC_MIN" "$STATF" - check_val_min "TRACE_INFO" "$TRACE_INFO_NUM" "$CHECK_TRACE_INFO_MIN" "$STATF" - echo ", Ok" >> "$STATF" -} - -perf_dump_aux_tid_verify() { - # Specifically crafted test will produce a list of Tread ID's to - # stdout that need to be checked to see that they have had trace - # info collected in AUX blocks in the perf data. This will go - # through all the TID's that are listed as CID=0xabcdef and see - # that all the Thread IDs the test tool reports are in the perf - # data AUX chunks - - # The TID test tools will print a TID per stdout line that are being - # tested - TIDS=$(cat "$2") - # Scan the perf report to find the TIDs that are actually CID in hex - # and build a list of the ones found - FOUND_TIDS=$(perf report --stdio --dump -i "$1" | \ - grep -o "CID=0x[0-9a-z]\+" | sed 's/CID=//g' | \ - uniq | sort | uniq) - # No CID=xxx found - maybe your kernel is reporting these as - # VMID=xxx so look there - if test -z "$FOUND_TIDS"; then - FOUND_TIDS=$(perf report --stdio --dump -i "$1" | \ - grep -o "VMID=0x[0-9a-z]\+" | sed 's/VMID=//g' | \ - uniq | sort | uniq) - fi - - # Iterate over the list of TIDs that the test says it has and find - # them in the TIDs found in the perf report - MISSING="" - for TID2 in $TIDS; do - FOUND="" - for TIDHEX in $FOUND_TIDS; do - TID=$(printf "%i" $TIDHEX) - if test "$TID" -eq "$TID2"; then - FOUND="y" - break - fi - done - if test -z "$FOUND"; then - MISSING="$MISSING $TID" - fi - done - if test -n "$MISSING"; then - err "Thread IDs $MISSING not found in perf AUX data" - fi -} diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/perf/tests/shell/lib/perf_metric_validation.py index dea8ef1977bf..3d52f94f22b9 100644 --- a/tools/perf/tests/shell/lib/perf_metric_validation.py +++ b/tools/perf/tests/shell/lib/perf_metric_validation.py @@ -383,10 +383,13 @@ def _run_perf(self, metric, workload: str): wl = workload.split() command.extend(wl) print(" ".join(command)) - cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8') - data = [x+'}' for x in cmd.stderr.split('}\n') if x] - if data[0][0] != '{': - data[0] = data[0][data[0].find('{'):] + cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') + lines = cmd.stderr.splitlines() + cmd.stdout.splitlines() + data = [] + for line in lines: + line = line.strip() + if line.startswith('{') and line.endswith('}'): + data.append(line) return data def collect_perf(self, workload: str): diff --git a/tools/perf/tests/shell/lib/perf_record.sh b/tools/perf/tests/shell/lib/perf_record.sh new file mode 100644 index 000000000000..2b9e11b66dc7 --- /dev/null +++ b/tools/perf/tests/shell/lib/perf_record.sh @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 + +PERF_RECORD_LOGS=() + +perf_record_with_retry() { + local perfdata="$1" + local check_cmd="$2" + local testprog_base="$3" + shift 3 + + local logfile + logfile=$(mktemp /tmp/__perf_record_retry.XXXXXX) + PERF_RECORD_LOGS+=("$logfile") + + # Save the e flag state and disable it + local save_e + if [[ $- == *e* ]]; then + save_e="set -e" + else + save_e="set +e" + fi + set +e + + local duration + local first_run=true + local ret=1 + local cmd_prefix="perf record" + if [ -n "${PERF_RECORD_CMD}" ]; then + cmd_prefix="${PERF_RECORD_CMD}" + fi + + for duration in 0.01 0.1 0.3 1.0 2.0; do + rm -f "${perfdata}".old + ${cmd_prefix} "$@" -o "${perfdata}" ${testprog_base} ${duration} > "$logfile" 2>&1 + local record_exit=$? + + if [ "$first_run" = true ] && [ $record_exit -ne 0 ]; then + ret=2 + break + fi + first_run=false + + if [ -e "${perfdata}" ] && eval "${check_cmd}"; then + ret=0 + break + fi + done + + eval "$save_e" + return $ret +} + +perf_record_cleanup() { + for logfile in "${PERF_RECORD_LOGS[@]}"; do + rm -f "$logfile" + done + PERF_RECORD_LOGS=() +} diff --git a/tools/perf/tests/shell/lib/setup_python.sh b/tools/perf/tests/shell/lib/setup_python.sh index a58e5536f2ed..2173215a0517 100644 --- a/tools/perf/tests/shell/lib/setup_python.sh +++ b/tools/perf/tests/shell/lib/setup_python.sh @@ -14,3 +14,16 @@ then echo Skipping test, python not detected please set environment variable PYTHON. exit 2 fi + +# Set PYTHONPATH to find the in-tree built perf.so first, avoiding system-wide perf.so +if [ -n "$PERF_EXEC_PATH" ] && [ -d "$PERF_EXEC_PATH/python" ]; then + PYTHONPATH_DIR="$PERF_EXEC_PATH/python" +elif [ -d "$(dirname "$0")/../../python" ]; then + PYTHONPATH_DIR="$(dirname "$0")/../../python" +elif [ -d "$(dirname "$0")/../python" ]; then + PYTHONPATH_DIR="$(dirname "$0")/../python" +fi + +if [ -n "$PYTHONPATH_DIR" ]; then + export PYTHONPATH="$PYTHONPATH_DIR${PYTHONPATH:+:$PYTHONPATH}" +fi diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh index 6dd90519f45c..ba598370be73 100755 --- a/tools/perf/tests/shell/lock_contention.sh +++ b/tools/perf/tests/shell/lock_contention.sh @@ -9,6 +9,10 @@ perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) result=$(mktemp /tmp/__perf_test.result.XXXXX) errout=$(mktemp /tmp/__perf_test.errout.XXXXX) +# Workload to generate lock contention. +# Using 1 group (-g 1) keeps runtime low while generating sufficient lock events. +msg_workload="perf bench sched messaging -g 1" + cleanup() { rm -f ${perfdata} rm -f ${result} @@ -50,7 +54,7 @@ check() { test_record() { echo "Testing perf lock record and perf lock contention" - perf lock record -o ${perfdata} -- perf bench sched messaging -p > /dev/null 2>&1 + perf lock record -o ${perfdata} -- ${msg_workload} > /dev/null 2>&1 # the output goes to the stderr and we expect only 1 output (-E 1) perf lock contention -i ${perfdata} -E 1 -q 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then @@ -70,7 +74,7 @@ test_bpf() fi # the perf lock contention output goes to the stderr - perf lock con -a -b -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" err=1 @@ -81,7 +85,7 @@ test_bpf() test_record_concurrent() { echo "Testing perf lock record and perf lock contention at the same time" - perf lock record -o- -- perf bench sched messaging -p 2> ${errout} | \ + perf lock record -o- -- ${msg_workload} 2> ${errout} | \ perf lock contention -i- -E 1 -q 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] Recorded result count is not 1:" "$(cat "${result}" | wc -l)" @@ -107,7 +111,7 @@ test_aggr_task() fi # the perf lock contention output goes to the stderr - perf lock con -a -b -t -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -t -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" err=1 @@ -130,7 +134,7 @@ test_aggr_addr() fi # the perf lock contention output goes to the stderr - perf lock con -a -b -l -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -l -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" err=1 @@ -148,7 +152,7 @@ test_aggr_cgroup() fi # the perf lock contention output goes to the stderr - perf lock con -a -b --lock-cgroup -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b --lock-cgroup -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" err=1 @@ -170,7 +174,7 @@ test_type_filter() return fi - perf lock con -a -b -Y spinlock -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -Y spinlock -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(grep -c -v spinlock "${result}")" != "0" ]; then echo "[Fail] BPF result should not have non-spinlocks:" "$(cat "${result}")" err=1 @@ -202,7 +206,18 @@ test_lock_filter() return fi - perf lock con -a -b -L tasklist_lock -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -L tasklist_lock -q -- ${msg_workload} > /dev/null 2> ${result} + if [ "$(grep -c -v "${test_lock_filter_type}" "${result}")" != "0" ]; then + echo "[Fail] BPF result should not have non-${test_lock_filter_type} locks:" "$(cat "${result}")" + err=1 + exit + fi + + perf lock con -b -L mmap_lock -q -- perf bench mem mmap -t 2 -l 10 > /dev/null 2> ${result} + + # find out the type of mmap_lock + test_lock_filter_type=$(head -1 "${result}" | awk '{ print $8 }' | sed -e 's/:.*//') + if [ "$(grep -c -v "${test_lock_filter_type}" "${result}")" != "0" ]; then echo "[Fail] BPF result should not have non-${test_lock_filter_type} locks:" "$(cat "${result}")" err=1 @@ -230,7 +245,7 @@ test_stack_filter() return fi - perf lock con -a -b -S unix_stream -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -S unix_stream -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result should have a lock from unix_stream:" "$(cat "${result}")" err=1 @@ -258,7 +273,7 @@ test_aggr_task_stack_filter() return fi - perf lock con -a -b -t -S unix_stream -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b -t -S unix_stream -E 1 -q -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result should have a task from unix_stream:" "$(cat "${result}")" err=1 @@ -274,7 +289,8 @@ test_cgroup_filter() return fi - perf lock con -a -b --lock-cgroup -E 1 -F wait_total -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b --lock-cgroup -E 1 -F wait_total -q \ + -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result should have a cgroup result:" "$(cat "${result}")" err=1 @@ -282,7 +298,8 @@ test_cgroup_filter() fi cgroup=$(cat "${result}" | awk '{ print $3 }') - perf lock con -a -b --lock-cgroup -E 1 -G "${cgroup}" -q -- perf bench sched messaging -p > /dev/null 2> ${result} + perf lock con -a -b --lock-cgroup -E 1 -G "${cgroup}" -q \ + -- ${msg_workload} > /dev/null 2> ${result} if [ "$(cat "${result}" | wc -l)" != "1" ]; then echo "[Fail] BPF result should have a result with cgroup filter:" "$(cat "${cgroup}")" err=1 @@ -317,7 +334,7 @@ test_csv_output() fi # the perf lock contention output goes to the stderr - perf lock con -a -b -E 1 -x , --output ${result} -- perf bench sched messaging -p > /dev/null 2>&1 + perf lock con -a -b -E 1 -x , --output ${result} -- ${msg_workload} > /dev/null 2>&1 output=$(grep -v "^#" ${result} | tr -d -c , | wc -c) if [ "${header}" != "${output}" ]; then echo "[Fail] BPF result does not match the number of commas: ${header} != ${output}" diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh index e459aa99a951..ce68d850c983 100755 --- a/tools/perf/tests/shell/pipe_test.sh +++ b/tools/perf/tests/shell/pipe_test.sh @@ -12,8 +12,8 @@ skip_test_missing_symbol ${sym} data=$(mktemp /tmp/perf.data.XXXXXX) data2=$(mktemp /tmp/perf.data2.XXXXXX) -prog="perf test -w noploop" -[ "$(uname -m)" = "s390x" ] && prog="$prog 3" +prog="perf test -w noploop 0.1" +[ "$(uname -m)" = "s390x" ] && prog="perf test -w noploop 3" err=0 set -e diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 7cb81cf3444a..dd90fef2088b 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -1,10 +1,13 @@ #!/bin/bash -# perf record tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record tests set -e shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + + # shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh @@ -39,6 +42,7 @@ cleanup() { rm -f "${perfdata}" rm -f "${perfdata}".old rm -f "${script_output}" + perf_record_cleanup trap - EXIT TERM INT } @@ -50,22 +54,20 @@ trap_cleanup() { } trap trap_cleanup EXIT TERM INT +check_per_thread() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_per_thread() { echo "Basic --per-thread mode test" - if ! perf record -o /dev/null --quiet ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_per_thread" "perf test -w thloop" \ + --per-thread || ret=$? + if [ $ret -eq 2 ]; then echo "Per-thread record [Skipped event not supported]" return - fi - if ! perf record --per-thread -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Per-thread record [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Per-thread record [Failed missing output]" + elif [ $ret -eq 1 ]; then + echo "Per-thread record [Failed record or missing output]" err=1 return fi @@ -96,6 +98,10 @@ test_per_thread() { echo "Basic --per-thread mode test [Success]" } +check_register_capture() { + perf script -F ip,sym,iregs -i "${perfdata}" 2>/dev/null | grep -q "DI:" +} + test_register_capture() { echo "Register capture test" if ! perf list pmu | grep -q 'br_inst_retired.near_call' @@ -108,11 +114,12 @@ test_register_capture() { echo "Register capture test [Skipped missing registers]" return fi - if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call \ - -c 1000 --per-thread ${testprog} 2> /dev/null \ - | perf script -F ip,sym,iregs -i - 2> /dev/null \ - | grep -q "DI:" - then + + local ret=0 + perf_record_with_retry "${perfdata}" "check_register_capture" "perf test -w thloop" \ + --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call -c 1000 --per-thread || ret=$? + + if [ $ret -ne 0 ]; then echo "Register capture test [Failed missing output]" err=1 return @@ -120,65 +127,66 @@ test_register_capture() { echo "Register capture test [Success]" } +check_system_wide() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_system_wide() { echo "Basic --system-wide mode test" - if ! perf record -aB --synth=no -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no || ret=$? + if [ $ret -eq 2 ]; then echo "System-wide record [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "System-wide record [Failed missing output]" err=1 return fi - if ! perf record -aB --synth=no -e cpu-clock,cs --threads=cpu \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "System-wide record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "System-wide record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no -e cpu-clock,cs --threads=cpu || ret=$? + if [ $ret -ne 0 ]; then + echo "System-wide record [Failed record --threads option or missing output]" err=1 return fi echo "Basic --system-wide mode test [Success]" } +check_workload() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_workload() { echo "Basic target workload test" - if ! perf record -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record or missing output]" err=1 return fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed missing output]" - err=1 - return - fi - if ! perf record -e cpu-clock,cs --threads=package \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" \ + -e cpu-clock,cs --threads=package || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record --threads option or missing output]" err=1 return fi echo "Basic target workload test [Success]" } +check_branch_counter() { + perf report -i "${perfdata}" -D -q 2>/dev/null | grep -q "$br_cntr_output" && \ + perf script -i "${perfdata}" -F +brstackinsn,+brcntr 2>/dev/null | \ + grep -q "$br_cntr_script_output" +} + test_branch_counter() { echo "Branch counter test" # Check if the branch counter feature is supported @@ -190,67 +198,60 @@ test_branch_counter() { return fi done - if ! perf record -o "${perfdata}" -e "{branches:p,instructions}" -j any,counter ${testprog} 2> /dev/null - then - echo "Branch counter record test [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -D -q | grep -q "$br_cntr_output" - then - echo "Branch counter report test [Failed missing output]" - err=1 - return - fi - if ! perf script -i "${perfdata}" -F +brstackinsn,+brcntr | grep -q "$br_cntr_script_output" - then - echo " Branch counter script test [Failed missing output]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_branch_counter" "perf test -w thloop" \ + -e "{branches:p,instructions}" -j any,counter || ret=$? + if [ $ret -ne 0 ]; then + echo "Branch counter test [Failed record or missing output]" err=1 return fi echo "Branch counter test [Success]" } +check_cgroup() { + perf report -i "${perfdata}" -D 2>/dev/null | grep -q "CGROUP" && \ + perf script -i "${perfdata}" -F cgroup 2>/dev/null | grep -q -v "unknown" +} + test_cgroup() { echo "Cgroup sampling test" - if ! perf record -aB --synth=cgroup --all-cgroups -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_cgroup" "perf test -w thloop" \ + -aB --synth=cgroup --all-cgroups || ret=$? + if [ $ret -eq 2 ]; then echo "Cgroup sampling [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -D | grep -q "CGROUP" - then + elif [ $ret -eq 1 ]; then echo "Cgroup sampling [Failed missing output]" err=1 return fi - if ! perf script -i "${perfdata}" -F cgroup | grep -q -v "unknown" - then - echo "Cgroup sampling [Failed cannot resolve cgroup names]" - err=1 - return - fi echo "Cgroup sampling test [Success]" } +check_uid() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_uid() { echo "Uid sampling test" - if ! perf record -aB --synth=no --uid "$(id -u)" -o "${perfdata}" ${testprog} \ - > "${script_output}" 2>&1 - then - if grep -q "libbpf.*EPERM" "${script_output}" + local ret=0 + perf_record_with_retry "${perfdata}" "check_uid" "perf test -w thloop" \ + -aB --synth=no --uid "$(id -u)" || ret=$? + if [ $ret -eq 2 ]; then + local logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + if grep -q -E "libbpf.*EPERM|Access to performance monitoring" "$logfile" || \ + grep -q -E "Permission denied|Failure to open any events" "$logfile" then echo "Uid sampling [Skipped permissions]" return else echo "Uid sampling [Failed to record]" err=1 - # cat "${script_output}" return fi - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "Uid sampling [Failed missing output]" err=1 return diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh index 78a02e90ece1..8d51afeb437b 100755 --- a/tools/perf/tests/shell/record_lbr.sh +++ b/tools/perf/tests/shell/record_lbr.sh @@ -1,9 +1,12 @@ #!/bin/bash -# perf record LBR tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record LBR tests set -e +shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } @@ -22,6 +25,7 @@ cleanup() { rm -rf "${perfdata}" rm -rf "${perfdata}".old rm -rf "${perfdata}".txt + perf_record_cleanup trap - EXIT TERM INT } @@ -34,22 +38,28 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT +check_lbr_callgraph() { + perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt 2>&1 +} + lbr_callgraph_test() { test="LBR callgraph" echo "$test" - if ! perf record -e cycles --call-graph lbr -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_callgraph" "perf test -w thloop" \ + -e cycles --call-graph lbr + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" if [ $err -eq 0 ] then err=2 fi return - fi - - if ! perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt - then + elif [ $ret -eq 1 ]; then cat "${perfdata}".txt echo "$test [Failed in perf report]" err=1 @@ -59,6 +69,12 @@ lbr_callgraph_test() { echo "$test [Success]" } +check_lbr_samples() { + local out + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + [ "$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)" -gt 0 ] +} + lbr_test() { local branch_flags=$1 local test="LBR $2 test" @@ -70,25 +86,27 @@ lbr_test() { local r echo "$test" - if ! perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_samples" "perf test -w thloop" \ + -e cycles $branch_flags + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" - perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop || true if [ $err -eq 0 ] then err=2 fi return - fi - - out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') - sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) - if [ $sam_nr -eq 0 ] - then + elif [ $ret -eq 1 ]; then echo "$test [Failed no samples captured]" err=1 return fi + + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) echo "$test: $sam_nr samples" bs_nr=$(echo "$out" | grep -c 'branch stack: nr:' || true) diff --git a/tools/perf/tests/shell/record_offcpu.sh b/tools/perf/tests/shell/record_offcpu.sh index 860a2d6f4b75..ee095df47358 100755 --- a/tools/perf/tests/shell/record_offcpu.sh +++ b/tools/perf/tests/shell/record_offcpu.sh @@ -45,7 +45,7 @@ test_offcpu_priv() { test_offcpu_basic() { echo "Basic off-cpu test" - if ! perf record --off-cpu -e dummy -o ${perfdata} sleep 1 2> /dev/null + if ! perf record --off-cpu -e dummy -o ${perfdata} sleep 0.3 2> /dev/null then echo "Basic off-cpu test [Failed record]" err=1 @@ -57,7 +57,7 @@ test_offcpu_basic() { err=1 return fi - if ! perf report -i ${perfdata} -q --percent-limit=90 | grep -E -q sleep + if ! perf report -i ${perfdata} -q --percent-limit=70 | grep -E -q sleep then echo "Basic off-cpu test [Failed missing output]" err=1 @@ -98,8 +98,8 @@ test_offcpu_child() { test_offcpu_above_thresh() { echo "${test_above_thresh}" - # collect direct off-cpu samples for tasks blocked for more than 999ms - if ! perf record -e dummy --off-cpu --off-cpu-thresh 999 -o ${perfdata} -- sleep 1 2> /dev/null + # collect direct off-cpu samples for tasks blocked for more than 50ms + if ! perf record -e dummy --off-cpu --off-cpu-thresh 50 -o ${perfdata} -- sleep 0.1 2> /dev/null then echo "${test_above_thresh} [Failed record]" err=1 @@ -115,7 +115,7 @@ test_offcpu_above_thresh() { fi # there should only be one direct sample, and its period should be higher than off-cpu-thresh if ! perf script --time "0, ${dummy_timestamp}" -i ${perfdata} -F period | \ - awk '{ if (int($1) > 999000000) exit 0; else exit 1; }' + awk '{ if (int($1) > 50000000) exit 0; else exit 1; }' then echo "${test_above_thresh} [Failed off-cpu time too short]" err=1 @@ -128,8 +128,8 @@ test_offcpu_above_thresh() { test_offcpu_below_thresh() { echo "${test_below_thresh}" - # collect direct off-cpu samples for tasks blocked for more than 1.2s - if ! perf record -e dummy --off-cpu --off-cpu-thresh 1200 -o ${perfdata} -- sleep 1 2> /dev/null + # collect direct off-cpu samples for tasks blocked for more than 500ms + if ! perf record -e dummy --off-cpu --off-cpu-thresh 500 -o ${perfdata} -- sleep 0.1 2> /dev/null then echo "${test_below_thresh} [Failed record]" err=1 diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 4edb04039036..1e17bee026bd 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -483,6 +483,58 @@ test_stat_pid() { wait $pid 2>/dev/null || true } +test_stat_delay() { + echo "stat -D test" + if ! env LC_ALL=C perf stat -D 1000 -e duration_time sleep 2 > "${stat_output}" 2>&1 + then + echo "stat -D test [Failed - command failed]" + cat "${stat_output}" + err=1 + return + fi + + duration=$(grep "duration_time" "${stat_output}" | awk '{print $1}' | tr -d ',') + elapsed=$(grep "seconds time elapsed" "${stat_output}" | awk '{print $1}') + + if [ -z "$duration" ] || [ -z "$elapsed" ] + then + echo "stat -D test [Failed - failed to find duration_time or time elapsed in output]" + cat "${stat_output}" + err=1 + return + fi + + # Compare duration (ns) and elapsed (s) using awk to handle float and allow tolerance. + if ! awk -v d="$duration" -v e="$elapsed" ' + BEGIN { + diff = d - (e * 1e9); + if (diff < 0) diff = -diff; + # Allow 200ms tolerance (200,000,000 ns) for loaded CI machines. + if (diff > 200000000) { + printf "Fail: duration (%d ns) and elapsed (%f s) mismatch (diff %d ns)\n", d, e, diff; + exit 1; + } + # Lower bound check: must be at least 0.5s. + if (d < 500000000) { + printf "Fail: duration (%d ns) is abnormally small\n", d; + exit 1; + } + # Upper bound check: must be strictly less than 1.7s (proving delay was excluded). + if (d > 1700000000) { + printf "Fail: duration (%d ns) is too large (delay might not be excluded)\n", d; + exit 1; + } + exit 0; + }' + then + echo "stat -D test [Failed - validation failed]" + cat "${stat_output}" + err=1 + return + fi + echo "stat -D test [Success]" +} + test_default_stat test_null_stat test_offline_cpu_stat @@ -498,6 +550,7 @@ test_stat_no_aggr test_stat_detailed test_stat_repeat test_stat_pid +test_stat_delay cleanup exit $err diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh index b582d23f28c9..df5fa27b21ae 100755 --- a/tools/perf/tests/shell/stat_all_metrics.sh +++ b/tools/perf/tests/shell/stat_all_metrics.sh @@ -7,43 +7,70 @@ ParanoidAndNotRoot() [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } -test_prog="sleep 0.01" +test_prog="true" system_wide_flag="-a" if ParanoidAndNotRoot 0 then system_wide_flag="" - test_prog="perf test -w noploop" + test_prog="perf test -w noploop 0.01" fi +check_metric() { + local output="$1" + local status="$2" + local metric="$3" + + if [[ $status -ne 0 || ! "$output" =~ ${metric:0:50} ]]; then + return 1 + fi + + if [[ "$output" =~ "" || "$output" =~ "" ]]; then + return 1 + fi + + return 0 +} + skip=0 err=3 for m in $(perf list --raw-dump metrics); do echo "Testing $m" result=$(perf stat -M "$m" $system_wide_flag -- $test_prog 2>&1) result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. + + if check_metric "$result" $result_err "$m"; then if [[ "$err" -ne 1 ]] then err=0 fi continue fi - if [[ "$result" =~ "Cannot resolve IDs for" || "$result" =~ "No supported events found" ]] + + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" || \ + "$result" =~ "in per-thread mode, enable system wide" || \ + "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" || \ + "$result" =~ "FP_ARITH" || \ + "$result" =~ "AMX" || \ + "$result" =~ "PMM" ]] then - if [[ $(perf list --raw-dump $m) == "Default"* ]] - then - echo "[Ignored $m] failed but as a Default metric this can be expected" - echo $result + true + else + result=$(perf stat -M "$m" $system_wide_flag -- perf test -w noploop 0.1 2>&1) + result_err=$? + + if check_metric "$result" $result_err "$m"; then + if [[ "$err" -ne 1 ]] + then + err=0 + fi continue fi - echo "[Failed $m] Metric contains missing events" - echo $result - err=1 # Fail - continue - elif [[ "$result" =~ \ - "Access to performance monitoring and observability operations is limited" ]] + fi + + # If retry also failed, determine if we skip, ignore, or fail + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" ]] then echo "[Skipped $m] Permission failure" echo $result @@ -61,7 +88,9 @@ for m in $(perf list --raw-dump metrics); do skip=1 fi continue - elif [[ "$result" =~ "" ]] + elif [[ "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" ]] then if [[ $(perf list --raw-dump $m) == "Default"* ]] then @@ -105,19 +134,7 @@ for m in $(perf list --raw-dump metrics); do continue fi - # Failed, possibly the workload was too small so retry with something longer. - result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1) - result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. - if [[ "$err" -ne 1 ]] - then - err=0 - fi - continue - fi - echo "[Failed $m] has non-zero error '$result_err' or not printed in:" + echo "[Failed $m] has non-zero error '$result_err' or not printed/counted in:" echo "$result" err=1 done diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh index 35463358b273..11de77ee38ad 100755 --- a/tools/perf/tests/shell/stat_bpf_counters.sh +++ b/tools/perf/tests/shell/stat_bpf_counters.sh @@ -4,21 +4,26 @@ set -e -workload="perf test -w sqrtloop" +# Get the first allowed CPU +CPU=$(taskset -c -p $$ | awk -F': ' '{print $2}' | awk -F'[,-]' '{print $1}') +if [ -z "$CPU" ]; then + CPU=0 +fi +workload=(taskset -c "$CPU" awk 'BEGIN { for (i=0; i<10000000; i++) sum+=i }') -# check whether $2 is within +/- 20% of $1 +# check whether $2 is within +/- 15% of $1 compare_number() { first_num=$1 second_num=$2 - # upper bound is first_num * 120% - upper=$(expr $first_num + $first_num / 5 ) - # lower bound is first_num * 80% - lower=$(expr $first_num - $first_num / 5 ) + # upper bound is first_num * 115% + upper=$(expr $first_num + $first_num / 20 \* 3 ) + # lower bound is first_num * 85% + lower=$(expr $first_num - $first_num / 20 \* 3 ) if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then - echo "The difference between $first_num and $second_num are greater than 20%." + echo "The difference between $first_num and $second_num are greater than 15%." exit 1 fi } @@ -41,11 +46,12 @@ check_counts() test_bpf_counters() { printf "Testing --bpf-counters " - base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | \ + base_instructions=$(perf stat --no-big-num -e instructions:u -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1 | \ + bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions:u \ + -- "${workload[@]}" 2>&1 | \ awk -v i=0 -v c=0 '/instructions/ { \ if ($1 != " 0) printf "%.0f", c; else print "&1) + stat_output=$(perf stat --no-big-num \ + -e instructions/name=base_instructions/u,instructions/name=bpf_instructions/bu \ + -- "${workload[@]}" 2>&1) base_instructions=$(echo "$stat_output"| \ awk -v i=0 -v c=0 '/base_instructions/ { \ if ($1 != " /dev/null 2>&1 + then + log_verbose "Skipping: system-wide monitoring not supported" + exit 2 + fi +} + +# find two cgroups to measure +find_cgroups() +{ + log_verbose "Finding cgroups..." + # try usual systemd slices first + if [ -d /sys/fs/cgroup/system.slice ] && [ -d /sys/fs/cgroup/user.slice ] + then + test_cgroups="system.slice,user.slice" + log_verbose "Found cgroups: ${test_cgroups}" + return + fi + + # try root and self cgroups + find_cgroups_self_cgrp=$(grep perf_event /proc/self/cgroup | cut -d: -f3) + if [ -z "${find_cgroups_self_cgrp}" ] + then + # cgroup v2 doesn't specify perf_event + find_cgroups_self_cgrp=$(grep ^0: /proc/self/cgroup | cut -d: -f3) + fi + + if [ -z "${find_cgroups_self_cgrp}" ] + then + test_cgroups="/" + else + test_cgroups="/,${find_cgroups_self_cgrp}" + fi + log_verbose "Found cgroups: ${test_cgroups}" +} + +# Check if metric is reported for each cgroup +# $1: extra options (e.g. --bpf-counters) +check_metric_reported() +{ + local opts="$1" + local output + + log_verbose "Running check_metric_reported with opts '${opts}'..." + # Run perf stat + if ! output=$(perf stat -a ${opts} \ + --metrics=insn_per_cycle \ + --for-each-cgroup "${test_cgroups}" \ + -x, sleep 0.1 2>&1) + then + echo "FAIL: perf stat failed with exit code $?" + echo "Output: ${output}" + exit 1 + fi + + log_verbose "perf stat output:" + log_verbose "${output}" + + # Split test_cgroups by comma + IFS=',' read -r -a cgrps <<< "${test_cgroups}" + + for cgrp in "${cgrps[@]}"; do + # Find metric lines for this cgroup + # We use exact cgroup match with surrounding commas + local cgrp_lines + cgrp_lines=$(echo "${output}" | grep -F ",${cgrp}," | grep "insn_per_cycle" || true) + + if [ -z "${cgrp_lines}" ] + then + echo "FAIL: No metric lines found for cgroup '${cgrp}'" + exit 1 + fi + + # Parse each metric line + while read -r line; do + [ -z "${line}" ] && continue + + local val1 + val1=$(echo "${line}" | cut -d, -f1) + + local event_name + event_name=$(echo "${line}" | cut -d, -f3) + + local cycles_val="" + local inst_val="" + + if echo "${event_name}" | grep -q -i "cycles" + then + cycles_val="${val1}" + # Find corresponding instructions event + local inst_event_name + inst_event_name="${event_name/cpu-cycles/instructions}" + inst_event_name="${inst_event_name/cycles/instructions}" + + local inst_line + inst_line=$(echo "${output}" | \ + grep -F ",${cgrp}," | \ + grep -F "${inst_event_name}" || true) + inst_val=$(echo "${inst_line}" | cut -d, -f1) + elif echo "${event_name}" | grep -q -i "instructions" + then + inst_val="${val1}" + # Find corresponding cycles event (try cpu-cycles + # first, then cycles) + local cycles_event_name + cycles_event_name="${event_name/instructions/cpu-cycles}" + local cycles_line + cycles_line=$(echo "${output}" | \ + grep -F ",${cgrp}," | \ + grep -F "${cycles_event_name}" || true) + + if [ -z "${cycles_line}" ] + then + # Try "cycles" instead of "cpu-cycles" + cycles_event_name="${event_name/instructions/cycles}" + cycles_line=$(echo "${output}" | \ + grep -F ",${cgrp}," | \ + grep -F "${cycles_event_name}" || true) + fi + cycles_val=$(echo "${cycles_line}" | cut -d, -f1) + fi + + log_verbose "Cgroup '${cgrp}': event '${event_name}' \ +val '${cycles_val}', inst val '${inst_val}'" + + # Only enforce metric check if both cycles and + # instructions have non-zero numeric counts + if is_numeric_and_non_zero "${cycles_val}" && \ + is_numeric_and_non_zero "${inst_val}" + then + log_verbose "Enforcing metric check for cgroup '${cgrp}' \ +event '${event_name}'" + # Check for nan or nested in the metric value (7th field) + # Actually we can just check the whole line for simplicity + if echo "${line}" | grep -q -i -E ",nan,|,nested," + then + echo "FAIL: Invalid metric value (nan/nested) \ +for cgroup '${cgrp}'" + echo "Line: ${line}" + exit 1 + fi + # Check for empty metric value (2 consecutive + # commas before the unit) + if echo "${line}" | grep -q -E ",,[[:space:]]*[^,]*insn_per_cycle" + then + echo "FAIL: Empty metric value for cgroup '${cgrp}'" + echo "Line: ${line}" + exit 1 + fi + else + log_verbose "Skipping metric check for cgroup '${cgrp}' \ +event '${event_name}' (idle or not counted)" + fi + done <<< "${cgrp_lines}" + done + log_verbose "check_metric_reported passed for opts '${opts}'" +} + +check_system_wide +find_cgroups + +# Test 1: Without BPF counters +check_metric_reported "" + +# Test 2: With BPF counters (if supported) +log_verbose "Checking BPF support..." +if perf stat -a --bpf-counters --for-each-cgroup / true > /dev/null 2>&1 +then + log_verbose "BPF supported, running Test 2..." + check_metric_reported "--bpf-counters" +else + log_verbose "BPF not supported, skipping Test 2" +fi + +exit 0 diff --git a/tools/perf/tests/shell/stat_metrics_values.sh b/tools/perf/tests/shell/stat_metrics_values.sh index 30566f0b5427..86c5c7f70933 100755 --- a/tools/perf/tests/shell/stat_metrics_values.sh +++ b/tools/perf/tests/shell/stat_metrics_values.sh @@ -8,10 +8,17 @@ shelldir=$(dirname "$0") grep -q GenuineIntel /proc/cpuinfo || { echo Skipping non-Intel; exit 2; } +# Skip if no permission to record system-wide events +if ! perf stat -a -e instructions sleep 0.01 >/dev/null 2>&1; then + echo "Skipping: no permission to record system-wide events (-a)" + exit 2 +fi + + pythonvalidator=$(dirname $0)/lib/perf_metric_validation.py rulefile=$(dirname $0)/lib/perf_metric_validation_rules.json tmpdir=$(mktemp -d /tmp/__perf_test.program.XXXXX) -workload="perf bench futex hash -r 2 -s" +workload="perf bench futex hash -r 1 -s" # Add -debug, save data file and full rule file echo "Launch python validation script $pythonvalidator" diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index eb5837f82e39..71550e0b37ba 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -110,20 +110,29 @@ test_trap_eret_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ - perf test -w traploop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver - check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" - check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ + perf test -w traploop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver + check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" + check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing trap & eret branches [Passed]" err=$start_err else echo "Testing trap & eret branches [Failed]" + err=1 fi } @@ -135,32 +144,40 @@ test_kernel_branches() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ - perf bench syscall basic --loop 1000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstack | \ - tr ' ' '\n' > $TMPDIR/perf.script - - # Example of branch entries: - # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." - # Source addresses come first in user or kernel code. Next is the target - # address that must be in the kernel. - - # Look for source addresses with top bit set - if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed kernel branches missing]" - err=1 - fi - # Look for no target addresses without top bit set - if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" $TMPDIR/perf.script; then - echo "Testing kernel branch sampling [Failed user branches found]" - err=1 - fi - if [ $err -eq 0 ]; then + local ret=1 + for loops in 1000 10000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstack | \ + tr ' ' '\n' > $TMPDIR/perf.script + + # Example of branch entries: + # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." + # Source addresses come first in user or kernel code. Next is the target + # address that must be in the kernel. + + # Look for source addresses with top bit set + if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then + err=1 + fi + # Look for no target addresses without top bit set + if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" \ + $TMPDIR/perf.script; then + err=1 + fi + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then echo "Testing kernel branch sampling [Passed]" err=$start_err else echo "Testing kernel branch sampling [Failed]" + err=1 fi } @@ -206,20 +223,28 @@ test_syscall() { return fi start_err=$err - err=0 - perf record -o $TMPDIR/perf.data --branch-filter \ - any_call,save_type,u,k -c 10007 -- \ - perf bench syscall basic --loop 8000 > "$TMPDIR/record.txt" 2>&1 - perf script -i $TMPDIR/perf.data --fields brstacksym | \ - tr ' ' '\n' > $TMPDIR/perf.script - - check_branches "getppid[^ ]*/SYSCALL/" + local ret=1 + for loops in 8000 30000 100000; do + err=0 + perf record -o $TMPDIR/perf.data --branch-filter \ + any_call,save_type,u,k -c 10007 -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 + perf script -i $TMPDIR/perf.data --fields brstacksym | \ + tr ' ' '\n' > $TMPDIR/perf.script + + check_branches "getppid[^ ]*/SYSCALL/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done - if [ $err -eq 0 ]; then + if [ $ret -eq 0 ]; then echo "Testing syscalls [Passed]" err=$start_err else echo "Testing syscalls [Failed]" + err=1 fi } set -e diff --git a/tools/perf/tests/shell/test_intel_pt.sh b/tools/perf/tests/shell/test_intel_pt.sh index 8ee761f03c38..26243ff760ec 100755 --- a/tools/perf/tests/shell/test_intel_pt.sh +++ b/tools/perf/tests/shell/test_intel_pt.sh @@ -21,9 +21,7 @@ tmpfile="${temp_dir}/tmp-perf.data" perfdatafile="${temp_dir}/test-perf.data" outfile="${temp_dir}/test-out.txt" errfile="${temp_dir}/test-err.txt" -workload="${temp_dir}/workload" awkscript="${temp_dir}/awkscript" -jitdump_workload="${temp_dir}/jitdump_workload" maxbrstack="${temp_dir}/maxbrstack.py" cleanup() @@ -60,37 +58,6 @@ perf_record_no_bpf() perf record --no-bpf-event "$@" } -have_workload=false -cat << _end_of_file_ | /usr/bin/cc -o "${workload}" -xc - -pthread && have_workload=true -#include -#include - -void work(void) { - struct timespec tm = { - .tv_nsec = 1000000, - }; - int i; - - /* Run for about 30 seconds */ - for (i = 0; i < 30000; i++) - nanosleep(&tm, NULL); -} - -void *threadfunc(void *arg) { - work(); - return NULL; -} - -int main(void) { - pthread_t th; - - pthread_create(&th, NULL, threadfunc, NULL); - work(); - pthread_join(th, NULL); - return 0; -} -_end_of_file_ - can_cpu_wide() { echo "Checking for CPU-wide recording on CPU $1" @@ -145,11 +112,6 @@ test_per_thread() echo "--- Test per-thread ${desc}recording ---" - if ! $have_workload ; then - echo "No workload, so skipping" - return 2 - fi - if [ "${k}" = "k" ] ; then can_kernel || return 2 fi @@ -252,9 +214,9 @@ test_per_thread() } _end_of_file_ - $workload & + perf test -w thloop 30 2 & w1=$! - $workload & + perf test -w thloop 30 2 & w2=$! echo "Workload PIDs are $w1 and $w2" wait_for_threads ${w1} 2 @@ -283,139 +245,14 @@ test_jitdump() { echo "--- Test tracing self-modifying code that uses jitdump ---" - script_path=$(realpath "$0") - script_dir=$(dirname "$script_path") - jitdump_incl_dir="${script_dir}/../../util" - jitdump_h="${jitdump_incl_dir}/jitdump.h" - if ! perf check feature -q libelf ; then echo "SKIP: libelf is needed for jitdump" return 2 fi - if [ ! -e "${jitdump_h}" ] ; then - echo "SKIP: Include file jitdump.h not found" - return 2 - fi - - if [ -z "${have_jitdump_workload}" ] ; then - have_jitdump_workload=false - # Create a workload that uses self-modifying code and generates its own jitdump file - cat <<- "_end_of_file_" | /usr/bin/cc -o "${jitdump_workload}" -I "${jitdump_incl_dir}" -xc - -pthread && have_jitdump_workload=true - #define _GNU_SOURCE - #include - #include - #include - #include - #include - #include - #include - - #include "jitdump.h" - - #define CHK_BYTE 0x5a - - static inline uint64_t rdtsc(void) - { - unsigned int low, high; - - asm volatile("rdtsc" : "=a" (low), "=d" (high)); - - return low | ((uint64_t)high) << 32; - } - - static FILE *open_jitdump(void) - { - struct jitheader header = { - .magic = JITHEADER_MAGIC, - .version = JITHEADER_VERSION, - .total_size = sizeof(header), - .pid = getpid(), - .timestamp = rdtsc(), - .flags = JITDUMP_FLAGS_ARCH_TIMESTAMP, - }; - char filename[256]; - FILE *f; - void *m; - - snprintf(filename, sizeof(filename), "jit-%d.dump", getpid()); - f = fopen(filename, "w+"); - if (!f) - goto err; - /* Create an MMAP event for the jitdump file. That is how perf tool finds it. */ - m = mmap(0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(f), 0); - if (m == MAP_FAILED) - goto err_close; - munmap(m, 4096); - if (fwrite(&header,sizeof(header),1,f) != 1) - goto err_close; - return f; - - err_close: - fclose(f); - err: - return NULL; - } - - static int write_jitdump(FILE *f, void *addr, const uint8_t *dat, size_t sz, uint64_t *idx) - { - struct jr_code_load rec = { - .p.id = JIT_CODE_LOAD, - .p.total_size = sizeof(rec) + sz, - .p.timestamp = rdtsc(), - .pid = getpid(), - .tid = gettid(), - .vma = (unsigned long)addr, - .code_addr = (unsigned long)addr, - .code_size = sz, - .code_index = ++*idx, - }; - - if (fwrite(&rec,sizeof(rec),1,f) != 1 || - fwrite(dat, sz, 1, f) != 1) - return -1; - return 0; - } - - static void close_jitdump(FILE *f) - { - fclose(f); - } - - int main() - { - /* Get a memory page to store executable code */ - void *addr = mmap(0, 4096, PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - /* Code to execute: mov CHK_BYTE, %eax ; ret */ - uint8_t dat[] = {0xb8, CHK_BYTE, 0x00, 0x00, 0x00, 0xc3}; - FILE *f = open_jitdump(); - uint64_t idx = 0; - int ret = 1; - - if (!f) - return 1; - /* Copy executable code to executable memory page */ - memcpy(addr, dat, sizeof(dat)); - /* Record it in the jitdump file */ - if (write_jitdump(f, addr, dat, sizeof(dat), &idx)) - goto out_close; - /* Call it */ - ret = ((int (*)(void))addr)() - CHK_BYTE; - out_close: - close_jitdump(f); - return ret; - } - _end_of_file_ - fi - - if ! $have_jitdump_workload ; then - echo "SKIP: No jitdump workload" - return 2 - fi - # Change to temp_dir so jitdump collateral files go there cd "${temp_dir}" - perf_record_no_bpf -o "${tmpfile}" -e intel_pt//u "${jitdump_workload}" + perf_record_no_bpf -o "${tmpfile}" -e intel_pt//u perf test -w jitdump perf inject -i "${tmpfile}" -o "${perfdatafile}" --jit decode_br_cnt=$(perf script -i "${perfdatafile}" --itrace=b | wc -l) # Note that overflow and lost errors are suppressed for the error count diff --git a/tools/perf/tests/shell/test_perf_data_converter_ctf.sh b/tools/perf/tests/shell/test_perf_data_converter_ctf.sh index 334eebc9945e..bc58a8dabf4c 100755 --- a/tools/perf/tests/shell/test_perf_data_converter_ctf.sh +++ b/tools/perf/tests/shell/test_perf_data_converter_ctf.sh @@ -24,11 +24,11 @@ trap_cleanup() } trap trap_cleanup exit term int -check_babeltrace_support() +check_babeltrace2_support() { - if ! perf check feature libbabeltrace + if ! perf check feature babeltrace2-ctf-writer then - echo "perf not linked with libbabeltrace, skipping test" + echo "perf not linked with babeltrace2-ctf-writer, skipping test" exit 2 fi } @@ -97,7 +97,7 @@ test_ctf_converter_pipe() fi } -check_babeltrace_support +check_babeltrace2_support test_ctf_converter_file test_ctf_converter_pipe diff --git a/tools/perf/tests/shell/test_test_junit_output.sh b/tools/perf/tests/shell/test_test_junit_output.sh new file mode 100755 index 000000000000..5104ac1e1e6d --- /dev/null +++ b/tools/perf/tests/shell/test_test_junit_output.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# perf test junit XML output validation + +set -e + +err=0 + +shelldir=$(dirname "$0") +# shellcheck source=lib/setup_python.sh +. "${shelldir}"/lib/setup_python.sh + +result=$(mktemp /tmp/__perf_test.output.xml.XXXXX) + +cleanup() +{ + rm -f "${result}" + trap - exit term int +} + +trap_cleanup() +{ + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup exit term int + +test_junit_output() +{ + echo "Testing perf test JUnit XML output command" + perf test -v -j"$result" util || true + if [ -s "$result" ] ; then + echo "perf test JUnit XML output command [SUCCESS]" + else + echo "perf test JUnit XML output command [FAILED]" + err=1 + fi +} + +validate_xml_format() +{ + echo "Validating perf test converted JUnit XML file" + if [ -f "$result" ] ; then + if $PYTHON -c \ + "import xml.etree.ElementTree as ET; ET.parse('$result')" \ + >/dev/null 2>&1 ; then + echo "The file contains valid XML format [SUCCESS]" + else + echo "The file does not contain valid XML format [FAILED]" + err=1 + fi + else + echo "File not found [FAILED]" + err=1 + fi +} + +test_junit_output +validate_xml_format + +cleanup +exit ${err} diff --git a/tools/perf/tests/shell/timechart.sh b/tools/perf/tests/shell/timechart.sh index b14b3472c284..40072b32171a 100755 --- a/tools/perf/tests/shell/timechart.sh +++ b/tools/perf/tests/shell/timechart.sh @@ -9,7 +9,7 @@ perfdata=$(mktemp /tmp/__perf_timechart_test.perf.data.XXXXX) output=$(mktemp /tmp/__perf_timechart_test.output.XXXXX.svg) cleanup() { - rm -f "${perfdata}" + rm -f "${perfdata}"* rm -f "${output}" trap - EXIT TERM INT } @@ -22,38 +22,41 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT test_timechart() { - echo "Basic perf timechart test" + NAME=$1 + OPTION=$2 + + echo "perf timechart ${NAME} test" # Try to record timechart data. # perf timechart record uses system-wide recording and specific tracepoints. # If it fails (e.g. permissions, missing tracepoints), skip the test. - if ! perf timechart record -o "${perfdata}" true > /dev/null 2>&1; then - echo "Basic perf timechart test [Skipped: perf timechart record failed (permissions/events?)]" + if ! perf timechart record -o "${perfdata}" ${OPTION} true > /dev/null 2>&1; then + echo "perf timechart ${NAME} test [Skipped: perf timechart record failed (permissions/events?)]" return fi # Generate the timechart if ! perf timechart -i "${perfdata}" -o "${output}" > /dev/null 2>&1; then - echo "Basic perf timechart test [Failed: perf timechart command failed]" + echo "perf timechart ${NAME} test [Failed: perf timechart command failed]" err=1 return fi # Check if output file exists and is not empty if [ ! -s "${output}" ]; then - echo "Basic perf timechart test [Failed: output file is empty or missing]" + echo "perf timechart ${NAME} test [Failed: output file is empty or missing]" err=1 return fi # Check if it looks like an SVG if ! grep -q "svg" "${output}"; then - echo "Basic perf timechart test [Failed: output doesn't look like SVG]" + echo "perf timechart ${NAME} test [Failed: output doesn't look like SVG]" err=1 return fi - echo "Basic perf timechart test [Success]" + echo "perf timechart ${NAME} test [Success]" } if ! perf check feature -q libtraceevent ; then @@ -62,6 +65,9 @@ if ! perf check feature -q libtraceevent ; then exit 2 fi -test_timechart +test_timechart "Basic" "" +test_timechart "IO-only" "-I" +test_timechart "Backtrace" "-g" + cleanup exit $err diff --git a/tools/perf/tests/shell/trace_record_replay.sh b/tools/perf/tests/shell/trace_record_replay.sh index 88d30a03dcec..38fcafcdfb91 100755 --- a/tools/perf/tests/shell/trace_record_replay.sh +++ b/tools/perf/tests/shell/trace_record_replay.sh @@ -6,16 +6,46 @@ # shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh +# shellcheck source=lib/perf_record.sh +. "$(dirname $0)"/lib/perf_record.sh skip_if_no_perf_trace || exit 2 [ "$(id -u)" = 0 ] || exit 2 file=$(mktemp /tmp/temporary_file.XXXXX) +err=0 -perf trace record -o ${file} sleep 1 || exit 1 -if ! perf trace -i ${file} 2>&1 | grep nanosleep; then - echo "Failed: cannot find *nanosleep syscall" +cleanup() { + rm -f ${file} + perf_record_cleanup + trap - EXIT INT TERM +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT INT TERM + +check_nanosleep() { + perf trace -i "${file}" 2>&1 | grep -q nanosleep +} + +PERF_RECORD_CMD="perf trace record" perf_record_with_retry "${file}" "check_nanosleep" "sleep" +err=$? + +if [ $err -ne 0 ]; then + if [ $err -eq 2 ]; then + logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + echo "perf trace record failed. Log output:" + cat "$logfile" + else + echo "Failed: cannot find *nanosleep syscall" + fi + cleanup exit 1 fi -rm -f ${file} +cleanup +exit 0 diff --git a/tools/perf/tests/shell/trace_summary.sh b/tools/perf/tests/shell/trace_summary.sh index 22e2651d5919..b80dea77cec6 100755 --- a/tools/perf/tests/shell/trace_summary.sh +++ b/tools/perf/tests/shell/trace_summary.sh @@ -14,7 +14,7 @@ OUTPUT=$(mktemp /tmp/perf_trace_test.XXXXX) test_perf_trace() { args=$1 - workload="true" + workload="cat /dev/null" search="^\s*(open|read|close).*[0-9]+%$" echo "testing: perf trace ${args} -- ${workload}" diff --git a/tools/perf/tests/sw-clock.c b/tools/perf/tests/sw-clock.c index b6e46975379c..d18185881635 100644 --- a/tools/perf/tests/sw-clock.c +++ b/tools/perf/tests/sw-clock.c @@ -59,7 +59,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evsel = evsel__new(&attr); if (evsel == NULL) { pr_debug("evsel__new\n"); - goto out_delete_evlist; + goto out_put_evlist; } evlist__add(evlist, evsel); @@ -68,10 +68,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); if (evlist__open(evlist)) { const char *knob = "/proc/sys/kernel/perf_event_max_sample_rate"; @@ -80,14 +80,14 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) pr_debug("Couldn't open evlist: %s\nHint: check %s, using %" PRIu64 " in this test.\n", str_error_r(errno, sbuf, sizeof(sbuf)), knob, (u64)attr.sample_freq); - goto out_delete_evlist; + goto out_put_evlist; } - err = evlist__mmap(evlist, 128); + err = evlist__do_mmap(evlist, 128); if (err < 0) { pr_debug("failed to mmap event: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } evlist__enable(evlist); @@ -98,7 +98,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) evlist__disable(evlist); - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -113,7 +113,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) if (err < 0) { pr_debug("Error during parse sample\n"); perf_sample__exit(&sample); - goto out_delete_evlist; + goto out_put_evlist; } total_periods += sample.period; @@ -131,10 +131,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id) err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c index 72a8289e846d..73568c782d72 100644 --- a/tools/perf/tests/switch-tracking.c +++ b/tools/perf/tests/switch-tracking.c @@ -138,10 +138,13 @@ static int process_sample_event(struct evlist *evlist, goto out; } - evsel = evlist__id2evsel(evlist, sample.id); + evsel = sample.evsel; + if (!evsel) + evsel = evlist__id2evsel(evlist, sample.id); + if (evsel == switch_tracking->switch_evsel) { - next_tid = evsel__intval(evsel, &sample, "next_pid"); - prev_tid = evsel__intval(evsel, &sample, "prev_pid"); + next_tid = perf_sample__intval(&sample, "next_pid"); + prev_tid = perf_sample__intval(&sample, "prev_pid"); cpu = sample.cpu; pr_debug3("sched_switch: cpu: %d prev_tid %d next_tid %d\n", cpu, prev_tid, next_tid); @@ -234,6 +237,7 @@ static int add_event(struct evlist *evlist, struct list_head *events, if (evlist__parse_sample(evlist, event, &sample)) { pr_debug("evlist__parse_sample failed\n"); + perf_sample__exit(&sample); return -1; } @@ -279,8 +283,8 @@ static int process_events(struct evlist *evlist, struct mmap *md; int i, ret; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + md = &evlist__mmap(evlist)[i]; if (perf_mmap__read_init(&md->core) < 0) continue; @@ -371,7 +375,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out_err; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* First event */ err = parse_event(evlist, "cpu-clock:u"); @@ -468,7 +472,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out; } - err = evlist__mmap(evlist, UINT_MAX); + err = evlist__do_mmap(evlist, UINT_MAX); if (err) { pr_debug("evlist__mmap failed!\n"); goto out_err; @@ -579,7 +583,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub out: if (evlist) { evlist__disable(evlist); - evlist__delete(evlist); + evlist__put(evlist); } perf_cpu_map__put(cpus); perf_thread_map__put(threads); diff --git a/tools/perf/tests/symbols.c b/tools/perf/tests/symbols.c index f4ffe5804f40..c09e04f36035 100644 --- a/tools/perf/tests/symbols.c +++ b/tools/perf/tests/symbols.c @@ -125,7 +125,7 @@ static int test_dso(struct dso *dso) for (nd = rb_first_cached(dso__symbols(dso)); nd; nd = rb_next(nd)) { struct symbol *sym = rb_entry(nd, struct symbol, rb_node); - if (sym->type != STT_FUNC && sym->type != STT_GNU_IFUNC) + if (symbol__type(sym) != STT_FUNC && symbol__type(sym) != STT_GNU_IFUNC) continue; /* Check for overlapping function symbols */ diff --git a/tools/perf/tests/task-exit.c b/tools/perf/tests/task-exit.c index 4053ff2813bb..95393edbfe36 100644 --- a/tools/perf/tests/task-exit.c +++ b/tools/perf/tests/task-exit.c @@ -74,15 +74,15 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (!cpus || !threads) { err = -ENOMEM; pr_debug("Not enough memory to create thread/cpu maps\n"); - goto out_delete_evlist; + goto out_put_evlist; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); err = evlist__prepare_workload(evlist, &target, argv, false, workload_exec_failed_signal); if (err < 0) { pr_debug("Couldn't run the workload!\n"); - goto out_delete_evlist; + goto out_put_evlist; } evsel = evlist__first(evlist); @@ -101,20 +101,20 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (err < 0) { pr_debug("Couldn't open the evlist: %s\n", str_error_r(-err, sbuf, sizeof(sbuf))); - goto out_delete_evlist; + goto out_put_evlist; } - if (evlist__mmap(evlist, 128) < 0) { + if (evlist__do_mmap(evlist, 128) < 0) { pr_debug("failed to mmap events: %d (%s)\n", errno, str_error_r(errno, sbuf, sizeof(sbuf))); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } evlist__start_workload(evlist); retry: - md = &evlist->mmap[0]; + md = &evlist__mmap(evlist)[0]; if (perf_mmap__read_init(&md->core) < 0) goto out_init; @@ -133,7 +133,7 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ if (retry_count++ > 1000) { pr_debug("Failed after retrying 1000 times\n"); err = -1; - goto out_delete_evlist; + goto out_put_evlist; } goto retry; @@ -144,10 +144,10 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _ err = -1; } -out_delete_evlist: +out_put_evlist: perf_cpu_map__put(cpus); perf_thread_map__put(threads); - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/tests/tests-scripts.c b/tools/perf/tests/tests-scripts.c index f18c4cd337c8..0370bc701373 100644 --- a/tools/perf/tests/tests-scripts.c +++ b/tools/perf/tests/tests-scripts.c @@ -31,6 +31,7 @@ static int shell_tests__dir_fd(void) { struct stat st; char path[PATH_MAX], path2[PATH_MAX], *exec_path; + ssize_t len; static const char * const devel_dirs[] = { "./tools/perf/tests/shell", "./tests/shell", @@ -47,13 +48,17 @@ static int shell_tests__dir_fd(void) } /* Use directory of executable */ - if (readlink("/proc/self/exe", path2, sizeof path2) < 0) + len = readlink("/proc/self/exe", path2, sizeof(path2) - 1); + if (len < 0) return -1; + path2[len] = '\0'; /* Follow another level of symlink if there */ if (lstat(path2, &st) == 0 && (st.st_mode & S_IFMT) == S_IFLNK) { - scnprintf(path, sizeof(path), path2); - if (readlink(path, path2, sizeof path2) < 0) + scnprintf(path, sizeof(path), "%s", path2); + len = readlink(path, path2, sizeof(path2) - 1); + if (len < 0) return -1; + path2[len] = '\0'; } /* Get directory */ p = strrchr(path2, '/'); @@ -78,43 +83,50 @@ static int shell_tests__dir_fd(void) static char *shell_test__description(int dir_fd, const char *name) { struct io io; - char buf[128], desc[256]; - int ch, pos = 0; + char buf[128], *line = NULL; + size_t line_len = 0; + ssize_t len; + char *desc = NULL; + const char *spdx = "SPDX-License"; io__init(&io, openat(dir_fd, name, O_RDONLY), buf, sizeof(buf)); if (io.fd < 0) return NULL; - /* Skip first line - should be #!/bin/bash Shebang */ - if (io__get_char(&io) != '#') - goto err_out; - if (io__get_char(&io) != '!') - goto err_out; - do { - ch = io__get_char(&io); - if (ch < 0) - goto err_out; - } while (ch != '\n'); - - do { - ch = io__get_char(&io); - if (ch < 0) - goto err_out; - } while (ch == '#' || isspace(ch)); - while (ch > 0 && ch != '\n') { - desc[pos++] = ch; - if (pos >= (int)sizeof(desc) - 1) + while ((len = io__getline(&io, &line, &line_len)) > 0) { + char *p = line; + + /* Skip leading whitespace */ + while (*p && isspace(*p)) + p++; + + /* Must be a comment */ + if (*p != '#') + continue; + p++; + + /* Skip shebang or SPDX lines */ + if (*p == '!' || (strstr(p, spdx) && strstr(p, "-Identifier:"))) + continue; + + /* Skip whitespace after # */ + while (*p && isspace(*p)) + p++; + + /* If we found non-empty text, this is the description! */ + if (*p && *p != '\n') { + char *end = p + strlen(p); + + while (end > p && isspace(end[-1])) + end--; + *end = '\0'; + desc = strdup(p); break; - ch = io__get_char(&io); + } } - while (pos > 0 && isspace(desc[--pos])) - ; - desc[++pos] = '\0'; - close(io.fd); - return strdup(desc); -err_out: + free(line); close(io.fd); - return NULL; + return desc; } /* Is this full file path a shell script */ @@ -178,9 +190,9 @@ static void append_script(int dir_fd, const char *name, char *desc, char *exclusive; snprintf(link, sizeof(link), "/proc/%d/fd/%d", getpid(), dir_fd); - len = readlink(link, filename, sizeof(filename)); - if (len < 0) { - pr_err("Failed to readlink %s", link); + len = readlink(link, filename, sizeof(filename) - 1); + if (len < 0 || (size_t)len > sizeof(filename) - strlen(name) - 2) { + pr_err("Failed to readlink %s or path too long", link); return; } filename[len++] = '/'; diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h index f5f1238d1f7f..cee9e6b62dcc 100644 --- a/tools/perf/tests/tests.h +++ b/tools/perf/tests/tests.h @@ -38,12 +38,14 @@ struct test_case { const char *skip_reason; test_fnptr run_case; bool exclusive; + void *priv; }; struct test_suite { const char *desc; struct test_case *test_cases; void *priv; + int (*setup)(struct test_suite *suite); }; #define DECLARE_SUITE(name) \ @@ -177,6 +179,7 @@ DECLARE_SUITE(sigtrap); DECLARE_SUITE(event_groups); DECLARE_SUITE(symbols); DECLARE_SUITE(util); +DECLARE_SUITE(uncore_event_sorting); DECLARE_SUITE(subcmd_help); DECLARE_SUITE(kallsyms_split); @@ -234,6 +237,7 @@ struct test_workload workload__##work = { \ /* The list of test workloads */ DECLARE_WORKLOAD(noploop); DECLARE_WORKLOAD(thloop); +DECLARE_WORKLOAD(named_threads); DECLARE_WORKLOAD(leafloop); DECLARE_WORKLOAD(sqrtloop); DECLARE_WORKLOAD(brstack); @@ -241,6 +245,10 @@ DECLARE_WORKLOAD(datasym); DECLARE_WORKLOAD(landlock); DECLARE_WORKLOAD(traploop); DECLARE_WORKLOAD(inlineloop); +DECLARE_WORKLOAD(jitdump); +DECLARE_WORKLOAD(context_switch_loop); +DECLARE_WORKLOAD(deterministic); +DECLARE_WORKLOAD(callchain); #ifdef HAVE_RUST_SUPPORT DECLARE_WORKLOAD(code_with_type); diff --git a/tools/perf/tests/thread-maps-share.c b/tools/perf/tests/thread-maps-share.c index e9ecd30a5c05..0431bff31b3a 100644 --- a/tools/perf/tests/thread-maps-share.c +++ b/tools/perf/tests/thread-maps-share.c @@ -27,7 +27,7 @@ static int test__thread_maps_share(struct test_suite *test __maybe_unused, int s * other group (pid: 4, tids: 4, 5) */ - machines__init(&machines); + TEST_ASSERT_VAL("failed to init machines", machines__init(&machines) == 0); machine = &machines.host; /* create process with 4 threads */ diff --git a/tools/perf/tests/time-utils-test.c b/tools/perf/tests/time-utils-test.c index 38df10373c1e..90a9a4b4f178 100644 --- a/tools/perf/tests/time-utils-test.c +++ b/tools/perf/tests/time-utils-test.c @@ -69,16 +69,19 @@ struct test_data { static bool test__perf_time__parse_for_ranges(struct test_data *d) { - struct evlist evlist = { - .first_sample_time = d->first, - .last_sample_time = d->last, - }; - struct perf_session session = { .evlist = &evlist }; + struct evlist *evlist = evlist__new(); + struct perf_session session = { .evlist = evlist }; struct perf_time_interval *ptime = NULL; int range_size, range_num; bool pass = false; int i, err; + if (!evlist) { + pr_debug("Missing evlist\n"); + return false; + } + evlist__set_first_sample_time(evlist, d->first); + evlist__set_last_sample_time(evlist, d->last); pr_debug("\nperf_time__parse_for_ranges(\"%s\")\n", d->str); if (strchr(d->str, '%')) @@ -127,6 +130,7 @@ static bool test__perf_time__parse_for_ranges(struct test_data *d) pass = true; out: + evlist__put(evlist); free(ptime); return pass; } diff --git a/tools/perf/tests/tool_pmu.c b/tools/perf/tests/tool_pmu.c index 1e900ef92e37..c6c5ebf0e935 100644 --- a/tools/perf/tests/tool_pmu.c +++ b/tools/perf/tests/tool_pmu.c @@ -40,9 +40,10 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) } ret = TEST_OK; - if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) { + if (with_pmu ? (evlist__nr_entries(evlist) != 1) + : (evlist__nr_entries(evlist) < 1)) { pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n", - __FILE__, __LINE__, str, evlist->core.nr_entries); + __FILE__, __LINE__, str, evlist__nr_entries(evlist)); ret = TEST_FAIL; goto out; } @@ -67,7 +68,7 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu) out: parse_events_error__exit(&err); - evlist__delete(evlist); + evlist__put(evlist); return ret; } diff --git a/tools/perf/tests/topology.c b/tools/perf/tests/topology.c index f54502ebef4b..77cb8318c0b1 100644 --- a/tools/perf/tests/topology.c +++ b/tools/perf/tests/topology.c @@ -11,6 +11,7 @@ #include "pmus.h" #include "target.h" #include +#include "dwarf-regs.h" #define TEMPL "/tmp/perf-test-XXXXXX" #define DATA_SIZE 10 @@ -45,7 +46,7 @@ static int session_write_header(char *path) session->evlist = evlist__new_default(&target, /*sample_callchains=*/false); TEST_ASSERT_VAL("can't get evlist", session->evlist); - session->evlist->session = session; + evlist__set_session(session->evlist, session); perf_header__set_feat(&session->header, HEADER_CPU_TOPOLOGY); perf_header__set_feat(&session->header, HEADER_NRCPUS); @@ -57,7 +58,7 @@ static int session_write_header(char *path) !perf_session__write_header(session, session->evlist, perf_data__fd(&data), true)); - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_session__delete(session); return 0; @@ -74,6 +75,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) struct aggr_cpu_id id; struct perf_cpu cpu; struct perf_env *env; + uint16_t e_machine; session = perf_session__new(&data, NULL); TEST_ASSERT_VAL("can't get session", !IS_ERR(session)); @@ -101,7 +103,9 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) * condition is true (see do_core_id_test in header.c). So always * run this test on those platforms. */ - if (!env->cpu && strncmp(env->arch, "s390", 4) && strncmp(env->arch, "aarch64", 7)) + e_machine = perf_env__e_machine(env, NULL); + + if (!env->cpu && e_machine != EM_S390 && e_machine != EM_AARCH64) return TEST_SKIP; /* @@ -110,7 +114,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) * physical_package_id will be set to -1. Hence skip this * test if physical_package_id returns -1 for cpu from perf_cpu_map. */ - if (!strncmp(env->arch, "ppc64le", 7)) { + if (e_machine == EM_PPC64) { if (cpu__get_socket_id(perf_cpu_map__cpu(map, 0)) == -1) return TEST_SKIP; } diff --git a/tools/perf/tests/uncore-event-sorting.c b/tools/perf/tests/uncore-event-sorting.c new file mode 100644 index 000000000000..7756777c54c2 --- /dev/null +++ b/tools/perf/tests/uncore-event-sorting.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include + +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "pmu.h" +#include "pmus.h" +#include "tests.h" + +struct match_state { + char *event1; + char *event2; +}; + +static char *clean_event_name(struct pmu_event_info *info) +{ + const char *name = info->name; + const char *pmu_name = info->pmu->name; + size_t pmu_len = strlen(pmu_name); + char *res; + size_t len; + + if (!strncmp(name, pmu_name, pmu_len) && name[pmu_len] == '/') + name += pmu_len + 1; + + res = strdup(name); + if (!res) + return NULL; + + len = strlen(res); + if (len > 0 && res[len - 1] == '/') + res[len - 1] = '\0'; + + return res; +} + +static int event_cb(void *state, struct pmu_event_info *info) +{ + struct match_state *m = state; + char *clean_name; + + if (m->event1 && m->event2) + return 1; + + clean_name = clean_event_name(info); + if (!clean_name) + return 0; + + if (!m->event1) { + m->event1 = clean_name; + } else { + if (strcmp(m->event1, clean_name)) { + m->event2 = clean_name; + return 1; + } + free(clean_name); + } + return 0; +} + +#define CHECK_COND(cond, text) \ +do { \ + if (!(cond)) { \ + pr_debug("FAILED %s:%d %s\n", __FILE__, __LINE__, text); \ + ret = TEST_FAIL; \ + goto out_err; \ + } \ +} while (0) + +#define CHECK_EQUAL(val, expected, text) \ +do { \ + if ((val) != (expected)) { \ + pr_debug("FAILED %s:%d %s (%d != %d)\n", \ + __FILE__, __LINE__, text, (val), (expected)); \ + ret = TEST_FAIL; \ + goto out_err; \ + } \ +} while (0) + +static int test__uncore_event_sorting(struct test_suite *test __maybe_unused, + int subtest __maybe_unused) +{ + struct evlist *evlist = NULL; + struct parse_events_error err; + struct evsel *evsel; + struct perf_pmu *pmu = NULL; + char *pmu_prefix = NULL; + struct match_state m = { NULL, NULL }; + char buf[1024]; + int ret; + + parse_events_error__init(&err); + + while ((pmu = perf_pmus__scan(pmu)) != NULL) { + size_t len; + struct perf_pmu *sibling; + + if (pmu->is_core) + continue; + + len = pmu_name_len_no_suffix(pmu->name); + if (len == strlen(pmu->name)) + continue; + + sibling = pmu; + while ((sibling = perf_pmus__scan(sibling)) != NULL) { + if (sibling->is_core) + continue; + if (pmu_name_len_no_suffix(sibling->name) == len && + !strncmp(pmu->name, sibling->name, len)) + break; + } + + if (!sibling) + continue; + + m.event1 = m.event2 = NULL; + perf_pmu__for_each_event(pmu, false, &m, event_cb); + + if (m.event1 && m.event2) { + pmu_prefix = strndup(pmu->name, len); + break; + } + zfree(&m.event1); + } + + if (!pmu_prefix) { + pr_debug("No suitable uncore PMU found\n"); + ret = TEST_SKIP; + goto out_err; + } + + evlist = evlist__new(); + if (!evlist) { + ret = TEST_FAIL; + goto out_err; + } + + snprintf(buf, sizeof(buf), "{%s/%s/,%s/%s/}", pmu_prefix, m.event1, pmu_prefix, m.event2); + pr_debug("Parsing: %s\n", buf); + + ret = parse_events(evlist, buf, &err); + if (ret) { + pr_debug("parse_events failed\n"); + ret = TEST_FAIL; + goto out_err; + } + + CHECK_COND(evlist__nr_entries(evlist) >= 4, "Number of events is >= 4"); + CHECK_EQUAL(evlist__nr_entries(evlist) % 2, 0, "Number of events is a multiple of 2"); + + evlist__for_each_entry(evlist, evsel) { + struct evsel *next; + + if (!evsel__is_group_leader(evsel)) + continue; + + next = evsel__next(evsel); + CHECK_EQUAL(evsel->core.nr_members, 2, "Group size is 2"); + CHECK_COND(evsel->pmu == next->pmu, "PMU match"); + CHECK_COND(strstr(evsel__name(evsel), m.event1) != NULL, "First event name"); + CHECK_COND(strstr(evsel__name(next), m.event2) != NULL, "Second event name"); + } + ret = TEST_OK; + +out_err: + evlist__put(evlist); + parse_events_error__exit(&err); + zfree(&pmu_prefix); + zfree(&m.event1); + zfree(&m.event2); + return ret; +} + +DEFINE_SUITE("Uncore event sorting", uncore_event_sorting); diff --git a/tools/perf/tests/util.c b/tools/perf/tests/util.c index bf2c5b133884..f9abd9911e6c 100644 --- a/tools/perf/tests/util.c +++ b/tools/perf/tests/util.c @@ -86,7 +86,12 @@ static int test_blake2s(void) return 0; } -static int test__util(struct test_suite *t __maybe_unused, int subtest __maybe_unused) +static int test__blake2s_case(struct test_suite *t __maybe_unused, int subtest __maybe_unused) +{ + return test_blake2s(); +} + +static int test__strreplace(struct test_suite *t __maybe_unused, int subtest __maybe_unused) { TEST_ASSERT_VAL("empty string", test_strreplace(' ', "", "123", "")); TEST_ASSERT_VAL("no match", test_strreplace('5', "123", "4", "123")); @@ -95,7 +100,16 @@ static int test__util(struct test_suite *t __maybe_unused, int subtest __maybe_u TEST_ASSERT_VAL("replace long", test_strreplace('a', "abcabc", "longlong", "longlongbclonglongbc")); - return test_blake2s(); + return 0; } -DEFINE_SUITE("util", util); +static struct test_case tests__util[] = { + TEST_CASE("String replacement", strreplace), + TEST_CASE("BLAKE2s hash", blake2s_case), + { .name = NULL, } +}; + +struct test_suite suite__util = { + .desc = "util", + .test_cases = tests__util, +}; diff --git a/tools/perf/tests/vmlinux-kallsyms.c b/tools/perf/tests/vmlinux-kallsyms.c index 524d46478364..9396c8a77c86 100644 --- a/tools/perf/tests/vmlinux-kallsyms.c +++ b/tools/perf/tests/vmlinux-kallsyms.c @@ -192,7 +192,7 @@ static int test__vmlinux_matches_kallsyms(struct test_suite *test __maybe_unused struct rb_node *nd; struct symbol *sym; struct map *kallsyms_map; - struct machine vmlinux; + struct machine vmlinux = { 0 }; struct maps *maps; u64 mem_start, mem_end; struct test__vmlinux_matches_kallsyms_cb_args args; @@ -203,8 +203,10 @@ static int test__vmlinux_matches_kallsyms(struct test_suite *test __maybe_unused * Init the machines that will hold kernel, modules obtained from * both vmlinux + .ko files and from /proc/kallsyms split by modules. */ - machine__init(&args.kallsyms, "", HOST_KERNEL_ID); - machine__init(&vmlinux, "", HOST_KERNEL_ID); + if (machine__init(&args.kallsyms, "", HOST_KERNEL_ID)) + goto out; + if (machine__init(&vmlinux, "", HOST_KERNEL_ID)) + goto out; maps = machine__kernel_maps(&vmlinux); @@ -346,7 +348,7 @@ static int test__vmlinux_matches_kallsyms(struct test_suite *test __maybe_unused * such as __indirect_thunk_end. */ continue; - } else if (is_ignored_symbol(sym->name, sym->type)) { + } else if (is_ignored_symbol(sym->name, symbol__type(sym))) { /* * Ignore hidden symbols, see scripts/kallsyms.c for the details */ diff --git a/tools/perf/tests/workloads/Build b/tools/perf/tests/workloads/Build index 2ef97f7affce..048e371eb63e 100644 --- a/tools/perf/tests/workloads/Build +++ b/tools/perf/tests/workloads/Build @@ -2,6 +2,7 @@ perf-test-y += noploop.o perf-test-y += thloop.o +perf-test-y += named_threads.o perf-test-y += leafloop.o perf-test-y += sqrtloop.o perf-test-y += brstack.o @@ -9,6 +10,10 @@ perf-test-y += datasym.o perf-test-y += landlock.o perf-test-y += traploop.o perf-test-y += inlineloop.o +perf-test-y += jitdump.o +perf-test-y += context_switch_loop.o +perf-test-y += deterministic.o +perf-test-y += callchain.o ifeq ($(CONFIG_RUST_SUPPORT),y) perf-test-y += code_with_type.o @@ -21,3 +26,6 @@ CFLAGS_brstack.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE CFLAGS_datasym.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE CFLAGS_traploop.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE CFLAGS_inlineloop.o = -g -O2 +CFLAGS_deterministic.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE +CFLAGS_named_threads.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE +CFLAGS_callchain.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE diff --git a/tools/perf/tests/workloads/callchain.c b/tools/perf/tests/workloads/callchain.c new file mode 100644 index 000000000000..abbb406ba90b --- /dev/null +++ b/tools/perf/tests/workloads/callchain.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../tests.h" + +/* + * Mark as noinline to establish the call chain, and avoid the static + * annotation to prevent LTO from renaming the functions. + */ +noinline void callchain_do_syscall(void); +noinline void callchain_foo(void); +noinline int callchain(int argc, const char **argv); + +noinline void callchain_do_syscall(void) +{ + syscall(SYS_gettid); +} + +noinline void callchain_foo(void) +{ + callchain_do_syscall(); +} + +noinline int callchain(int argc __maybe_unused, + const char **argv __maybe_unused) +{ + callchain_foo(); + + return 0; +} + +DEFINE_WORKLOAD(callchain); diff --git a/tools/perf/tests/workloads/context_switch_loop.c b/tools/perf/tests/workloads/context_switch_loop.c new file mode 100644 index 000000000000..5431af6147e6 --- /dev/null +++ b/tools/perf/tests/workloads/context_switch_loop.c @@ -0,0 +1,110 @@ + +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include + +#include "../tests.h" + +static int loops = 100; +static char buf; +int context_switch_loop_work = 1234; + +#define write_block(fd) \ + do { \ + if (write(fd, &buf, 1) <= 0) \ + return 1; \ + } while (0) + +#define read_block(fd) \ + do { \ + if (read(fd, &buf, 1) <= 0) \ + return 1; \ + } while (0) + +/* Not static to avoid LTO clobbering the function name */ +int context_switch_loop_proc1(int in_fd, int out_fd); +int context_switch_loop_proc1(int in_fd, int out_fd) +{ + for (int i = 0; i < loops; i++) { + read_block(in_fd); + context_switch_loop_work += i * 3; + write_block(out_fd); + } + return 0; +} + +int context_switch_loop_proc2(int in_fd, int out_fd); +int context_switch_loop_proc2(int in_fd, int out_fd) +{ + for (int i = 0; i < loops; i++) { + write_block(out_fd); + context_switch_loop_work += i * 7; + read_block(in_fd); + } + return 0; +} + +/* + * Launches two processes that take turns to execute a multiplication N times + */ +static int context_switch_loop(int argc, const char **argv) +{ + int a_to_b[2], b_to_a[2]; + pid_t proc1_pid; + int status; + int ret; + + if (argc > 0) { + loops = atoi(argv[0]); + if (loops < 0) { + fprintf(stderr, "Invalid number of loops: %s\n", argv[0]); + return 1; + } + } + + if (pipe(a_to_b) || pipe(b_to_a)) { + perror("Pipe error"); + return 1; + } + + proc1_pid = fork(); + if (proc1_pid < 0) { + perror("Fork error"); + return 1; + } + + if (!proc1_pid) { + close(a_to_b[0]); + close(b_to_a[1]); + prctl(PR_SET_NAME, "proc1", 0, 0, 0); + ret = context_switch_loop_proc1(b_to_a[0], a_to_b[1]); + close(a_to_b[1]); + close(b_to_a[0]); + exit(ret); + } + + close(a_to_b[1]); + close(b_to_a[0]); + prctl(PR_SET_NAME, "proc2", 0, 0, 0); + ret = context_switch_loop_proc2(a_to_b[0], b_to_a[1]); + close(a_to_b[0]); + close(b_to_a[1]); + + if (ret) { + kill(proc1_pid, SIGKILL); + return ret; + } + + if (waitpid(proc1_pid, &status, 0) != proc1_pid || !WIFEXITED(status) || + WEXITSTATUS(status)) + return 1; + + return 0; +} + +DEFINE_WORKLOAD(context_switch_loop); diff --git a/tools/perf/tests/workloads/deterministic.c b/tools/perf/tests/workloads/deterministic.c new file mode 100644 index 000000000000..8a78519fd075 --- /dev/null +++ b/tools/perf/tests/workloads/deterministic.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "../tests.h" + +int dt_work = 1234; + +static void function1(void) +{ + dt_work += 7; + dt_work += 7; + dt_work += 7; +} + +static void function2(void) +{ + dt_work += 7; + dt_work += 7; + dt_work += 7; +} + +static int deterministic(int argc __maybe_unused, + const char **argv __maybe_unused) +{ + dt_work += 7; + dt_work += 7; + dt_work += 7; + + function1(); + + dt_work += 7; + dt_work += 7; + dt_work += 7; + + function2(); + + return 0; +} + +DEFINE_WORKLOAD(deterministic); diff --git a/tools/perf/tests/workloads/jitdump.c b/tools/perf/tests/workloads/jitdump.c new file mode 100644 index 000000000000..01cbbbd564e8 --- /dev/null +++ b/tools/perf/tests/workloads/jitdump.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/jitdump.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../tests.h" + +#ifndef HAVE_GETTID +#include +static inline pid_t gettid(void) +{ + return (pid_t)syscall(SYS_gettid); +} +#endif + +#define CHK_BYTE 0x5a + +static inline uint64_t get_timestamp(void) +{ +#if defined(__x86_64__) || defined(__i386__) + unsigned int low, high; + + asm volatile("rdtsc" : "=a"(low), "=d"(high)); + + return low | ((uint64_t)high) << 32; +#else + struct timespec ts; + int ret; + + ret = clock_gettime(CLOCK_MONOTONIC, &ts); + if (ret) + return 0; + + return ((uint64_t)ts.tv_sec * 1000000000) + ts.tv_nsec; +#endif +} + +static FILE *open_jitdump(void) +{ + struct jitheader header = { + .magic = JITHEADER_MAGIC, + .version = JITHEADER_VERSION, + .total_size = sizeof(header), + .pid = getpid(), + .timestamp = get_timestamp(), + .flags = +#if defined(__x86_64__) || defined(__i386__) + JITDUMP_FLAGS_ARCH_TIMESTAMP, +#else + 0, +#endif + }; + char filename[256]; + int fd; + FILE *f; + void *m; + + snprintf(filename, sizeof(filename), "jit-%d.dump", getpid()); + /* Securely open using O_CREAT | O_EXCL to prevent symlink attacks. */ + fd = open(filename, O_CREAT | O_EXCL | O_RDWR, 0644); + if (fd < 0) { + pr_err("Failed to open jitdump '%s': %s\n", filename, strerror(errno)); + return NULL; + } + f = fdopen(fd, "w+"); + if (!f) { + pr_err("Failed to associate stream with fd for '%s'\n", filename); + close(fd); + unlink(filename); + return NULL; + } + /* Create an MMAP event for the jitdump file. That is how perf tool finds it. */ + m = mmap(0, getpagesize(), PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(f), 0); + if (m == MAP_FAILED) { + pr_err("mmap failed: %s\n", strerror(errno)); + fclose(f); + unlink(filename); + return NULL; + } + munmap(m, getpagesize()); + + if (fwrite(&header, sizeof(header), 1, f) != 1) { + pr_err("Error writing jitdump header\n"); + fclose(f); + unlink(filename); + return NULL; + } + return f; +} + +static int write_jitdump(FILE *f, void *addr, const void *dat, size_t sz, uint64_t *idx) +{ + const char *sym = "jit_workload"; + size_t sym_len = strlen(sym) + 1; + struct jr_code_load rec = { + .p.id = JIT_CODE_LOAD, + .p.total_size = sizeof(rec) + sym_len + sz, + .p.timestamp = get_timestamp(), + .pid = getpid(), + .tid = gettid(), + .vma = (unsigned long)addr, + .code_addr = (unsigned long)addr, + .code_size = sz, + .code_index = ++*idx, + }; + + if (fwrite(&rec, sizeof(rec), 1, f) != 1 || + fwrite(sym, sym_len, 1, f) != 1 || + fwrite(dat, sz, 1, f) != 1) + return -1; + return 0; +} + +static void close_jitdump(FILE *f) +{ + fclose(f); +} + +static int jitdump(int argc __maybe_unused, const char **argv __maybe_unused) +{ +#if defined(__x86_64__) || defined(__i386__) + /* Code to execute: mov CHK_BYTE, %eax ; ret */ + uint8_t dat[] = { 0xb8, CHK_BYTE, 0x00, 0x00, 0x00, 0xc3 }; +#elif defined(__aarch64__) + /* Code to execute: mov w0, #CHK_BYTE ; ret */ + uint8_t dat[] = { + (CHK_BYTE << 5) & 0xff, (CHK_BYTE >> 3) & 0xff, 0x80, 0x52, + 0xc0, 0x03, 0x5f, 0xd6 + }; +#elif defined(__riscv) + /* Code to execute: li a0, CHK_BYTE ; ret */ + uint8_t dat[] = { + 0x13, 0x05, (CHK_BYTE << 4) & 0xff, (CHK_BYTE >> 4) & 0xff, + 0x67, 0x80, 0x00, 0x00 + }; +#elif defined(__powerpc__) + /* Code to execute: li r3, CHK_BYTE ; blr */ + uint32_t dat[] = { 0x38600000 | (CHK_BYTE & 0xffff), 0x4e800020 }; +#elif defined(__s390x__) + /* Code to execute: lhi %r2, CHK_BYTE ; br %r14 */ + uint8_t dat[] = { 0xa7, 0x28, (CHK_BYTE >> 8) & 0xff, CHK_BYTE & 0xff, 0x07, 0xfe }; +#elif defined(__arm__) + /* Code to execute: mov r0, #CHK_BYTE ; bx lr */ + uint8_t dat[] = { + CHK_BYTE & 0xff, 0x00, 0xa0, 0xe3, + 0x1e, 0xff, 0x2f, 0xe1 + }; +#elif defined(__mips__) + /* Code to execute: addiu $v0, $zero, CHK_BYTE ; jr $ra ; nop */ + uint32_t dat[] = { 0x24020000 | (CHK_BYTE & 0xffff), 0x03e00008, 0x00000000 }; +#elif defined(__loongarch__) + /* Code to execute: addi.w $a0, $zero, CHK_BYTE ; jirl $zero, $ra, 0 */ + uint32_t dat[] = { 0x02800004 | ((CHK_BYTE & 0xfff) << 10), 0x4c000020 }; +#else + uint32_t dat[0]; +#endif + void *addr; + FILE *f; + uint64_t idx = 0; + int ret = 1; + + /* Reachable fallback check for unsupported architectures right at start. */ + if (sizeof(dat) == 0) { + pr_err("JITDUMP workload not supported on this architecture\n"); + return 1; + } + + /* Get a memory page to store executable code. */ + addr = mmap(0, getpagesize(), PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (addr == MAP_FAILED) { + pr_err("Failed to map 1 -rwx page\n"); + return 1; + } + + f = open_jitdump(); + if (!f) { + pr_err("Failed to open JITDUMP\n"); + munmap(addr, getpagesize()); + return 1; + } + /* Copy executable code to executable memory page. */ + memcpy(addr, dat, sizeof(dat)); + /* Synchronize the Instruction and Data caches. */ + __builtin___clear_cache(addr, (char *)addr + sizeof(dat)); + + /* Record it in the jitdump file */ + if (write_jitdump(f, addr, dat, sizeof(dat), &idx) == 0) { + int (*fn)(void) = addr; + + /* Call the function. */ + ret = fn() - CHK_BYTE; + } + close_jitdump(f); + munmap(addr, getpagesize()); + return ret; +} + +DEFINE_WORKLOAD(jitdump); diff --git a/tools/perf/tests/workloads/leafloop.c b/tools/perf/tests/workloads/leafloop.c index f7561767e32c..c20c75f7ba49 100644 --- a/tools/perf/tests/workloads/leafloop.c +++ b/tools/perf/tests/workloads/leafloop.c @@ -6,26 +6,48 @@ #include "../tests.h" /* We want to check these symbols in perf script */ -noinline void leaf(volatile int b); -noinline void parent(volatile int b); +noinline void leaf(void); +noinline void parent(void); -static volatile int a; -static volatile sig_atomic_t done; +static volatile sig_atomic_t done asm("leafloop_done"); static void sighandler(int sig __maybe_unused) { done = 1; } -noinline void leaf(volatile int b) +#if defined(__aarch64__) +/* + * Write leaf() in assembly so it stays as a minimal leaf function with no + * stack frame and won't get silently broken in the future by any Perf wide + * compilation options like -fstack-protector-all. + */ +asm( + ".pushsection .text,\"ax\",%progbits\n" + ".global leaf\n" + ".type leaf, %function\n" + "leaf:\n" + " adrp x1, leafloop_done\n" + " ldr w2, [x1, #:lo12:leafloop_done]\n" + " cbz w2, leaf\n" + " ret\n" + ".size leaf, .-leaf\n" + ".popsection\n" +); + +#else + +noinline void leaf(void) { while (!done) - a += b; + ; } -noinline void parent(volatile int b) +#endif + +noinline void parent(void) { - leaf(b); + leaf(); } static int leafloop(int argc, const char **argv) @@ -39,7 +61,7 @@ static int leafloop(int argc, const char **argv) signal(SIGALRM, sighandler); alarm(sec); - parent(sec); + parent(); return 0; } diff --git a/tools/perf/tests/workloads/named_threads.c b/tools/perf/tests/workloads/named_threads.c new file mode 100644 index 000000000000..d051d41a3cfe --- /dev/null +++ b/tools/perf/tests/workloads/named_threads.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include "../tests.h" + +#define MAX_THREADS 25 + +static int iterations = 500; +int named_threads_work = 1234; + +typedef void *(*thread_fn_t)(void *); + +#define DEFINE_THREAD(n) \ +noinline void *named_threads_thread##n(void *arg __maybe_unused) \ +{ \ + pthread_setname_np(pthread_self(), "thread" #n); \ + for (int i = 0; i < iterations; i++) \ + named_threads_work += 3; \ + \ + return NULL; \ +} + +#define THREAD_LIST(macro) \ + macro(1) \ + macro(2) \ + macro(3) \ + macro(4) \ + macro(5) \ + macro(6) \ + macro(7) \ + macro(8) \ + macro(9) \ + macro(10) \ + macro(11) \ + macro(12) \ + macro(13) \ + macro(14) \ + macro(15) \ + macro(16) \ + macro(17) \ + macro(18) \ + macro(19) \ + macro(20) \ + macro(21) \ + macro(22) \ + macro(23) \ + macro(24) \ + macro(25) + +#define DECLARE_THREAD(n) void *named_threads_thread##n(void *arg); + +THREAD_LIST(DECLARE_THREAD) +THREAD_LIST(DEFINE_THREAD) + +#define THREAD_ENTRY(n) named_threads_thread##n, + +static thread_fn_t thread_fns[MAX_THREADS] = { + THREAD_LIST(THREAD_ENTRY) +}; + +/* + * Creates argv[0] threads that run a unique function named "thread[x]" which performs + * a multiplication in a loop for argv[1] loops. + */ +static int named_threads(int argc, const char **argv) +{ + pthread_t threads[MAX_THREADS]; + int nr_threads = 1; + int err = 0; + + if (argc > 0) + nr_threads = atoi(argv[0]); + + if (nr_threads <= 0 || nr_threads > MAX_THREADS) { + fprintf(stderr, "Error: num threads must be 1 - %d\n", MAX_THREADS); + return 1; + } + + if (argc > 1) + iterations = atoi(argv[1]); + + if (iterations < 0) { + fprintf(stderr, "Error: iterations must be non-negative\n"); + return 1; + } + + for (int i = 0; i < nr_threads; i++) { + int ret; + + ret = pthread_create(&threads[i], NULL, thread_fns[i], NULL); + if (ret) { + fprintf(stderr, "Error: failed to create thread%d: %s\n", + i + 1, strerror(ret)); + return 1; + } + } + + for (int i = 0; i < nr_threads; i++) + pthread_join(threads[i], NULL); + + return err; +} + +DEFINE_WORKLOAD(named_threads); diff --git a/tools/perf/tests/workloads/noploop.c b/tools/perf/tests/workloads/noploop.c index 656e472e6188..ca9f871e136f 100644 --- a/tools/perf/tests/workloads/noploop.c +++ b/tools/perf/tests/workloads/noploop.c @@ -15,15 +15,26 @@ static void sighandler(int sig __maybe_unused) static int noploop(int argc, const char **argv) { - int sec = 1; + double sec = 1.0; pthread_setname_np(pthread_self(), "perf-noploop"); if (argc > 0) - sec = atoi(argv[0]); + sec = atof(argv[0]); + + if (!(sec > 0.0)) { + fprintf(stderr, "Error: seconds (%f) must be > 0\n", sec); + return 1; + } signal(SIGINT, sighandler); signal(SIGALRM, sighandler); - alarm(sec); + + if (sec < 1.0) { + useconds_t usecs = (useconds_t)(sec * 1000000.0); + + ualarm(usecs > 0 ? usecs : 1, 0); + } else + alarm((unsigned int)sec); while (!done) continue; diff --git a/tools/perf/tests/workloads/thloop.c b/tools/perf/tests/workloads/thloop.c index bd8168f883fb..c830d739489f 100644 --- a/tools/perf/tests/workloads/thloop.c +++ b/tools/perf/tests/workloads/thloop.c @@ -31,14 +31,15 @@ static void *thfunc(void *arg) static int thloop(int argc, const char **argv) { - int nt = 2, sec = 1, err = 1; + int nt = 2, err = 1; + double sec = 1.0; pthread_t *thread_list = NULL; if (argc > 0) - sec = atoi(argv[0]); + sec = atof(argv[0]); - if (sec <= 0) { - fprintf(stderr, "Error: seconds (%d) must be >= 1\n", sec); + if (!(sec > 0.0)) { + fprintf(stderr, "Error: seconds (%f) must be > 0\n", sec); return 1; } @@ -67,7 +68,12 @@ static int thloop(int argc, const char **argv) goto out; } } - alarm(sec); + if (sec < 1.0) { + useconds_t usecs = (useconds_t)(sec * 1000000.0); + + ualarm(usecs > 0 ? usecs : 1, 0); + } else + alarm((unsigned int)sec); test_loop(); err = 0; out: diff --git a/tools/perf/trace/beauty/Build b/tools/perf/trace/beauty/Build index 561590ee8cda..bf9553f683f8 100644 --- a/tools/perf/trace/beauty/Build +++ b/tools/perf/trace/beauty/Build @@ -19,6 +19,24 @@ perf-y += socket.o perf-y += statx.o perf-y += sync_file_range.o perf-y += timespec.o +perf-util-y += syscalltbl.o +perf-y += fsconfig.o +perf-y += eventfd.o +perf-y += futex_op.o +perf-y += futex_val3.o +perf-y += mmap.o +perf-y += mode_t.o +perf-y += msg_flags.o +perf-y += open_flags.o +perf-y += perf_event_open.o +perf-y += pid.o +perf-y += sched_policy.o +perf-y += seccomp.o +perf-y += signum.o +perf-y += socket_type.o +perf-y += waitid_options.o +perf-util-y += arch_errno_names.o + perf-y += tracepoints/ ifdef SHELLCHECK @@ -34,3 +52,269 @@ $(OUTPUT)%.shellcheck_log: % $(Q)$(call echo-cmd,test)$(SHELLCHECK) "$<" > $@ || (cat $@ && rm $@ && false) perf-y += $(SHELL_TEST_LOGS) + +beauty_linux_dir := $(srctree)/tools/perf/trace/beauty/include/linux/ +beauty_uapi_linux_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/linux/ +beauty_uapi_sound_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/sound/ +beauty_arch_asm_dir := $(srctree)/tools/perf/trace/beauty/arch/x86/include/asm/ +beauty_x86_arch_asm_uapi_dir := $(srctree)/tools/perf/trace/beauty/arch/x86/include/uapi/asm/ + +linux_uapi_dir := $(srctree)/tools/include/uapi/linux +asm_generic_uapi_dir := $(srctree)/tools/include/uapi/asm-generic +arch_asm_uapi_dir := $(srctree)/tools/arch/$(SRCARCH)/include/uapi/asm/ +x86_arch_asm_dir := $(srctree)/tools/arch/x86/include/asm/ + +beauty_outdir := $(OUTPUT)trace/beauty/generated +beauty_ioctl_outdir := $(beauty_outdir)/ioctl + +syscall_array := $(beauty_outdir)/syscalltbl.c +syscall_tbl := $(srctree)/tools/perf/trace/beauty/syscalltbl.sh +syscall_tbl_data := $(srctree)/tools/scripts/syscall.tbl \ + $(wildcard $(srctree)/tools/perf/arch/*/entry/syscalls/syscall*.tbl) + +$(syscall_array): $(syscall_tbl) $(syscall_tbl_data) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(syscall_tbl)' $(srctree)/tools $@ + +fs_at_flags_array := $(beauty_outdir)/fs_at_flags_array.c +fs_at_flags_tbl := $(srctree)/tools/perf/trace/beauty/fs_at_flags.sh + +$(fs_at_flags_array): $(beauty_uapi_linux_dir)/fcntl.h $(fs_at_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fs_at_flags_tbl)' $(beauty_uapi_linux_dir) > $@ + +clone_flags_array := $(beauty_outdir)/clone_flags_array.c +clone_flags_tbl := $(srctree)/tools/perf/trace/beauty/clone.sh + +$(clone_flags_array): $(beauty_uapi_linux_dir)/sched.h $(clone_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(clone_flags_tbl)' $(beauty_uapi_linux_dir) > $@ + +drm_ioctl_array := $(beauty_ioctl_outdir)/drm_ioctl_array.c +drm_hdr_dir := $(srctree)/tools/perf/trace/beauty/include/uapi/drm +drm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/drm_ioctl.sh + +$(drm_ioctl_array): $(drm_hdr_dir)/drm.h $(drm_hdr_dir)/i915_drm.h $(drm_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(drm_ioctl_tbl)' $(drm_hdr_dir) > $@ + +fadvise_advice_array := $(beauty_outdir)/fadvise_advice_array.c +fadvise_advice_tbl := $(srctree)/tools/perf/trace/beauty/fadvise.sh + +$(fadvise_advice_array): $(beauty_uapi_linux_dir)/fadvise.h $(fadvise_advice_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fadvise_advice_tbl)' $(beauty_uapi_linux_dir) > $@ + +fsmount_arrays := $(beauty_outdir)/fsmount_arrays.c +fsmount_tbls := $(srctree)/tools/perf/trace/beauty/fsmount.sh + +$(fsmount_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_tbls) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fsmount_tbls)' $(beauty_uapi_linux_dir) > $@ + +fsmount_attr_arrays := $(beauty_outdir)/fsmount_attr_arrays.c +fsmount_attr_tbls := $(srctree)/tools/perf/trace/beauty/fsmount_attr.sh + +$(fsmount_attr_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsmount_attr_tbls) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fsmount_attr_tbls)' $(beauty_uapi_linux_dir) > $@ + +fspick_arrays := $(beauty_outdir)/fspick_arrays.c +fspick_tbls := $(srctree)/tools/perf/trace/beauty/fspick.sh + +$(fspick_arrays): $(beauty_uapi_linux_dir)/mount.h $(fspick_tbls) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fspick_tbls)' $(beauty_uapi_linux_dir) > $@ + +fsconfig_arrays := $(beauty_outdir)/fsconfig_arrays.c +fsconfig_tbls := $(srctree)/tools/perf/trace/beauty/fsconfig.sh + +$(fsconfig_arrays): $(beauty_uapi_linux_dir)/mount.h $(fsconfig_tbls) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(fsconfig_tbls)' $(beauty_uapi_linux_dir) > $@ + +pkey_alloc_access_rights_array := $(beauty_outdir)/pkey_alloc_access_rights_array.c +asm_generic_hdr_dir := $(srctree)/tools/include/uapi/asm-generic/ +pkey_alloc_access_rights_tbl := $(srctree)/tools/perf/trace/beauty/pkey_alloc_access_rights.sh + +$(pkey_alloc_access_rights_array): $(asm_generic_hdr_dir)/mman-common.h $(pkey_alloc_access_rights_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(pkey_alloc_access_rights_tbl)' $(asm_generic_hdr_dir) > $@ + +sndrv_ctl_ioctl_array := $(beauty_ioctl_outdir)/sndrv_ctl_ioctl_array.c +sndrv_ctl_hdr_dir := $(srctree)/tools/include/uapi/sound +sndrv_ctl_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/sndrv_ctl_ioctl.sh + +$(sndrv_ctl_ioctl_array): $(beauty_uapi_sound_dir)/asound.h $(sndrv_ctl_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(sndrv_ctl_ioctl_tbl)' $(beauty_uapi_sound_dir) > $@ + +sndrv_pcm_ioctl_array := $(beauty_ioctl_outdir)/sndrv_pcm_ioctl_array.c +sndrv_pcm_hdr_dir := $(srctree)/tools/include/uapi/sound +sndrv_pcm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/sndrv_pcm_ioctl.sh + +$(sndrv_pcm_ioctl_array): $(beauty_uapi_sound_dir)/asound.h $(sndrv_pcm_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(sndrv_pcm_ioctl_tbl)' $(beauty_uapi_sound_dir) > $@ + +kcmp_type_array := $(beauty_outdir)/kcmp_type_array.c +kcmp_hdr_dir := $(srctree)/tools/include/uapi/linux/ +kcmp_type_tbl := $(srctree)/tools/perf/trace/beauty/kcmp_type.sh + +$(kcmp_type_array): $(kcmp_hdr_dir)/kcmp.h $(kcmp_type_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(kcmp_type_tbl)' $(kcmp_hdr_dir) > $@ + +kvm_ioctl_array := $(beauty_ioctl_outdir)/kvm_ioctl_array.c +kvm_hdr_dir := $(srctree)/tools/include/uapi/linux +kvm_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/kvm_ioctl.sh + +$(kvm_ioctl_array): $(kvm_hdr_dir)/kvm.h $(kvm_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(kvm_ioctl_tbl)' $(kvm_hdr_dir) > $@ + +socket_arrays := $(beauty_outdir)/socket.c +socket_tbl := $(srctree)/tools/perf/trace/beauty/socket.sh + +$(socket_arrays): $(linux_uapi_dir)/in.h $(beauty_linux_dir)/socket.h $(socket_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(socket_tbl)' $(linux_uapi_dir) $(beauty_linux_dir) > $@ + +sockaddr_arrays := $(beauty_outdir)/sockaddr.c +sockaddr_tbl := $(srctree)/tools/perf/trace/beauty/sockaddr.sh + +$(sockaddr_arrays): $(beauty_linux_dir)/socket.h $(sockaddr_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(sockaddr_tbl)' $(beauty_linux_dir) > $@ + +vhost_virtio_ioctl_array := $(beauty_ioctl_outdir)/vhost_virtio_ioctl_array.c +vhost_virtio_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/vhost_virtio_ioctl.sh + +$(vhost_virtio_ioctl_array): $(beauty_uapi_linux_dir)/vhost.h $(vhost_virtio_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(vhost_virtio_ioctl_tbl)' $(beauty_uapi_linux_dir) > $@ + +perf_ioctl_array := $(beauty_ioctl_outdir)/perf_ioctl_array.c +perf_hdr_dir := $(srctree)/tools/include/uapi/linux +perf_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/perf_ioctl.sh + +$(perf_ioctl_array): $(perf_hdr_dir)/perf_event.h $(perf_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(perf_ioctl_tbl)' $(perf_hdr_dir) > $@ + +madvise_behavior_array := $(beauty_outdir)/madvise_behavior_array.c +madvise_hdr_dir := $(srctree)/tools/include/uapi/asm-generic/ +madvise_behavior_tbl := $(srctree)/tools/perf/trace/beauty/madvise_behavior.sh + +$(madvise_behavior_array): $(madvise_hdr_dir)/mman-common.h $(madvise_behavior_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(madvise_behavior_tbl)' $(madvise_hdr_dir) > $@ + +mmap_flags_array := $(beauty_outdir)/mmap_flags_array.c +mmap_flags_tbl := $(srctree)/tools/perf/trace/beauty/mmap_flags.sh + +$(mmap_flags_array): $(linux_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman-common.h $(mmap_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(mmap_flags_tbl)' $(linux_uapi_dir) $(asm_generic_uapi_dir) $(arch_asm_uapi_dir) > $@ + +mremap_flags_array := $(beauty_outdir)/mremap_flags_array.c +mremap_flags_tbl := $(srctree)/tools/perf/trace/beauty/mremap_flags.sh + +$(mremap_flags_array): $(linux_uapi_dir)/mman.h $(mremap_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(mremap_flags_tbl)' $(linux_uapi_dir) > $@ + +mount_flags_array := $(beauty_outdir)/mount_flags_array.c +mount_flags_tbl := $(srctree)/tools/perf/trace/beauty/mount_flags.sh + +$(mount_flags_array): $(beauty_uapi_linux_dir)/mount.h $(mount_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(mount_flags_tbl)' $(beauty_uapi_linux_dir) > $@ + +move_mount_flags_array := $(beauty_outdir)/move_mount_flags_array.c +move_mount_flags_tbl := $(srctree)/tools/perf/trace/beauty/move_mount_flags.sh + +$(move_mount_flags_array): $(beauty_uapi_linux_dir)/mount.h $(move_mount_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(move_mount_flags_tbl)' $(beauty_uapi_linux_dir) > $@ + +mmap_prot_array := $(beauty_outdir)/mmap_prot_array.c +mmap_prot_tbl := $(srctree)/tools/perf/trace/beauty/mmap_prot.sh + +$(mmap_prot_array): $(asm_generic_uapi_dir)/mman.h $(asm_generic_uapi_dir)/mman-common.h $(mmap_prot_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(mmap_prot_tbl)' $(asm_generic_uapi_dir) $(arch_asm_uapi_dir) > $@ + +prctl_option_array := $(beauty_outdir)/prctl_option_array.c +prctl_option_tbl := $(srctree)/tools/perf/trace/beauty/prctl_option.sh + +$(prctl_option_array): $(beauty_uapi_linux_dir)/prctl.h $(prctl_option_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(prctl_option_tbl)' $(beauty_uapi_linux_dir) > $@ + +usbdevfs_ioctl_array := $(beauty_ioctl_outdir)/usbdevfs_ioctl_array.c +usbdevfs_ioctl_tbl := $(srctree)/tools/perf/trace/beauty/usbdevfs_ioctl.sh + +$(usbdevfs_ioctl_array): $(beauty_uapi_linux_dir)/usbdevice_fs.h $(usbdevfs_ioctl_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(usbdevfs_ioctl_tbl)' $(beauty_uapi_linux_dir) > $@ + +x86_arch_prctl_code_array := $(beauty_outdir)/x86_arch_prctl_code_array.c +x86_arch_prctl_code_tbl := $(srctree)/tools/perf/trace/beauty/x86_arch_prctl.sh + +$(x86_arch_prctl_code_array): $(beauty_x86_arch_asm_uapi_dir)/prctl.h $(x86_arch_prctl_code_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(x86_arch_prctl_code_tbl)' $(beauty_x86_arch_asm_uapi_dir) > $@ + + +rename_flags_array := $(beauty_outdir)/rename_flags_array.c +rename_flags_tbl := $(srctree)/tools/perf/trace/beauty/rename_flags.sh + +$(rename_flags_array): $(beauty_uapi_linux_dir)/fs.h $(rename_flags_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(rename_flags_tbl)' $(beauty_uapi_linux_dir) > $@ + + +statx_mask_array := $(beauty_outdir)/statx_mask_array.c +statx_mask_tbl := $(srctree)/tools/perf/trace/beauty/statx_mask.sh + +$(statx_mask_array): $(beauty_uapi_linux_dir)/stat.h $(statx_mask_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(statx_mask_tbl)' $(beauty_uapi_linux_dir) > $@ + +sync_file_range_arrays := $(beauty_outdir)/sync_file_range_arrays.c +sync_file_range_tbls := $(srctree)/tools/perf/trace/beauty/sync_file_range.sh + +$(sync_file_range_arrays): $(beauty_uapi_linux_dir)/fs.h $(sync_file_range_tbls) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(sync_file_range_tbls)' $(beauty_uapi_linux_dir) > $@ + +$(OUTPUT)trace/beauty/syscalltbl.o: $(syscall_array) +$(OUTPUT)trace/beauty/fsconfig.o: $(fsconfig_arrays) +$(OUTPUT)trace/beauty/clone.o: $(clone_flags_array) +$(OUTPUT)trace/beauty/fs_at_flags.o: $(fs_at_flags_array) +$(OUTPUT)trace/beauty/fsmount.o: $(fsmount_arrays) $(fsmount_attr_arrays) +$(OUTPUT)trace/beauty/fspick.o: $(fspick_arrays) +$(OUTPUT)trace/beauty/ioctl.o: $(drm_ioctl_array) $(sndrv_pcm_ioctl_array) $(sndrv_ctl_ioctl_array) $(kvm_ioctl_array) $(vhost_virtio_ioctl_array) $(perf_ioctl_array) $(usbdevfs_ioctl_array) +$(OUTPUT)trace/beauty/kcmp.o: $(kcmp_type_array) +$(OUTPUT)trace/beauty/mmap.o: $(mmap_prot_array) $(mmap_flags_array) $(mremap_flags_array) $(madvise_behavior_array) +$(OUTPUT)trace/beauty/mount_flags.o: $(mount_flags_array) +$(OUTPUT)trace/beauty/move_mount.o: $(move_mount_flags_array) +$(OUTPUT)trace/beauty/pkey_alloc.o: $(pkey_alloc_access_rights_array) +$(OUTPUT)trace/beauty/prctl.o: $(prctl_option_array) +$(OUTPUT)trace/beauty/renameat.o: $(rename_flags_array) +$(OUTPUT)trace/beauty/sockaddr.o: $(sockaddr_arrays) +$(OUTPUT)trace/beauty/socket.o: $(socket_arrays) +$(OUTPUT)trace/beauty/statx.o: $(statx_mask_array) +$(OUTPUT)trace/beauty/sync_file_range.o: $(sync_file_range_arrays) +$(OUTPUT)trace/beauty/arch_prctl.o: $(x86_arch_prctl_code_array) + +arch_errno_name_array := $(beauty_outdir)/arch_errno_name_array.c +arch_errno_hdr_dir := $(srctree)/tools +arch_errno_tbl := $(srctree)/tools/perf/trace/beauty/arch_errno_names.sh + +$(arch_errno_name_array): $(arch_errno_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(arch_errno_tbl)' '$(patsubst -%,,$(CC))' $(arch_errno_hdr_dir) > $@ + +$(OUTPUT)trace/beauty/arch_errno_names.o: $(arch_errno_name_array) diff --git a/tools/perf/trace/beauty/arch_errno_names.c b/tools/perf/trace/beauty/arch_errno_names.c index ede031c3a9e0..156c6537e747 100644 --- a/tools/perf/trace/beauty/arch_errno_names.c +++ b/tools/perf/trace/beauty/arch_errno_names.c @@ -1 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/env.h" #include "trace/beauty/generated/arch_errno_name_array.c" diff --git a/tools/perf/trace/beauty/arch_errno_names.sh b/tools/perf/trace/beauty/arch_errno_names.sh index b22890b8d272..8751bfa4a2b2 100755 --- a/tools/perf/trace/beauty/arch_errno_names.sh +++ b/tools/perf/trace/beauty/arch_errno_names.sh @@ -52,21 +52,50 @@ process_arch() |IFS=, create_errno_lookup_func "$arch" } +arch_to_e_machine() +{ + case "$1" in + alpha) printf '\tcase EM_ALPHA:\n' ;; + arc) printf '\tcase EM_ARC:\n' ;; + arm) printf '\tcase EM_ARM:\n' ;; + arm64) printf '\tcase EM_AARCH64:\n' ;; + csky) printf '\tcase EM_CSKY:\n' ;; + hexagon) printf '\tcase EM_HEXAGON:\n' ;; + loongarch) printf '\tcase EM_LOONGARCH:\n' ;; + microblaze) printf '\tcase EM_MICROBLAZE:\n' ;; + mips) printf '\tcase EM_MIPS:\n' ;; + parisc) printf '\tcase EM_PARISC:\n' ;; + powerpc) printf '\tcase EM_PPC:\n\tcase EM_PPC64:\n' ;; + riscv) printf '\tcase EM_RISCV:\n' ;; + s390) printf '\tcase EM_S390:\n' ;; + sh) printf '\tcase EM_SH:\n' ;; + sparc) printf '\tcase EM_SPARC:\n\tcase EM_SPARCV9:\n' ;; + x86) printf '\tcase EM_386:\n\tcase EM_X86_64:\n' ;; + xtensa) printf '\tcase EM_XTENSA:\n' ;; + esac +} + create_arch_errno_table_func() { archlist="$1" default="$2" - printf 'static arch_syscalls__strerrno_t *\n' - printf 'arch_syscalls__strerrno_function(const char *arch)\n' + printf 'const char *arch_syscalls__strerrno(uint16_t e_machine, int err);\n\n' + printf '__attribute__((unused)) const char *\n' + printf 'arch_syscalls__strerrno(uint16_t e_machine, int err)\n' printf '{\n' + printf '\tswitch (e_machine) {\n' for arch in $archlist; do arch_str=$(arch_string "$arch") - printf '\tif (!strcmp(arch, "%s"))\n' "$arch_str" - printf '\t\treturn errno_to_name__%s;\n' "$arch_str" + ems=$(arch_to_e_machine "$arch_str") + if [ -n "$ems" ]; then + printf '%s\n' "$ems" + printf '\t\treturn errno_to_name__%s(err);\n' "$arch_str" + fi done arch_str=$(arch_string "$default") - printf '\treturn errno_to_name__%s;\n' "$arch_str" + printf '\tdefault:\n\t\treturn errno_to_name__%s(err);\n' "$arch_str" + printf '\t}\n' printf '}\n' } @@ -74,6 +103,20 @@ cat < +#include +#include + +#ifndef EM_AARCH64 +#define EM_AARCH64 183 +#endif + +#ifndef EM_CSKY +#define EM_CSKY 252 +#endif + +#ifndef EM_LOONGARCH +#define EM_LOONGARCH 258 +#endif EoHEADER diff --git a/tools/perf/trace/beauty/beauty.h b/tools/perf/trace/beauty/beauty.h index a90c35fa5c12..58a3206481ae 100644 --- a/tools/perf/trace/beauty/beauty.h +++ b/tools/perf/trace/beauty/beauty.h @@ -35,6 +35,8 @@ bool strarray__strtoul(struct strarray *sa, char *bf, size_t size, u64 *ret); bool strarray__strtoul_flags(struct strarray *sa, char *bf, size_t size, u64 *ret); struct trace; +bool trace__show_zeros(const struct trace *trace); +struct machine *trace__host(const struct trace *trace); struct thread; struct file { @@ -268,4 +270,62 @@ size_t open__scnprintf_flags(unsigned long flags, char *bf, size_t size, bool sh void syscall_arg__set_ret_scnprintf(struct syscall_arg *arg, size_t (*ret_scnprintf)(char *bf, size_t size, struct syscall_arg *arg)); +extern struct strarray strarray__fsconfig_cmds; + +size_t syscall_arg__scnprintf_eventfd_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_EFD_FLAGS syscall_arg__scnprintf_eventfd_flags + +size_t syscall_arg__scnprintf_futex_op(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_FUTEX_OP syscall_arg__scnprintf_futex_op + +size_t syscall_arg__scnprintf_futex_val3(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_FUTEX_VAL3 syscall_arg__scnprintf_futex_val3 + +size_t syscall_arg__scnprintf_mmap_prot(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MMAP_PROT syscall_arg__scnprintf_mmap_prot + +extern struct strarray strarray__mmap_flags; + +size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MMAP_FLAGS syscall_arg__scnprintf_mmap_flags + +size_t syscall_arg__scnprintf_mremap_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MREMAP_FLAGS syscall_arg__scnprintf_mremap_flags + +size_t syscall_arg__scnprintf_madvise_behavior(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MADV_BHV syscall_arg__scnprintf_madvise_behavior + +size_t syscall_arg__scnprintf_mode_t(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MODE_T syscall_arg__scnprintf_mode_t + +size_t syscall_arg__scnprintf_msg_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_MSG_FLAGS syscall_arg__scnprintf_msg_flags + +size_t syscall_arg__scnprintf_perf_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_PERF_FLAGS syscall_arg__scnprintf_perf_flags + +size_t syscall_arg__scnprintf_perf_event_attr(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_PERF_ATTR syscall_arg__scnprintf_perf_event_attr +#define SCA_PERF_ATTR_FROM_USER(argname) \ + { .scnprintf = SCA_PERF_ATTR, \ + .from_user = true, } + +size_t syscall_arg__scnprintf_sched_policy(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_SCHED_POLICY syscall_arg__scnprintf_sched_policy + +size_t syscall_arg__scnprintf_seccomp_op(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_SECCOMP_OP syscall_arg__scnprintf_seccomp_op + +size_t syscall_arg__scnprintf_seccomp_flags(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_SECCOMP_FLAGS syscall_arg__scnprintf_seccomp_flags + +size_t syscall_arg__scnprintf_signum(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_SIGNUM syscall_arg__scnprintf_signum + +size_t syscall_arg__scnprintf_socket_type(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_SK_TYPE syscall_arg__scnprintf_socket_type + +size_t syscall_arg__scnprintf_waitid_options(char *bf, size_t size, struct syscall_arg *arg); +#define SCA_WAITID_OPTIONS syscall_arg__scnprintf_waitid_options + #endif /* _PERF_TRACE_BEAUTY_H */ diff --git a/tools/perf/trace/beauty/eventfd.c b/tools/perf/trace/beauty/eventfd.c index 4bab106213c6..18b661282834 100644 --- a/tools/perf/trace/beauty/eventfd.c +++ b/tools/perf/trace/beauty/eventfd.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #ifndef EFD_SEMAPHORE #define EFD_SEMAPHORE 1 #endif @@ -11,7 +13,7 @@ #define EFD_CLOEXEC 02000000 #endif -static size_t syscall_arg__scnprintf_eventfd_flags(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_eventfd_flags(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "EFD_"; @@ -35,5 +37,3 @@ static size_t syscall_arg__scnprintf_eventfd_flags(char *bf, size_t size, struct return printed; } - -#define SCA_EFD_FLAGS syscall_arg__scnprintf_eventfd_flags diff --git a/tools/perf/trace/beauty/fcntl.c b/tools/perf/trace/beauty/fcntl.c index d075904dccce..e1b99b8f55eb 100644 --- a/tools/perf/trace/beauty/fcntl.c +++ b/tools/perf/trace/beauty/fcntl.c @@ -9,6 +9,22 @@ #include #include +#ifndef F_GET_RW_HINT +#define F_GET_RW_HINT (F_LINUX_SPECIFIC_BASE + 11) +#endif + +#ifndef F_SET_RW_HINT +#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) +#endif + +#ifndef F_GET_FILE_RW_HINT +#define F_GET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 13) +#endif + +#ifndef F_SET_FILE_RW_HINT +#define F_SET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 14) +#endif + static size_t fcntl__scnprintf_getfd(unsigned long val, char *bf, size_t size, bool show_prefix) { return val ? scnprintf(bf, size, "%s", "0") : diff --git a/tools/perf/trace/beauty/fsconfig.c b/tools/perf/trace/beauty/fsconfig.c new file mode 100644 index 000000000000..98aa05315673 --- /dev/null +++ b/tools/perf/trace/beauty/fsconfig.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" +#include "trace/beauty/generated/fsconfig_arrays.c" + +DEFINE_STRARRAY(fsconfig_cmds, "FSCONFIG_"); diff --git a/tools/perf/trace/beauty/futex_op.c b/tools/perf/trace/beauty/futex_op.c index 00365156782b..05d2111e504b 100644 --- a/tools/perf/trace/beauty/futex_op.c +++ b/tools/perf/trace/beauty/futex_op.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" #include #ifndef FUTEX_WAIT_BITSET @@ -17,7 +18,7 @@ #define FUTEX_CLOCK_REALTIME 256 #endif -static size_t syscall_arg__scnprintf_futex_op(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_futex_op(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "FUTEX_"; @@ -59,5 +60,3 @@ static size_t syscall_arg__scnprintf_futex_op(char *bf, size_t size, struct sysc return printed; } - -#define SCA_FUTEX_OP syscall_arg__scnprintf_futex_op diff --git a/tools/perf/trace/beauty/futex_val3.c b/tools/perf/trace/beauty/futex_val3.c index 9114f7620571..fe4e82741ffc 100644 --- a/tools/perf/trace/beauty/futex_val3.c +++ b/tools/perf/trace/beauty/futex_val3.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" #include #ifndef FUTEX_BITSET_MATCH_ANY #define FUTEX_BITSET_MATCH_ANY 0xffffffff #endif -static size_t syscall_arg__scnprintf_futex_val3(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_futex_val3(char *bf, size_t size, struct syscall_arg *arg) { const char *prefix = "FUTEX_BITSET_"; unsigned int bitset = arg->val; @@ -15,5 +16,3 @@ static size_t syscall_arg__scnprintf_futex_val3(char *bf, size_t size, struct sy return scnprintf(bf, size, "%#xd", bitset); } - -#define SCA_FUTEX_VAL3 syscall_arg__scnprintf_futex_val3 diff --git a/tools/perf/trace/beauty/mmap.c b/tools/perf/trace/beauty/mmap.c index 3c5e97b93dd5..c8a4cd49c845 100644 --- a/tools/perf/trace/beauty/mmap.c +++ b/tools/perf/trace/beauty/mmap.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 #include +#include "trace/beauty/beauty.h" #include "trace/beauty/generated/mmap_prot_array.c" static DEFINE_STRARRAY(mmap_prot, "PROT_"); @@ -8,8 +9,7 @@ static size_t mmap__scnprintf_prot(unsigned long prot, char *bf, size_t size, bo { return strarray__scnprintf_flags(&strarray__mmap_prot, bf, size, show_prefix, prot); } - -static size_t syscall_arg__scnprintf_mmap_prot(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_mmap_prot(char *bf, size_t size, struct syscall_arg *arg) { unsigned long prot = arg->val; @@ -19,18 +19,18 @@ static size_t syscall_arg__scnprintf_mmap_prot(char *bf, size_t size, struct sys return mmap__scnprintf_prot(prot, bf, size, arg->show_string_prefix); } -#define SCA_MMAP_PROT syscall_arg__scnprintf_mmap_prot + #include "trace/beauty/generated/mmap_flags_array.c" -static DEFINE_STRARRAY(mmap_flags, "MAP_"); +DEFINE_STRARRAY(mmap_flags, "MAP_"); static size_t mmap__scnprintf_flags(unsigned long flags, char *bf, size_t size, bool show_prefix) { return strarray__scnprintf_flags(&strarray__mmap_flags, bf, size, show_prefix, flags); } -static size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size, + struct syscall_arg *arg) { unsigned long flags = arg->val; @@ -40,7 +40,7 @@ static size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size, return mmap__scnprintf_flags(flags, bf, size, arg->show_string_prefix); } -#define SCA_MMAP_FLAGS syscall_arg__scnprintf_mmap_flags + #include "trace/beauty/generated/mremap_flags_array.c" static DEFINE_STRARRAY(mremap_flags, "MREMAP_"); @@ -50,7 +50,7 @@ static size_t mremap__scnprintf_flags(unsigned long flags, char *bf, size_t size return strarray__scnprintf_flags(&strarray__mremap_flags, bf, size, show_prefix, flags); } -static size_t syscall_arg__scnprintf_mremap_flags(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_mremap_flags(char *bf, size_t size, struct syscall_arg *arg) { unsigned long flags = arg->val; @@ -60,7 +60,7 @@ static size_t syscall_arg__scnprintf_mremap_flags(char *bf, size_t size, struct return mremap__scnprintf_flags(flags, bf, size, arg->show_string_prefix); } -#define SCA_MREMAP_FLAGS syscall_arg__scnprintf_mremap_flags + static size_t madvise__scnprintf_behavior(int behavior, char *bf, size_t size) { @@ -73,10 +73,8 @@ static size_t madvise__scnprintf_behavior(int behavior, char *bf, size_t size) return scnprintf(bf, size, "%#", behavior); } -static size_t syscall_arg__scnprintf_madvise_behavior(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_madvise_behavior(char *bf, size_t size, + struct syscall_arg *arg) { return madvise__scnprintf_behavior(arg->val, bf, size); } - -#define SCA_MADV_BHV syscall_arg__scnprintf_madvise_behavior diff --git a/tools/perf/trace/beauty/mode_t.c b/tools/perf/trace/beauty/mode_t.c index 29a8fadfb7f9..9304b0bf3094 100644 --- a/tools/perf/trace/beauty/mode_t.c +++ b/tools/perf/trace/beauty/mode_t.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include #include #include @@ -20,7 +22,7 @@ #define S_IXUGO (S_IXUSR|S_IXGRP|S_IXOTH) #endif -static size_t syscall_arg__scnprintf_mode_t(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_mode_t(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "S_"; @@ -67,5 +69,3 @@ static size_t syscall_arg__scnprintf_mode_t(char *bf, size_t size, struct syscal return printed; } - -#define SCA_MODE_T syscall_arg__scnprintf_mode_t diff --git a/tools/perf/trace/beauty/msg_flags.c b/tools/perf/trace/beauty/msg_flags.c index 2da581ff0c80..be7f82677cbf 100644 --- a/tools/perf/trace/beauty/msg_flags.c +++ b/tools/perf/trace/beauty/msg_flags.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include #include @@ -27,8 +29,8 @@ # define MSG_CMSG_CLOEXEC 0x40000000 #endif -static size_t syscall_arg__scnprintf_msg_flags(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_msg_flags(char *bf, size_t size, + struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "MSG_"; @@ -72,5 +74,3 @@ static size_t syscall_arg__scnprintf_msg_flags(char *bf, size_t size, return printed; } - -#define SCA_MSG_FLAGS syscall_arg__scnprintf_msg_flags diff --git a/tools/perf/trace/beauty/open_flags.c b/tools/perf/trace/beauty/open_flags.c index 78f6566ef110..c2c7769e6595 100644 --- a/tools/perf/trace/beauty/open_flags.c +++ b/tools/perf/trace/beauty/open_flags.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include #include #include diff --git a/tools/perf/trace/beauty/perf_event_open.c b/tools/perf/trace/beauty/perf_event_open.c index 9f1ed989c775..6315b46bcdf0 100644 --- a/tools/perf/trace/beauty/perf_event_open.c +++ b/tools/perf/trace/beauty/perf_event_open.c @@ -1,4 +1,9 @@ // SPDX-License-Identifier: LGPL-2.1 +#include +#include "trace/beauty/beauty.h" +#include "util/evsel_fprintf.h" +#include + #ifndef PERF_FLAG_FD_NO_GROUP # define PERF_FLAG_FD_NO_GROUP (1UL << 0) #endif @@ -15,8 +20,8 @@ # define PERF_FLAG_FD_CLOEXEC (1UL << 3) /* O_CLOEXEC */ #endif -static size_t syscall_arg__scnprintf_perf_flags(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_perf_flags(char *bf, size_t size, + struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "PERF_"; @@ -43,7 +48,7 @@ static size_t syscall_arg__scnprintf_perf_flags(char *bf, size_t size, return printed; } -#define SCA_PERF_FLAGS syscall_arg__scnprintf_perf_flags + struct attr_fprintf_args { size_t size, printed; @@ -76,19 +81,34 @@ static size_t perf_event_attr___scnprintf(struct perf_event_attr *attr, char *bf static size_t syscall_arg__scnprintf_augmented_perf_event_attr(struct syscall_arg *arg, char *bf, size_t size) { - return perf_event_attr___scnprintf((void *)arg->augmented.args->value, bf, size, arg->trace->show_zeros); + struct perf_event_attr *attr = (void *)arg->augmented.args->value; + struct perf_event_attr local_attr; + + /* + * augmented_raw_syscalls.bpf.c (shipped with perf) copies + * PERF_ATTR_SIZE_VER0 bytes when the tracee passes size=0, + * but leaves the size field as 0. The payload size is + * guaranteed by perf's own BPF program, not externally + * controllable. Copy to a local so we can fix up size + * without writing to the potentially read-only augmented + * args buffer. + */ + if (!attr->size) { + memcpy(&local_attr, attr, PERF_ATTR_SIZE_VER0); + memset((void *)&local_attr + PERF_ATTR_SIZE_VER0, 0, + sizeof(local_attr) - PERF_ATTR_SIZE_VER0); + local_attr.size = PERF_ATTR_SIZE_VER0; + attr = &local_attr; + } + + return perf_event_attr___scnprintf(attr, bf, size, + trace__show_zeros(arg->trace)); } -static size_t syscall_arg__scnprintf_perf_event_attr(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_perf_event_attr(char *bf, size_t size, struct syscall_arg *arg) { if (arg->augmented.args) return syscall_arg__scnprintf_augmented_perf_event_attr(arg, bf, size); return scnprintf(bf, size, "%#lx", arg->val); } - -#define SCA_PERF_ATTR syscall_arg__scnprintf_perf_event_attr -// 'argname' is just documentational at this point, to remove the previous comment with that info -#define SCA_PERF_ATTR_FROM_USER(argname) \ - { .scnprintf = SCA_PERF_ATTR, \ - .from_user = true, } diff --git a/tools/perf/trace/beauty/pid.c b/tools/perf/trace/beauty/pid.c index 8f9c9950f8ba..cca4a3a5d9bd 100644 --- a/tools/perf/trace/beauty/pid.c +++ b/tools/perf/trace/beauty/pid.c @@ -1,11 +1,14 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" +#include "util/machine.h" +#include "util/thread.h" size_t syscall_arg__scnprintf_pid(char *bf, size_t size, struct syscall_arg *arg) { int pid = arg->val; struct trace *trace = arg->trace; size_t printed = scnprintf(bf, size, "%d", pid); - struct thread *thread = machine__findnew_thread(trace->host, pid, pid); + struct thread *thread = machine__findnew_thread(trace__host(trace), pid, pid); if (thread != NULL) { if (!thread__comm_set(thread)) diff --git a/tools/perf/trace/beauty/sched_policy.c b/tools/perf/trace/beauty/sched_policy.c index 68aa59eeed8d..3fb6d9e0fae9 100644 --- a/tools/perf/trace/beauty/sched_policy.c +++ b/tools/perf/trace/beauty/sched_policy.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include /* @@ -14,8 +16,8 @@ #define SCHED_RESET_ON_FORK 0x40000000 #endif -static size_t syscall_arg__scnprintf_sched_policy(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_sched_policy(char *bf, size_t size, + struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "SCHED_"; @@ -46,5 +48,3 @@ static size_t syscall_arg__scnprintf_sched_policy(char *bf, size_t size, return printed; } - -#define SCA_SCHED_POLICY syscall_arg__scnprintf_sched_policy diff --git a/tools/perf/trace/beauty/seccomp.c b/tools/perf/trace/beauty/seccomp.c index 637722e2796b..f345c66c1bfa 100644 --- a/tools/perf/trace/beauty/seccomp.c +++ b/tools/perf/trace/beauty/seccomp.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #ifndef SECCOMP_SET_MODE_STRICT #define SECCOMP_SET_MODE_STRICT 0 #endif @@ -6,7 +8,7 @@ #define SECCOMP_SET_MODE_FILTER 1 #endif -static size_t syscall_arg__scnprintf_seccomp_op(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_seccomp_op(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "SECCOMP_SET_MODE_"; @@ -24,14 +26,14 @@ static size_t syscall_arg__scnprintf_seccomp_op(char *bf, size_t size, struct sy return printed; } -#define SCA_SECCOMP_OP syscall_arg__scnprintf_seccomp_op + #ifndef SECCOMP_FILTER_FLAG_TSYNC #define SECCOMP_FILTER_FLAG_TSYNC 1 #endif -static size_t syscall_arg__scnprintf_seccomp_flags(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_seccomp_flags(char *bf, size_t size, + struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "SECCOMP_FILTER_FLAG_"; @@ -51,5 +53,3 @@ static size_t syscall_arg__scnprintf_seccomp_flags(char *bf, size_t size, return printed; } - -#define SCA_SECCOMP_FLAGS syscall_arg__scnprintf_seccomp_flags diff --git a/tools/perf/trace/beauty/signum.c b/tools/perf/trace/beauty/signum.c index 21220c56500a..6817e9febab9 100644 --- a/tools/perf/trace/beauty/signum.c +++ b/tools/perf/trace/beauty/signum.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include -static size_t syscall_arg__scnprintf_signum(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_signum(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "SIG"; @@ -53,5 +55,3 @@ static size_t syscall_arg__scnprintf_signum(char *bf, size_t size, struct syscal return scnprintf(bf, size, "%#x", sig); } - -#define SCA_SIGNUM syscall_arg__scnprintf_signum diff --git a/tools/perf/trace/beauty/socket_type.c b/tools/perf/trace/beauty/socket_type.c index bed8d5761ca8..059e3041d338 100644 --- a/tools/perf/trace/beauty/socket_type.c +++ b/tools/perf/trace/beauty/socket_type.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include #include @@ -18,7 +20,7 @@ #define SOCK_TYPE_MASK 0xf #endif -static size_t syscall_arg__scnprintf_socket_type(char *bf, size_t size, struct syscall_arg *arg) +size_t syscall_arg__scnprintf_socket_type(char *bf, size_t size, struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "SOCK_"; @@ -59,5 +61,3 @@ static size_t syscall_arg__scnprintf_socket_type(char *bf, size_t size, struct s return printed; } - -#define SCA_SK_TYPE syscall_arg__scnprintf_socket_type diff --git a/tools/perf/util/syscalltbl.c b/tools/perf/trace/beauty/syscalltbl.c similarity index 100% rename from tools/perf/util/syscalltbl.c rename to tools/perf/trace/beauty/syscalltbl.c diff --git a/tools/perf/util/syscalltbl.h b/tools/perf/trace/beauty/syscalltbl.h similarity index 100% rename from tools/perf/util/syscalltbl.h rename to tools/perf/trace/beauty/syscalltbl.h diff --git a/tools/perf/trace/beauty/tracepoints/Build b/tools/perf/trace/beauty/tracepoints/Build index e35087fdd108..9924ad24a6f1 100644 --- a/tools/perf/trace/beauty/tracepoints/Build +++ b/tools/perf/trace/beauty/tracepoints/Build @@ -1,2 +1,23 @@ perf-y += x86_irq_vectors.o perf-y += x86_msr.o + +beauty_outdir := $(OUTPUT)trace/beauty/generated +beauty_arch_asm_dir := $(srctree)/tools/perf/trace/beauty/arch/x86/include/asm/ +x86_arch_asm_dir := $(srctree)/tools/arch/x86/include/asm/ + +x86_arch_irq_vectors_array := $(beauty_outdir)/x86_arch_irq_vectors_array.c +x86_arch_irq_vectors_tbl := $(srctree)/tools/perf/trace/beauty/tracepoints/x86_irq_vectors.sh + +$(x86_arch_irq_vectors_array): $(beauty_arch_asm_dir)/irq_vectors.h $(x86_arch_irq_vectors_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(x86_arch_irq_vectors_tbl)' $(beauty_arch_asm_dir) > $@ + +x86_arch_MSRs_array := $(beauty_outdir)/x86_arch_MSRs_array.c +x86_arch_MSRs_tbl := $(srctree)/tools/perf/trace/beauty/tracepoints/x86_msr.sh + +$(x86_arch_MSRs_array): $(x86_arch_asm_dir)/msr-index.h $(x86_arch_MSRs_tbl) + $(call rule_mkdir) + $(Q)$(call echo-cmd,gen)$(SHELL) '$(x86_arch_MSRs_tbl)' $(x86_arch_asm_dir) > $@ + +$(OUTPUT)trace/beauty/tracepoints/x86_irq_vectors.o: $(x86_arch_irq_vectors_array) +$(OUTPUT)trace/beauty/tracepoints/x86_msr.o: $(x86_arch_MSRs_array) diff --git a/tools/perf/trace/beauty/waitid_options.c b/tools/perf/trace/beauty/waitid_options.c index d4d10b33ba0e..78ead5df985a 100644 --- a/tools/perf/trace/beauty/waitid_options.c +++ b/tools/perf/trace/beauty/waitid_options.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: LGPL-2.1 +#include "trace/beauty/beauty.h" + #include #include -static size_t syscall_arg__scnprintf_waitid_options(char *bf, size_t size, - struct syscall_arg *arg) +size_t syscall_arg__scnprintf_waitid_options(char *bf, size_t size, + struct syscall_arg *arg) { bool show_prefix = arg->show_string_prefix; const char *prefix = "W"; @@ -25,5 +27,3 @@ static size_t syscall_arg__scnprintf_waitid_options(char *bf, size_t size, return printed; } - -#define SCA_WAITID_OPTIONS syscall_arg__scnprintf_waitid_options diff --git a/tools/perf/ui/browser.c b/tools/perf/ui/browser.c index dc88427b4ae5..321187b204d3 100644 --- a/tools/perf/ui/browser.c +++ b/tools/perf/ui/browser.c @@ -513,6 +513,9 @@ unsigned int ui_browser__list_head_refresh(struct ui_browser *browser) struct list_head *head = browser->entries; int row = 0; + if (browser->nr_entries == 0) + return 0; + if (browser->top == NULL || browser->top == browser->entries) browser->top = ui_browser__list_head_filter_entries(browser, head->next); diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c index ea17e6d29a7e..d25761a8d25e 100644 --- a/tools/perf/ui/browsers/annotate.c +++ b/tools/perf/ui/browsers/annotate.c @@ -449,6 +449,9 @@ static bool annotate_browser__toggle_source(struct annotate_browser *browser, struct annotation_line *al; off_t offset = browser->b.index - browser->b.top_idx; + if (browser->b.nr_entries == 0) + return false; + browser->b.seek(&browser->b, offset, SEEK_CUR); al = list_entry(browser->b.top, struct annotation_line, node); @@ -542,8 +545,8 @@ static void annotate_browser__show_full_location(struct ui_browser *browser) static void ui_browser__init_asm_mode(struct ui_browser *browser) { struct annotation *notes = browser__annotation(browser); - ui_browser__reset_index(browser); browser->nr_entries = notes->src->nr_asm_entries; + ui_browser__reset_index(browser); } static int sym_title(struct symbol *sym, struct map *map, char *title, @@ -594,7 +597,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser, notes = symbol__annotation(dl->ops.target.sym); annotation__lock(notes); - if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) { + if (!symbol__hists(dl->ops.target.sym, evlist__nr_entries(evsel->evlist))) { annotation__unlock(notes); ui__warning("Not enough memory for annotating '%s' symbol!\n", dl->ops.target.sym->name); @@ -1185,7 +1188,7 @@ int __hist_entry__tui_annotate(struct hist_entry *he, struct map_symbol *ms, if (dso__annotate_warned(dso)) return -1; - if (not_annotated || !sym->annotate2) { + if (not_annotated || !symbol__is_annotate2(sym)) { err = symbol__annotate2(ms, evsel, &browser.arch); if (err) { annotate_browser__symbol_annotate_error(&browser, err); diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index cfa6386e6e1d..da7cc195b9f4 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -688,10 +688,10 @@ static int hist_browser__handle_hotkey(struct hist_browser *browser, bool warn_l ui_browser__update_nr_entries(&browser->b, nr_entries); if (warn_lost_event && - (evsel->evlist->stats.nr_lost_warned != - evsel->evlist->stats.nr_events[PERF_RECORD_LOST])) { - evsel->evlist->stats.nr_lost_warned = - evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + (evlist__stats(evsel->evlist)->nr_lost_warned != + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST])) { + evlist__stats(evsel->evlist)->nr_lost_warned = + evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; ui_browser__warn_lost_events(&browser->b); } @@ -3321,7 +3321,7 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h * No need to refresh, resort/decay histogram * entries if we are not collecting samples: */ - if (top->evlist->enabled) { + if (evlist__enabled(top->evlist)) { helpline = "Press 'f' to disable the events or 'h' to see other hotkeys"; hbt->refresh = delay_secs; } else { @@ -3493,7 +3493,7 @@ static void perf_evsel_menu__write(struct ui_browser *browser, unit, unit == ' ' ? "" : " ", ev_name); ui_browser__printf(browser, "%s", bf); - nr_events = evsel->evlist->stats.nr_events[PERF_RECORD_LOST]; + nr_events = evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST]; if (nr_events != 0) { menu->lost_events = true; if (!current_entry) @@ -3559,13 +3559,13 @@ static int perf_evsel_menu__run(struct evsel_menu *menu, ui_browser__show_title(&menu->b, title); switch (key) { case K_TAB: - if (pos->core.node.next == &evlist->core.entries) + if (pos->core.node.next == &evlist__core(evlist)->entries) pos = evlist__first(evlist); else pos = evsel__next(pos); goto browse_hists; case K_UNTAB: - if (pos->core.node.prev == &evlist->core.entries) + if (pos->core.node.prev == &evlist__core(evlist)->entries) pos = evlist__last(evlist); else pos = evsel__prev(pos); @@ -3618,7 +3618,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con struct evsel *pos; struct evsel_menu menu = { .b = { - .entries = &evlist->core.entries, + .entries = &evlist__core(evlist)->entries, .refresh = ui_browser__list_head_refresh, .seek = ui_browser__list_head_seek, .write = perf_evsel_menu__write, @@ -3646,7 +3646,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con static bool evlist__single_entry(struct evlist *evlist) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (nr_entries == 1) return true; @@ -3664,7 +3664,7 @@ static bool evlist__single_entry(struct evlist *evlist) int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt, float min_pcnt, struct perf_env *env, bool warn_lost_event) { - int nr_entries = evlist->core.nr_entries; + int nr_entries = evlist__nr_entries(evlist); if (evlist__single_entry(evlist)) { single_entry: { diff --git a/tools/perf/ui/browsers/map.c b/tools/perf/ui/browsers/map.c index c61ba3174a24..075a575cdc5d 100644 --- a/tools/perf/ui/browsers/map.c +++ b/tools/perf/ui/browsers/map.c @@ -32,8 +32,8 @@ static void map_browser__write(struct ui_browser *browser, void *nd, int row) ui_browser__set_percent_color(browser, 0, current_entry); ui_browser__printf(browser, "%*" PRIx64 " %*" PRIx64 " %c ", mb->addrlen, sym->start, mb->addrlen, sym->end, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w'); + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w'); width = browser->width - ((mb->addrlen * 2) + 4); if (width > 0) ui_browser__write_nstring(browser, sym->name, width); diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 70cc91d00804..330311cac550 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -6,6 +6,7 @@ perf-util-y += arm64-frame-pointer-unwind-support.o perf-util-y += addr2line.o perf-util-y += addr_location.o perf-util-y += annotate.o +perf-util-y += aslr.o perf-util-y += blake2s.o perf-util-y += block-info.o perf-util-y += block-range.o @@ -75,7 +76,6 @@ perf-util-y += sample.o perf-util-y += sample-raw.o perf-util-y += s390-sample-raw.o perf-util-y += amd-sample-raw.o -perf-util-$(CONFIG_TRACE) += syscalltbl.o perf-util-y += ordered-events.o perf-util-y += namespaces.o perf-util-y += comm.o @@ -216,6 +216,7 @@ ifndef CONFIG_SETNS perf-util-y += setns.o endif +perf-util-y += unwind.o perf-util-$(CONFIG_LIBDW) += probe-finder.o perf-util-$(CONFIG_LIBDW) += dwarf-aux.o perf-util-$(CONFIG_LIBDW) += dwarf-regs.o @@ -225,13 +226,11 @@ perf-util-$(CONFIG_LIBDW) += annotate-data.o perf-util-$(CONFIG_LIBDW) += libdw.o perf-util-$(CONFIG_LIBDW) += unwind-libdw.o -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind-local.o perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o -perf-util-$(CONFIG_LIBUNWIND_X86) += libunwind/x86_32.o -perf-util-$(CONFIG_LIBUNWIND_AARCH64) += libunwind/arm64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch/ ifeq ($(CONFIG_LIBTRACEEVENT),y) - perf-util-$(CONFIG_LIBBABELTRACE) += data-convert-bt.o + perf-util-$(CONFIG_BABELTRACE2_CTF_WRITER) += data-convert-bt.o endif perf-util-y += data-convert-json.o @@ -441,3 +440,18 @@ $(OUTPUT)%.pylint_log: % $(Q)$(call echo-cmd,test)pylint "$<" > $@ || (cat $@ && rm $@ && false) perf-util-y += $(PYLINT_TEST_LOGS) + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +include $(srctree)/tools/perf/bpf_skel.mak + +$(OUTPUT)util/bpf_ftrace.o: $(SKEL_OUT)/func_latency.skel.h +$(OUTPUT)util/bpf-filter.o: $(SKEL_OUT)/sample_filter.skel.h +$(OUTPUT)util/bpf_kwork_top.o: $(SKEL_OUT)/kwork_top.skel.h +$(OUTPUT)util/bpf_off_cpu.o: $(SKEL_OUT)/off_cpu.skel.h +$(OUTPUT)util/bpf-trace-summary.o: $(SKEL_OUT)/syscall_summary.skel.h +$(OUTPUT)util/bpf_counter_cgroup.o: $(SKEL_OUT)/bperf_cgroup.skel.h +$(OUTPUT)util/bpf_trace_augment.o: $(SKEL_OUT)/augmented_raw_syscalls.skel.h +$(OUTPUT)util/bpf_counter.o: $(SKEL_OUT)/bpf_prog_profiler.skel.h $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h +$(OUTPUT)util/bpf_lock_contention.o: $(SKEL_OUT)/lock_contention.skel.h +$(OUTPUT)util/bpf_kwork.o: $(SKEL_OUT)/kwork_trace.skel.h +endif diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index b084dee76b1a..cda3836329c3 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -21,6 +21,31 @@ static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; static bool zen4_ibs_extensions; static bool ldlat_cap; static bool dtlb_pgsize_cap; +static bool rmtsocket_cap; +static bool strmst_cap; + +/* + * Status fields of IBS_FETCH_CTL and IBS_FETCH_CTL_EXT are valid only if + * IBS_FETCH_CTL[PhyAddrValid] is set. + */ +static int fetch_ctl_depends_on_phy_addr_valid(void) +{ + static int depends = -1; /* -1: Don't know, 1: Yes, 0: No */ + + if (depends != -1) + return depends; + + depends = 0; + if (cpu_family > 0x1a || + (cpu_family == 0x1a && ( + (cpu_model >= 0x50 && cpu_model <= 0x5f) || + (cpu_model >= 0x80 && cpu_model <= 0xaf) || + (cpu_model >= 0xc0 && cpu_model <= 0xcf)))) { + depends = 1; + } + + return depends; +} static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) { @@ -43,6 +68,18 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) const char *ic_miss_str = NULL; const char *l1tlb_pgsz_str = NULL; char l3_miss_str[sizeof(" L3MissOnly _ FetchOcMiss _ FetchL3Miss _")] = ""; + char l3_miss_only_str[sizeof(" L3MissOnly _")] = ""; + + if (fetch_ctl_depends_on_phy_addr_valid() && !reg.phy_addr_valid) { + snprintf(l3_miss_only_str, sizeof(l3_miss_only_str), + " L3MissOnly %d", reg.l3_miss_only); + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d En %d Val %d Comp %d " + "PhyAddrValid 0 RandEn %d%s\n", reg.val, reg.fetch_maxcnt << 4, + reg.fetch_cnt << 4, reg.fetch_en, reg.fetch_val, reg.fetch_comp, + reg.rand_en, l3_miss_only_str); + return; + } if (cpu_family == 0x19 && cpu_model < 0x10) { /* @@ -72,8 +109,11 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) l3_miss_str); } -static void pr_ic_ibs_extd_ctl(union ic_ibs_extd_ctl reg) +static void pr_ic_ibs_extd_ctl(union ibs_fetch_ctl fetch_ctl, union ic_ibs_extd_ctl reg) { + if (fetch_ctl_depends_on_phy_addr_valid() && !fetch_ctl.phy_addr_valid) + return; + printf("ic_ibs_ext_ctl:\t%016llx IbsItlbRefillLat %3d\n", reg.val, reg.itlb_refill_lat); } @@ -126,8 +166,16 @@ static void pr_ibs_op_data2_extended(union ibs_op_data2 reg) /* 13 to 31 are reserved. Avoid printing them. */ }; int data_src = (reg.data_src_hi << 3) | reg.data_src_lo; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, (data_src == 1 || data_src == 2 || data_src == 5) ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, @@ -146,8 +194,16 @@ static void pr_ibs_op_data2_default(union ibs_op_data2 reg) " DataSrc 6=(reserved)", " DataSrc 7=Other" }; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, reg.data_src_lo == 2 ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, data_src_str[reg.data_src_lo]); @@ -172,6 +228,7 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char tlb_refill_lat_str[sizeof(" TlbRefillLat _____")] = ""; char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; @@ -214,17 +271,23 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); } + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (!zen4_ibs_extensions || reg.dc_phy_addr_valid) { + snprintf(tlb_refill_lat_str, sizeof(tlb_refill_lat_str), + " TlbRefillLat %5d", reg.tlb_refill_lat); + } + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " - "DcMissLat %5d TlbRefillLat %5d\n", + "DcMissLat %5d%s\n", reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, - reg.dc_miss_lat, reg.tlb_refill_lat); + reg.dc_miss_lat, tlb_refill_lat_str); } /* @@ -253,8 +316,12 @@ static void amd_dump_ibs_op(struct perf_sample *sample) pr_ibs_op_data3(*op_data3); if (op_data3->dc_lin_addr_valid) printf("IbsDCLinAd:\t%016llx\n", *(rip + 4)); - if (op_data3->dc_phy_addr_valid) + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (op_data3->dc_phy_addr_valid && *(rip + 5) && + (!zen4_ibs_extensions || op_data3->dc_lin_addr_valid)) { printf("IbsDCPhysAd:\t%016llx\n", *(rip + 5)); + } if (op_data->op_brn_ret && *(rip + 6)) printf("IbsBrTarget:\t%016llx\n", *(rip + 6)); } @@ -274,7 +341,7 @@ static void amd_dump_ibs_fetch(struct perf_sample *sample) printf("IbsFetchLinAd:\t%016llx\n", *addr++); if (fetch_ctl->phy_addr_valid) printf("IbsFetchPhysAd:\t%016llx\n", *addr); - pr_ic_ibs_extd_ctl(*extd_ctl); + pr_ic_ibs_extd_ctl(*fetch_ctl, *extd_ctl); } /* @@ -354,7 +421,7 @@ static void parse_cpuid(struct perf_env *env) */ bool evlist__has_amd_ibs(struct evlist *evlist) { - struct perf_env *env = perf_session__env(evlist->session); + struct perf_env *env = perf_session__env(evlist__session(evlist)); int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env); const char *pmu_mapping = perf_env__pmu_mappings(env); char name[sizeof("ibs_fetch")]; @@ -380,6 +447,12 @@ bool evlist__has_amd_ibs(struct evlist *evlist) if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) dtlb_pgsize_cap = 1; + if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket")) + rmtsocket_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "strmst")) + strmst_cap = 1; + if (ibs_fetch_type || ibs_op_type) { if (!cpu_family) parse_cpuid(env); diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c index 1eff0a27237d..4e4c58764082 100644 --- a/tools/perf/util/annotate-data.c +++ b/tools/perf/util/annotate-data.c @@ -74,7 +74,8 @@ void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind) break; } - dwarf_aggregate_size(die, &size); + if (dwarf_aggregate_size(die, &size) != 0) + size = 0; strbuf_init(&sb, 32); die_get_typename_from_type(die, &sb); @@ -146,9 +147,9 @@ static void pr_debug_scope(Dwarf_Die *scope_die) tag = dwarf_tag(scope_die); if (tag == DW_TAG_subprogram) - pr_info("[function] %s\n", dwarf_diename(scope_die)); + pr_info("[function] %s\n", die_name(scope_die)); else if (tag == DW_TAG_inlined_subroutine) - pr_info("[inlined] %s\n", dwarf_diename(scope_die)); + pr_info("[inlined] %s\n", die_name(scope_die)); else if (tag == DW_TAG_lexical_block) pr_info("[block]\n"); else @@ -250,9 +251,12 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) if (dwarf_aggregate_size(&die_mem, &size) < 0) size = 0; - if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) - dwarf_formudata(&attr, &loc); - else { + if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) { + if (dwarf_formudata(&attr, &loc) != 0) { + if (die_get_data_member_location(die, &loc) != 0) + loc = 0; + } + } else { /* bitfield member */ if (dwarf_attr_integrate(die, DW_AT_data_bit_offset, &attr) && dwarf_formudata(&attr, &loc) == 0) @@ -273,7 +277,9 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) dwarf_diename(die), (long)bit_size) < 0) member->var_name = NULL; } else { - member->var_name = strdup(dwarf_diename(die)); + const char *name = dwarf_diename(die); + + member->var_name = name ? strdup(name) : NULL; } if (member->var_name == NULL) { @@ -370,7 +376,8 @@ static struct annotated_data_type *dso__findnew_data_type(struct dso *dso, if (dwarf_tag(type_die) == DW_TAG_typedef) die_get_real_type(type_die, type_die); - dwarf_aggregate_size(type_die, &size); + if (dwarf_aggregate_size(type_die, &size) != 0) + size = 0; /* Check existing nodes in dso->data_types tree */ key.self.type_name = type_name; @@ -1569,7 +1576,7 @@ static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) offset = loc->offset; pr_debug_dtp("CU for %s (die:%#lx)\n", - dwarf_diename(&cu_die), (long)dwarf_dieoffset(&cu_die)); + die_name(&cu_die), (long)dwarf_dieoffset(&cu_die)); if (reg == DWARF_REG_PC) { if (get_global_var_type(&cu_die, dloc, dloc->ip, dloc->var_addr, @@ -1636,7 +1643,7 @@ static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) } pr_debug_dtp("found \"%s\" (die: %#lx) in scope=%d/%d (die: %#lx) ", - dwarf_diename(&var_die), (long)dwarf_dieoffset(&var_die), + die_name(&var_die), (long)dwarf_dieoffset(&var_die), i+1, nr_scopes, (long)dwarf_dieoffset(&scopes[i])); if (reg == DWARF_REG_PC) { @@ -1822,7 +1829,7 @@ int annotated_data_type__update_samples(struct annotated_data_type *adt, return 0; if (adt->histograms == NULL) { - int nr = evsel->evlist->core.nr_entries; + int nr = evlist__nr_entries(evsel->evlist); if (alloc_data_type_histograms(adt, nr) < 0) return -1; diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index e745f3034a0e..53b2a224b21d 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -213,9 +213,10 @@ static int __symbol__account_cycles(struct cyc_hist *ch, } static int __symbol__inc_addr_samples(struct map_symbol *ms, - struct annotated_source *src, struct evsel *evsel, u64 addr, + struct annotated_source *src, u64 addr, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; struct symbol *sym = ms->sym; long hash_key; u64 offset; @@ -235,7 +236,8 @@ static int __symbol__inc_addr_samples(struct map_symbol *ms, h = annotated_source__histogram(src, evsel); if (h == NULL) { pr_debug("%s(%d): ENOMEM! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 ", func: %d\n", - __func__, __LINE__, sym->name, sym->start, addr, sym->end, sym->type == STT_FUNC); + __func__, __LINE__, sym->name, sym->start, addr, sym->end, + symbol__type(sym) == STT_FUNC); return -ENOMEM; } @@ -318,7 +320,7 @@ struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists) } static int symbol__inc_addr_samples(struct map_symbol *ms, - struct evsel *evsel, u64 addr, + u64 addr, struct perf_sample *sample) { struct symbol *sym = ms->sym; @@ -326,8 +328,8 @@ static int symbol__inc_addr_samples(struct map_symbol *ms, if (sym == NULL) return 0; - src = symbol__hists(sym, evsel->evlist->core.nr_entries); - return src ? __symbol__inc_addr_samples(ms, src, evsel, addr, sample) : 0; + src = symbol__hists(sym, evlist__nr_entries(sample->evsel->evlist)); + return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; } static int symbol__account_br_cntr(struct annotated_branch *branch, @@ -337,7 +339,7 @@ static int symbol__account_br_cntr(struct annotated_branch *branch, { unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr; unsigned int base = evsel__leader(evsel)->br_cntr_idx; - unsigned int off = offset * evsel->evlist->nr_br_cntr; + unsigned int off = offset * evlist__nr_br_cntr(evsel->evlist); u64 *branch_br_cntr = branch->br_cntr; unsigned int i, mask, width; @@ -367,7 +369,7 @@ static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym, if (sym == NULL) return 0; - branch = symbol__find_branch_hist(sym, evsel->evlist->nr_br_cntr); + branch = symbol__find_branch_hist(sym, evlist__nr_br_cntr(evsel->evlist)); if (!branch) return -ENOMEM; if (addr < sym->start || addr >= sym->end) @@ -509,7 +511,7 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 static int annotation__compute_ipc(struct annotation *notes, size_t size, struct evsel *evsel) { - unsigned int br_cntr_nr = evsel->evlist->nr_br_cntr; + unsigned int br_cntr_nr = evlist__nr_br_cntr(evsel->evlist); int err = 0; s64 offset; @@ -581,16 +583,14 @@ static int annotation__compute_ipc(struct annotation *notes, size_t size, return 0; } -int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample, - struct evsel *evsel) +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample) { - return symbol__inc_addr_samples(&ams->ms, evsel, ams->al_addr, sample); + return symbol__inc_addr_samples(&ams->ms, ams->al_addr, sample); } -int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, - struct evsel *evsel, u64 ip) +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 ip) { - return symbol__inc_addr_samples(&he->ms, evsel, ip, sample); + return symbol__inc_addr_samples(&he->ms, ip, sample); } @@ -1813,7 +1813,7 @@ int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header) struct evsel *pos; struct strbuf sb; - if (evsel->evlist->nr_br_cntr <= 0) + if (evlist__nr_br_cntr(evsel->evlist) <= 0) return -ENOTSUP; strbuf_init(&sb, /*hint=*/ 0); @@ -2224,7 +2224,7 @@ int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel, annotation__init_column_widths(notes, sym); annotation__update_column_widths(notes); - sym->annotate2 = 1; + symbol__set_annotate2(sym, true); return 0; } diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 696e36dbf013..1aa6df7d1618 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -422,8 +422,7 @@ static inline struct annotation *symbol__annotation(struct symbol *sym) return (void *)sym - symbol_conf.priv_size; } -int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample, - struct evsel *evsel); +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample); struct annotated_branch *annotation__get_branch(struct annotation *notes); @@ -433,8 +432,7 @@ int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, struct evsel *evsel, u64 br_cntr); -int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, - struct evsel *evsel, u64 addr); +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 addr); struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists); void symbol__annotate_zero_histograms(struct symbol *sym); diff --git a/tools/perf/util/arm-spe-decoder/Build b/tools/perf/util/arm-spe-decoder/Build index ab500e0efe24..97a298d1e279 100644 --- a/tools/perf/util/arm-spe-decoder/Build +++ b/tools/perf/util/arm-spe-decoder/Build @@ -1 +1,3 @@ perf-util-y += arm-spe-pkt-decoder.o arm-spe-decoder.o + +CFLAGS_arm-spe-pkt-decoder.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c index 9e02b2bdd117..7a3a4815fd37 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c @@ -120,7 +120,8 @@ static int arm_spe_get_data(struct arm_spe_decoder *decoder) return decoder->len; } -static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) +static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder, + struct arm_spe_pkt *packet) { int ret; @@ -134,7 +135,8 @@ static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) } ret = arm_spe_get_packet(decoder->buf, decoder->len, - &decoder->packet); + packet, decoder->midr); + if (ret <= 0) { /* Move forward for 1 byte */ decoder->buf += 1; @@ -144,7 +146,7 @@ static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) decoder->buf += ret; decoder->len -= ret; - } while (decoder->packet.type == ARM_SPE_PAD); + } while (packet->type == ARM_SPE_PAD); return 1; } @@ -154,19 +156,20 @@ static int arm_spe_read_record(struct arm_spe_decoder *decoder) int err; int idx; u64 payload, ip; + struct arm_spe_pkt packet; memset(&decoder->record, 0x0, sizeof(decoder->record)); decoder->record.context_id = (u64)-1; while (1) { - err = arm_spe_get_next_packet(decoder); + err = arm_spe_get_next_packet(decoder, &packet); if (err <= 0) return err; - idx = decoder->packet.index; - payload = decoder->packet.payload; + idx = packet.index; + payload = packet.payload; - switch (decoder->packet.type) { + switch (packet.type) { case ARM_SPE_TIMESTAMP: decoder->record.timestamp = payload; return 1; diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h index 3310e05122f0..0cbcb501edc9 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -147,8 +147,7 @@ struct arm_spe_decoder { const unsigned char *buf; size_t len; - - struct arm_spe_pkt packet; + u64 midr; }; struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params); diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c index 5769ba2f4140..600677318f84 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include "arm-spe-pkt-decoder.h" +#include "../../arm64/include/asm/cputype.h" + static const char * const arm_spe_packet_name[] = { [ARM_SPE_PAD] = "PAD", [ARM_SPE_END] = "END", @@ -222,11 +225,12 @@ static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, } int arm_spe_get_packet(const unsigned char *buf, size_t len, - struct arm_spe_pkt *packet) + struct arm_spe_pkt *packet, u64 midr) { int ret; ret = arm_spe_do_get_packet(buf, len, packet); + packet->midr = midr; /* put multiple consecutive PADs on the same line, up to * the fixed-width output format of 16 bytes per line. */ @@ -276,6 +280,73 @@ static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, return ret; } +struct ev_string { + u8 event; + const char *desc; +}; + +static const struct ev_string common_ev_strings[] = { + { .event = EV_EXCEPTION_GEN, .desc = "EXCEPTION-GEN" }, + { .event = EV_RETIRED, .desc = "RETIRED" }, + { .event = EV_L1D_ACCESS, .desc = "L1D-ACCESS" }, + { .event = EV_L1D_REFILL, .desc = "L1D-REFILL" }, + { .event = EV_TLB_ACCESS, .desc = "TLB-ACCESS" }, + { .event = EV_TLB_WALK, .desc = "TLB-REFILL" }, + { .event = EV_NOT_TAKEN, .desc = "NOT-TAKEN" }, + { .event = EV_MISPRED, .desc = "MISPRED" }, + { .event = EV_LLC_ACCESS, .desc = "LLC-ACCESS" }, + { .event = EV_LLC_MISS, .desc = "LLC-REFILL" }, + { .event = EV_REMOTE_ACCESS, .desc = "REMOTE-ACCESS" }, + { .event = EV_ALIGNMENT, .desc = "ALIGNMENT" }, + { .event = EV_TRANSACTIONAL, .desc = "TXN" }, + { .event = EV_PARTIAL_PREDICATE, .desc = "SVE-PARTIAL-PRED" }, + { .event = EV_EMPTY_PREDICATE, .desc = "SVE-EMPTY-PRED" }, + { .event = EV_L2D_ACCESS, .desc = "L2D-ACCESS" }, + { .event = EV_L2D_MISS, .desc = "L2D-MISS" }, + { .event = EV_CACHE_DATA_MODIFIED, .desc = "HITM" }, + { .event = EV_RECENTLY_FETCHED, .desc = "LFB" }, + { .event = EV_DATA_SNOOPED, .desc = "SNOOPED" }, + { .event = EV_STREAMING_SVE_MODE, .desc = "STREAMING-SVE" }, + { .event = EV_SMCU, .desc = "SMCU" }, + { .event = 0, .desc = NULL }, +}; + +static const struct ev_string n1_event_strings[] = { + { .event = 12, .desc = "LATE-PREFETCH" }, + { .event = 0, .desc = NULL }, +}; + +static u64 print_event_list(int *err, char **buf, size_t *buf_len, + const struct ev_string *ev_strings, u64 payload) +{ + for (const struct ev_string *ev = ev_strings; ev->desc != NULL; ev++) { + if (payload & BIT_ULL(ev->event)) + arm_spe_pkt_out_string(err, buf, buf_len, " %s", ev->desc); + payload &= ~BIT_ULL(ev->event); + } + return payload; +} + +struct event_print_handle { + const struct midr_range *midr_ranges; + const struct ev_string *ev_strings; +}; + +#define EV_PRINT(range, strings) \ + { \ + .midr_ranges = range, \ + .ev_strings = strings, \ + } + +static const struct midr_range n1_event_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, +}; + +static const struct event_print_handle event_print_handles[] = { + EV_PRINT(n1_event_encoding_cpus, n1_event_strings), +}; + static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, char *buf, size_t buf_len) { @@ -283,51 +354,34 @@ static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, int err = 0; arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); + payload = print_event_list(&err, &buf, &buf_len, common_ev_strings, + payload); + + /* Try to decode IMPDEF bits for known CPUs */ + for (unsigned int i = 0; i < ARRAY_SIZE(event_print_handles); i++) { + if (is_midr_in_range_list(packet->midr, + event_print_handles[i].midr_ranges)) + payload = print_event_list(&err, &buf, &buf_len, + event_print_handles[i].ev_strings, + payload); + } - if (payload & BIT(EV_EXCEPTION_GEN)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); - if (payload & BIT(EV_RETIRED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); - if (payload & BIT(EV_L1D_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); - if (payload & BIT(EV_L1D_REFILL)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); - if (payload & BIT(EV_TLB_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); - if (payload & BIT(EV_TLB_WALK)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); - if (payload & BIT(EV_NOT_TAKEN)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); - if (payload & BIT(EV_MISPRED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); - if (payload & BIT(EV_LLC_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); - if (payload & BIT(EV_LLC_MISS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); - if (payload & BIT(EV_REMOTE_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); - if (payload & BIT(EV_ALIGNMENT)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); - if (payload & BIT(EV_TRANSACTIONAL)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TXN"); - if (payload & BIT(EV_PARTIAL_PREDICATE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); - if (payload & BIT(EV_EMPTY_PREDICATE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); - if (payload & BIT(EV_L2D_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-ACCESS"); - if (payload & BIT(EV_L2D_MISS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-MISS"); - if (payload & BIT(EV_CACHE_DATA_MODIFIED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " HITM"); - if (payload & BIT(EV_RECENTLY_FETCHED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LFB"); - if (payload & BIT(EV_DATA_SNOOPED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SNOOPED"); - if (payload & BIT(EV_STREAMING_SVE_MODE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " STREAMING-SVE"); - if (payload & BIT(EV_SMCU)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SMCU"); + /* + * Print remaining IMPDEF bits that weren't printed above as raw + * "IMPDEF:1,2,3,4" etc. + */ + if (payload) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IMPDEF:"); + for (int i = 0; i < 64; i++) { + const char *sep = payload & (payload - 1) ? "," : ""; + + if (payload & BIT_ULL(i)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%d%s", i, + sep); + payload &= ~BIT_ULL(i); + } + } + } return err; } diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h index adf4cde320aa..a3300bec4990 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -11,7 +11,7 @@ #include #include -#define ARM_SPE_PKT_DESC_MAX 256 +#define ARM_SPE_PKT_DESC_MAX 512 #define ARM_SPE_NEED_MORE_BYTES -1 #define ARM_SPE_BAD_PACKET -2 @@ -35,6 +35,7 @@ struct arm_spe_pkt { enum arm_spe_pkt_type type; unsigned char index; uint64_t payload; + uint64_t midr; }; /* Short header (HEADER0) and extended header (HEADER1) */ @@ -184,7 +185,7 @@ enum arm_spe_events { const char *arm_spe_pkt_name(enum arm_spe_pkt_type); int arm_spe_get_packet(const unsigned char *buf, size_t len, - struct arm_spe_pkt *packet); + struct arm_spe_pkt *packet, u64 midr); int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); #endif diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index e5835042acdf..552f063f126e 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -134,8 +134,10 @@ struct data_source_handle { .ds_synth = arm_spe__synth_##func, \ } +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr); + static void arm_spe_dump(struct arm_spe *spe __maybe_unused, - unsigned char *buf, size_t len) + unsigned char *buf, size_t len, u64 midr) { struct arm_spe_pkt packet; size_t pos = 0; @@ -148,7 +150,8 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, len); while (len) { - ret = arm_spe_get_packet(buf, len, &packet); + ret = arm_spe_get_packet(buf, len, &packet, midr); + if (ret > 0) pkt_len = ret; else @@ -174,10 +177,10 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, } static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, - size_t len) + size_t len, u64 midr) { printf(".\n"); - arm_spe_dump(spe, buf, len); + arm_spe_dump(spe, buf, len, midr); } static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) @@ -302,8 +305,10 @@ static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, if (speq->thread) { speq->pid = thread__pid(speq->thread); - if (queue->cpu == -1) + if (queue->cpu == -1) { speq->cpu = thread__cpu(speq->thread); + arm_spe__get_midr(spe, speq->cpu, &speq->decoder->midr); + } } } @@ -482,10 +487,30 @@ static void arm_spe__prep_branch_stack(struct arm_spe_queue *speq) bstack->hw_idx = -1ULL; } -static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type) +static int arm_spe__inject_event(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && spe->session && spe->session->evlist) + evsel = evlist__id2evsel(spe->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int @@ -497,7 +522,7 @@ arm_spe_deliver_synth_event(struct arm_spe *spe, int ret; if (spe->synth_opts.inject) { - ret = arm_spe__inject_event(event, sample, spe->sample_type); + ret = arm_spe__inject_event(spe, event, sample, spe->sample_type); if (ret) return ret; } @@ -972,14 +997,9 @@ static void arm_spe__synth_memory_level(struct arm_spe_queue *speq, } } -static void arm_spe__synth_ds(struct arm_spe_queue *speq, - const struct arm_spe_record *record, - union perf_mem_data_src *data_src) +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr) { - struct arm_spe *spe = speq->spe; - u64 *metadata = NULL; - u64 midr; - unsigned int i; + u64 *metadata; /* Metadata version 1 assumes all CPUs are the same (old behavior) */ if (spe->metadata_ver == 1) { @@ -987,15 +1007,35 @@ static void arm_spe__synth_ds(struct arm_spe_queue *speq, pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n"); cpuid = perf_env__cpuid(perf_session__env(spe->session)); - midr = strtol(cpuid, NULL, 16); - } else { - metadata = arm_spe__get_metadata_by_cpu(spe, speq->cpu); - if (!metadata) - return; + if (!cpuid) + goto err; - midr = metadata[ARM_SPE_CPU_MIDR]; + *midr = strtol(cpuid, NULL, 16); + return 0; } + metadata = arm_spe__get_metadata_by_cpu(spe, cpu); + if (!metadata) + goto err; + + *midr = metadata[ARM_SPE_CPU_MIDR]; + return 0; + +err: + pr_warning_once("Failed to get MIDR for CPU %d\n", cpu); + return -EINVAL; +} + +static void arm_spe__synth_ds(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + u64 midr; + unsigned int i; + + if (arm_spe__get_midr(speq->spe, speq->cpu, &midr)) + return; + for (i = 0; i < ARRAY_SIZE(data_source_handles); i++) { if (is_midr_in_range_list(midr, data_source_handles[i].midr_ranges)) { return data_source_handles[i].ds_synth(record, data_src); @@ -1233,6 +1273,7 @@ static int arm_spe__setup_queue(struct arm_spe *spe, if (queue->cpu != -1) speq->cpu = queue->cpu; + arm_spe__get_midr(spe, queue->cpu, &speq->decoder->midr); if (!speq->on_heap) { int ret; @@ -1475,8 +1516,11 @@ static int arm_spe_process_auxtrace_event(struct perf_session *session, /* Dump here now we have copied a piped trace out of the pipe */ if (dump_trace) { if (auxtrace_buffer__get_data(buffer, fd)) { + u64 midr = 0; + + arm_spe__get_midr(spe, buffer->cpu.cpu, &midr); arm_spe_dump_event(spe, buffer->data, - buffer->size); + buffer->size, midr); auxtrace_buffer__put_data(buffer); } } @@ -1958,7 +2002,7 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->tc.time_mult = tc->time_mult; spe->tc.time_zero = tc->time_zero; - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { spe->tc.time_cycles = tc->time_cycles; spe->tc.time_mask = tc->time_mask; spe->tc.cap_user_time_zero = tc->cap_user_time_zero; diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.c b/tools/perf/util/arm64-frame-pointer-unwind-support.c index 858ce2b01812..3af8c7a466e0 100644 --- a/tools/perf/util/arm64-frame-pointer-unwind-support.c +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.c @@ -2,6 +2,7 @@ #include "arm64-frame-pointer-unwind-support.h" #include "callchain.h" #include "event.h" +#include "record.h" #include "unwind.h" #include @@ -16,6 +17,11 @@ struct entries { #define SMPL_REG_MASK(b) (1ULL << (b)) +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts) +{ + opts->sample_user_regs |= SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + static bool get_leaf_frame_caller_enabled(struct perf_sample *sample) { struct regs_dump *regs; diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.h b/tools/perf/util/arm64-frame-pointer-unwind-support.h index 42d3a45490f5..ba35b295bfcd 100644 --- a/tools/perf/util/arm64-frame-pointer-unwind-support.h +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.h @@ -5,8 +5,10 @@ #include struct perf_sample; +struct record_opts; struct thread; +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts); u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int user_idx); #endif /* __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H */ diff --git a/tools/perf/util/aslr.c b/tools/perf/util/aslr.c new file mode 100644 index 000000000000..027695d96779 --- /dev/null +++ b/tools/perf/util/aslr.c @@ -0,0 +1,1410 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "aslr.h" + +#include "addr_location.h" +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "evlist.h" +#include "machine.h" +#include "map.h" +#include "thread.h" +#include "tool.h" +#include "session.h" +#include "data.h" +#include "dso.h" +#include "pmu.h" +#include "pmus.h" + +#include /* page_size */ +#include +#include +#include +#include +#include +#include + +/** + * struct remap_addresses_key - Key for mapping original addresses to remapped ones. + * @dso: Pointer to the DSO (Dynamic Shared Object) associated with the mapping. + * @invariant: Unique offset invariant within the VMA (Virtual Memory Area). + * Calculated as `start - pgoff`. This value remains constant when + * perf's internal `maps__fixup_overlap_and_insert` splits a map into + * fragmented VMA pieces due to overlapping events, allowing us to + * resolve split maps consistently back to the original VMA. + * @pid: Process ID associated with the mapping. + */ +struct remap_addresses_key { + struct machine *machine; + struct dso *dso; + u64 invariant; + pid_t pid; +}; + +struct aslr_mapping { + struct list_head node; + u64 orig_start; + u64 len; + u64 remap_start; +}; + +struct aslr_evsel_priv { + u64 orig_sample_type; + u64 orig_sample_regs_user; + u64 orig_sample_regs_intr; + int orig_sample_size; +}; + +static size_t evsel_hash(long key, void *ctx __maybe_unused) +{ + return (size_t)key; +} + +static bool evsel_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +struct process_top_address { + u64 remapped_max; +}; +struct aslr_tool { + /** @tool: The tool implemented here and a pointer to a delegate to process the data. */ + struct delegate_tool tool; + /** @machines: The machines with the input, not remapped, virtual address layout. */ + struct machines machines; + /** @event_copy: Buffer used to create an event to pass to the delegate. */ + char event_copy[PERF_SAMPLE_MAX_SIZE] __aligned(8); + /** @remap_addresses: mapping from remap_addresses_key to remapped address. */ + struct hashmap remap_addresses; + /** @top_addresses: mapping from process to max remapped address. */ + struct hashmap top_addresses; + /** + * @evsel_orig_attrs: mapping from evsel pointer to its original + * unstripped sample_type and registers bitmasks. + */ + struct hashmap evsel_orig_attrs; +}; + +static const pid_t kernel_pid = -1; + +/* Start remapping user processes from a small non-zero offset. */ +static const u64 user_space_start = 0x200000; +static const u64 kernel_space_start_64 = 0xffff800010000000ULL; +static const u64 kernel_space_start_32 = 0x80000000ULL; + +static size_t remap_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key = (struct remap_addresses_key *)_key; + void *dso_ptr = key->dso ? RC_CHK_ACCESS(key->dso) : NULL; + + return (size_t)key->machine ^ (size_t)dso_ptr ^ key->invariant ^ key->pid; +} + +static bool remap_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key1 = (struct remap_addresses_key *)_key1; + struct remap_addresses_key *key2 = (struct remap_addresses_key *)_key2; + + return key1->machine == key2->machine && + RC_CHK_EQUAL(key1->dso, key2->dso) && + key1->invariant == key2->invariant && + key1->pid == key2->pid; +} + +struct top_addresses_key { + struct machine *machine; + pid_t pid; +}; + +static size_t top_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct top_addresses_key *key = (struct top_addresses_key *)_key; + + return (size_t)key->machine ^ key->pid; +} + +static bool top_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct top_addresses_key *key1 = (struct top_addresses_key *)_key1; + struct top_addresses_key *key2 = (struct top_addresses_key *)_key2; + + return key1->machine == key2->machine && key1->pid == key2->pid; +} + +static u64 round_up_to_page_size(u64 addr) +{ + return (addr + page_size - 1) & ~((u64)page_size - 1); +} + +static u64 aslr_tool__remap_address(struct aslr_tool *aslr, + struct thread *aslr_thread, + u8 cpumode, + u64 addr) +{ + struct addr_location al; + struct remap_addresses_key key; + u64 *remapped_invariant_ptr = NULL; + u64 remap_addr = 0; + u8 effective_cpumode = cpumode; + struct dso *dso; + const char *dso_name; + + if (!aslr_thread) + return 0; /* No thread. */ + + addr_location__init(&al); + if (!thread__find_map(aslr_thread, cpumode, addr, &al)) { + /* + * If lookup fails with specified cpumode, try fallback to the other space + * to be robust against bad cpumode in samples. + */ + if (cpumode == PERF_RECORD_MISC_KERNEL) + effective_cpumode = PERF_RECORD_MISC_USER; + else if (cpumode == PERF_RECORD_MISC_USER) + effective_cpumode = PERF_RECORD_MISC_KERNEL; + else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL) + effective_cpumode = PERF_RECORD_MISC_GUEST_USER; + else if (cpumode == PERF_RECORD_MISC_GUEST_USER) + effective_cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + + if (!thread__find_map(aslr_thread, effective_cpumode, addr, &al)) { + addr_location__exit(&al); + return 0; /* No mmap. */ + } + } + + dso = map__dso(al.map); + dso_name = dso ? dso__long_name(dso) : NULL; + + key.machine = maps__machine(thread__maps(aslr_thread)); + key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + key.invariant = map__start(al.map) - map__pgoff(al.map); + else + key.invariant = map__start(al.map); + key.pid = (effective_cpumode == PERF_RECORD_MISC_KERNEL || + effective_cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + + if (hashmap__find(&aslr->remap_addresses, &key, &remapped_invariant_ptr)) { + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (addr - map__start(al.map)); + } else { + pr_debug("Cannot find a remapped entry for address %" PRIx64 " in mapping %" PRIx64 "(%zu) for pid=%d\n", + addr, map__start(al.map), map__size(al.map), key.pid); + } + + addr_location__exit(&al); + return remap_addr; +} + +struct aslr_machine_priv { + bool kernel_maps_loaded; +}; + +static int aslr_tool__preload_kernel_maps(struct machine *machine) +{ + struct aslr_machine_priv *mpriv = machine->priv; + + if (!mpriv) { + mpriv = zalloc(sizeof(*mpriv)); + if (!mpriv) + return -ENOMEM; + machine->priv = mpriv; + } + + if (!mpriv->kernel_maps_loaded) { + struct maps *kmaps = machine__kernel_maps(machine); + + if (kmaps) { + int err = maps__load_maps(kmaps); + + if (err < 0) { + pr_err("ASLR: Failed to preload kernel maps for machine pid %d\n", + machine->pid); + return err; + } + } + mpriv->kernel_maps_loaded = true; + } + return 0; +} + +static void aslr_tool__free_machine_priv(struct machine *machine) +{ + free(machine->priv); + machine->priv = NULL; +} + +static void aslr_tool__destroy_machines_priv(struct machines *machines) +{ + struct rb_node *nd; + + aslr_tool__free_machine_priv(&machines->host); + for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + aslr_tool__free_machine_priv(machine); + } +} + +static u64 aslr_tool__findnew_mapping(struct aslr_tool *aslr, + struct machine *session_machine, + struct thread *aslr_thread, + u8 cpumode, u64 start, + u64 len, u64 pgoff) +{ + /* Address location for dso lookup. */ + struct addr_location al; + /* Original ASLR address based key for the remap table. */ + struct remap_addresses_key remap_key; + /* The address in the ASLR sanitized address space less pg_off. */ + u64 *remapped_invariant_ptr; + /* Key for the maximum address in a process. */ + struct top_addresses_key top_addr_key; + /* Value in top address table. */ + struct process_top_address *top = NULL; + /* Address in ASLR sanitized address space. */ + u64 remap_addr; + /* Potentially allocated remap table key. */ + struct remap_addresses_key *new_remap_key = NULL; + /* + * Potentially allocated remap table key. + * TODO: Avoid allocation necessary for perf 32-bit binary support. + */ + u64 *new_remap_val = NULL; + int err; + + if (!aslr_thread) + return 0; + + /* The key to look up an incoming address to the outgoing value. */ + addr_location__init(&al); + remap_key.machine = maps__machine(thread__maps(aslr_thread)); + remap_key.pid = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + if (thread__find_map(aslr_thread, cpumode, start, &al)) { + struct dso *dso = map__dso(al.map); + const char *dso_name = dso ? dso__long_name(dso) : NULL; + + remap_key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + remap_key.invariant = map__start(al.map) - map__pgoff(al.map); + else + remap_key.invariant = map__start(al.map); + } else { + remap_key.dso = NULL; + remap_key.invariant = start; + } + + /* The key to look up top allocated address. */ + top_addr_key.machine = remap_key.machine; + top_addr_key.pid = remap_key.pid; + + if (hashmap__find(&aslr->remap_addresses, &remap_key, &remapped_invariant_ptr)) { + /* Mmap already exists. */ + u64 calculated_max; + + if (al.map) { + /* + * The cached value is the base of the invariant. We add the + * offset into the VMA (start - map__start), plus the map's + * pgoff, to get the precise virtual address within this chunk. + */ + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (start - map__start(al.map)); + } else { + /* + * For unmapped memory (e.g. kernel anonymous), the cached value + * was stored offset by pgoff. Adding pgoff yields the true remap_addr. + */ + remap_addr = *remapped_invariant_ptr + pgoff; + } + + calculated_max = remap_addr + len; + + /* See if top mapping was expanded. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + if (calculated_max > top->remapped_max) + top->remapped_max = calculated_max; + } + addr_location__exit(&al); + return remap_addr; + } + /* No mmap, create an entry from the top address. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + struct addr_location prev_al; + bool is_contiguous = false; + + /* Current max allocated mmap address within the process. */ + remap_addr = top->remapped_max; + + addr_location__init(&prev_al); + if (thread__find_map(aslr_thread, cpumode, start - 1, &prev_al)) { + if (map__end(prev_al.map) == start) + is_contiguous = true; + } + addr_location__exit(&prev_al); + + if (is_contiguous) { + /* Contiguous mapping, do not add 1 page gap! */ + remap_addr = round_up_to_page_size(remap_addr); + } else { + /* Give 1 page gap from current max page. */ + remap_addr = round_up_to_page_size(remap_addr); + remap_addr += page_size; + } + if (remap_addr + len > top->remapped_max) + top->remapped_max = remap_addr + len; + } else { + /* First address of the process, allocate key and first top address. */ + struct top_addresses_key *tk; + struct process_top_address *top_val; + struct perf_env *env = session_machine ? session_machine->env : NULL; + bool is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + u64 kernel_start_addr = is_64 ? kernel_space_start_64 : kernel_space_start_32; + + remap_addr = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_start_addr : user_space_start; + remap_addr = round_up_to_page_size(remap_addr); + + tk = malloc(sizeof(*tk)); + top_val = malloc(sizeof(*top_val)); + if (!tk || !top_val) { + err = -ENOMEM; + } else { + *tk = top_addr_key; + top_val->remapped_max = remap_addr + len; + err = hashmap__insert(&aslr->top_addresses, tk, top_val, + HASHMAP_ADD, NULL, NULL); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR process top address %m\n"); + free(tk); + free(top_val); + addr_location__exit(&al); + return 0; + } + } + /* Create rmeapping entry. */ + new_remap_key = malloc(sizeof(*new_remap_key)); + new_remap_val = malloc(sizeof(u64)); + if (!new_remap_key || !new_remap_val) { + err = -ENOMEM; + } else { + *new_remap_key = remap_key; + new_remap_key->dso = dso__get(remap_key.dso); + if (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) { + if (al.map) { + *new_remap_val = remap_addr - + (start - map__start(al.map)) - + map__pgoff(al.map); + } else { + /* + * Subtract pgoff from the base virtual address so that + * when the lookup path adds pgoff back, it perfectly + * cancels out and returns remap_addr. + */ + *new_remap_val = remap_addr - pgoff; + } + } else { + *new_remap_val = remap_addr - (al.map ? (start - map__start(al.map)) + + map__pgoff(al.map) : pgoff); + } + err = hashmap__add(&aslr->remap_addresses, new_remap_key, new_remap_val); + if (err) + dso__put(new_remap_key->dso); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR remapping %m\n"); + free(new_remap_key); + free(new_remap_val); + addr_location__exit(&al); + return 0; + } + addr_location__exit(&al); + return remap_addr; +} + +static int aslr_tool__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap.pid, event->mmap.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap, &event->mmap, event->mmap.header.size); + /* Remaps the mmap.start. */ + new_event->mmap.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap.start, + event->mmap.len, + event->mmap.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap.filename) || is_no_dso_memory(event->mmap.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap.filename, cpumode))) + new_event->mmap.pgoff = new_event->mmap.start; + err = delegate->mmap(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap2(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap2.pid, event->mmap2.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap2, &event->mmap2, event->mmap2.header.size); + /* Remaps the mmap.start. */ + new_event->mmap2.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap2.start, + event->mmap2.len, + event->mmap2.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap2.filename) || is_no_dso_memory(event->mmap2.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap2.filename, cpumode))) + new_event->mmap2.pgoff = new_event->mmap2.start; + err = delegate->mmap2(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_comm(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->comm(delegate, event, sample, machine); +} + +static int aslr_tool__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_fork(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->fork(delegate, event, sample, machine); +} + +static int aslr_tool__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_exit(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->exit(delegate, event, sample, machine); +} + +static int aslr_tool__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + /* Drop in case the instruction encodes an ASLR revealing address. */ + return 0; +} + +static int aslr_tool__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + struct thread *thread; + struct machine *aslr_machine; + bool is_unregister; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, kernel_pid, 0); + if (!thread) + return -ENOMEM; + + is_unregister = (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER); + + memcpy(&new_event->ksymbol, &event->ksymbol, event->ksymbol.header.size); + + if (is_unregister) { + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + } else { + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + } + if (err) { + thread__put(thread); + return err; + } + + err = delegate->ksymbol(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_sample(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct evsel *evsel = sample->evsel; + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + int ret; + int orig_sample_size; + u64 sample_type; + struct thread *thread; + struct machine *aslr_machine; + __u64 max_i; + __u64 max_j; + union perf_event *new_event; + struct perf_sample new_sample; + __u64 *in_array, *out_array; + u8 cpumode; + u64 addr; + size_t i; + size_t j; + struct aslr_evsel_priv *priv = NULL; + u64 orig_sample_type; + u64 orig_regs_user; + u64 orig_regs_intr; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + if (evsel__is_dummy_event(evsel)) + return delegate->sample(delegate, event, sample, machine); + + ret = -EFAULT; + + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + orig_sample_type = priv->orig_sample_type; + orig_regs_user = priv->orig_sample_regs_user; + orig_regs_intr = priv->orig_sample_regs_intr; + } else { + orig_sample_type = evsel->core.attr.sample_type; + orig_regs_user = evsel->core.attr.sample_regs_user; + orig_regs_intr = evsel->core.attr.sample_regs_intr; + } + + orig_sample_size = evsel->sample_size; + + sample_type = orig_sample_type; + sample_type &= ~PERF_SAMPLE_REGS_USER; + sample_type &= ~PERF_SAMPLE_REGS_INTR; + sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + max_i = (event->header.size - sizeof(struct perf_event_header)) / sizeof(__u64); + max_j = (PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_event_header)) / sizeof(__u64); + new_event = (union perf_event *)aslr->event_copy; + cpumode = sample->cpumode; + i = 0; + j = 0; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, sample->pid, sample->tid); + + if (!thread) + return -ENOMEM; + + if (max_i > PERF_SAMPLE_MAX_SIZE / sizeof(u64)) + goto out_put; + + new_event->sample.header = event->sample.header; + + in_array = &event->sample.array[0]; + out_array = &new_event->sample.array[0]; + +#define CHECK_BOUNDS(required_i, required_j) \ + (i + (required_i) > max_i || j + (required_j) > max_j) + +#define COPY_U64() \ + do { \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + out_array[j++] = in_array[i++]; \ + } while (0) + +#define REMAP_U64(addr_field) \ + do { \ + u64 remapped; \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + remapped = aslr_tool__remap_address(aslr, thread, cpumode, addr_field); \ + out_array[j++] = remapped; \ + i++; \ + } while (0) + + if (orig_sample_type & PERF_SAMPLE_IDENTIFIER) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_IP) + REMAP_U64(sample->ip); + if (orig_sample_type & PERF_SAMPLE_TID) { + union { + u64 val64; + u32 val32[2]; + } u; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + u.val32[0] = sample->pid; + u.val32[1] = sample->tid; + out_array[j++] = u.val64; + i++; + } + if (orig_sample_type & PERF_SAMPLE_TIME) + COPY_U64(); /* time */ + if (orig_sample_type & PERF_SAMPLE_ADDR) + REMAP_U64(sample->addr); + if (orig_sample_type & PERF_SAMPLE_ID) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_STREAM_ID) + COPY_U64(); /* stream_id */ + if (orig_sample_type & PERF_SAMPLE_CPU) + COPY_U64(); /* cpu, res */ + if (orig_sample_type & PERF_SAMPLE_PERIOD) + COPY_U64(); /* period */ + if (orig_sample_type & PERF_SAMPLE_READ) { + if ((evsel->core.attr.read_format & PERF_FORMAT_GROUP) == 0) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } else { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + for (u64 cntr = 0; cntr < nr; cntr++) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } + } + } + if (orig_sample_type & PERF_SAMPLE_CALLCHAIN) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + for (u64 cntr = 0; cntr < nr; cntr++) { + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + addr = in_array[i++]; + if (addr >= PERF_CONTEXT_MAX) { + out_array[j++] = addr; + switch (addr) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + case PERF_CONTEXT_GUEST: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_KERNEL: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_USER: + cpumode = PERF_RECORD_MISC_GUEST_USER; + break; + case PERF_CONTEXT_USER_DEFERRED: + if (cntr + 1 >= nr) { + pr_debug("Truncated callchain deferred cookie context\n"); + ret = 0; + goto out_put; + } + /* + * Immediately followed by a 64-bit + * stitching cookie. Skip/Copy it! + */ + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j++] = in_array[i++]; + cntr++; + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %"PRIx64"\n", addr); + ret = 0; + goto out_put; + } + continue; + } + addr = aslr_tool__remap_address(aslr, thread, cpumode, addr); + out_array[j++] = addr; + } + } + if (orig_sample_type & PERF_SAMPLE_RAW) { + size_t bytes = sizeof(u32) + sample->raw_size; + size_t u64_words = (bytes + 7) / 8; + + if (i + u64_words > max_i || j + u64_words > max_j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], bytes); + i += u64_words; + j += u64_words; + /* + * TODO: certain raw samples can be remapped, such as + * tracepoints by examining their fields. + */ + pr_debug("Dropping raw samples as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_BRANCH_STACK) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + COPY_U64(); /* hw_idx */ + + if (nr > (ULLONG_MAX / 3)) { + ret = -EFAULT; + goto out_put; + } + if (nr * 3 > max_i - i || nr * 3 > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) { + u64 from = in_array[i++]; + u64 to = in_array[i++]; + + from = aslr_tool__remap_address(aslr, thread, sample->cpumode, from); + to = aslr_tool__remap_address(aslr, thread, sample->cpumode, to); + + out_array[j++] = from; + out_array[j++] = to; + out_array[j++] = in_array[i++]; /* flags */ + } + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + if (nr > max_i - i || nr > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) + COPY_U64(); + } + } + if (orig_sample_type & PERF_SAMPLE_REGS_USER) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_user); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_STACK_USER) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + size = in_array[i]; + COPY_U64(); + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping stack user sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + COPY_U64(); /* perf_sample_weight */ + if (orig_sample_type & PERF_SAMPLE_DATA_SRC) + COPY_U64(); /* data_src */ + if (orig_sample_type & PERF_SAMPLE_TRANSACTION) + COPY_U64(); /* transaction */ + if (orig_sample_type & PERF_SAMPLE_REGS_INTR) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_intr); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_PHYS_ADDR) { + COPY_U64(); /* phys_addr */ + /* TODO: can this be less conservative? */ + pr_debug("Dropping physical address sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_CGROUP) + COPY_U64(); /* cgroup */ + if (orig_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) + COPY_U64(); /* data_page_size */ + if (orig_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) + COPY_U64(); /* code_page_size */ + + if (orig_sample_type & PERF_SAMPLE_AUX) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j] = in_array[i]; + size = out_array[j++]; + i++; + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping aux sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + if (evsel__is_offcpu_event(evsel)) { + /* TODO: can this be less conservative? */ + pr_debug("Dropping off-CPU sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + new_event->sample.header.size = sizeof(struct perf_event_header) + j * sizeof(u64); + /* Temporarily override evsel attributes to match the stripped new_event format! */ + evsel->sample_size = __evsel__sample_size(sample_type); + evsel->core.attr.sample_type = sample_type; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + perf_sample__init(&new_sample, /*all=*/ true); + ret = __evsel__parse_sample(evsel, new_event, &new_sample, /*needs_swap=*/false); + + if (ret) { + /* Restore original attributes immediately if parsing fails */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + perf_sample__exit(&new_sample); + goto out_put; + } + + new_sample.evsel = evsel; + ret = delegate->sample(delegate, new_event, &new_sample, machine); + perf_sample__exit(&new_sample); + + /* Restore original attributes so trace ingestion never desynchronizes! */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + +out_put: + thread__put(thread); + return ret; +} + +#undef CHECK_BOUNDS +#undef COPY_U64 +#undef REMAP_U64 + +static int skipn(int fd, off_t n) +{ + char buf[4096]; + ssize_t ret; + + while (n > 0) { + ret = read(fd, buf, min_t(off_t, n, (off_t)sizeof(buf))); + if (ret <= 0) + return ret; + n -= ret; + } + + return 0; +} + +static s64 aslr_tool__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + pr_warning_once("ASLR: Dropping auxtrace data as it cannot be obfuscated.\n"); + if (perf_data__is_pipe(session->data)) { + /* Copy behavior of the stub by reading all pipe data. */ + int err = skipn(perf_data__fd(session->data), event->auxtrace.size); + + if (err < 0) + return err; + } + return event->auxtrace.size; +} + +static int aslr_tool__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static int aslr_tool__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist) +{ + u32 attr_size; + + if (!evlist) + return; + + attr_size = event->attr.attr.size ?: PERF_ATTR_SIZE_VER0; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_type) + sizeof(u64))) { + event->attr.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_user) + sizeof(u64))) + event->attr.attr.sample_regs_user = 0; + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_intr) + sizeof(u64))) + event->attr.attr.sample_regs_intr = 0; + } + + if (attr_size >= (offsetof(struct perf_event_attr, type) + sizeof(u32))) { + u32 type = event->attr.attr.type; + + if (type == PERF_TYPE_BREAKPOINT && + attr_size >= (offsetof(struct perf_event_attr, bp_addr) + sizeof(u64))) { + event->attr.attr.bp_addr = 0; + } else if (type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu; + + pmu = perf_pmus__find_by_type(type); + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + if (attr_size >= (offsetof(struct perf_event_attr, config1) + sizeof(u64))) + event->attr.attr.config1 = 0; + if (attr_size >= (offsetof(struct perf_event_attr, config2) + sizeof(u64))) + event->attr.attr.config2 = 0; + } + } + } +} + +static int aslr_tool__init(struct aslr_tool *aslr, struct perf_tool *delegate) +{ + delegate_tool__init(&aslr->tool, delegate); + aslr->tool.tool.ordered_events = true; + + if (machines__init(&aslr->machines)) + return -ENOMEM; + + hashmap__init(&aslr->remap_addresses, + remap_addresses__hash, remap_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->top_addresses, + top_addresses__hash, top_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->evsel_orig_attrs, + evsel_hash, evsel_equal, + /*ctx=*/NULL); + + aslr->tool.tool.sample = aslr_tool__process_sample; + /* read - reads a counter, okay to delegate. */ + aslr->tool.tool.mmap = aslr_tool__process_mmap; + aslr->tool.tool.mmap2 = aslr_tool__process_mmap2; + aslr->tool.tool.comm = aslr_tool__process_comm; + aslr->tool.tool.fork = aslr_tool__process_fork; + aslr->tool.tool.exit = aslr_tool__process_exit; + /* namesspaces, cgroup, lost, lost_sample, aux, */ + /* itrace_start, aux_output_hw_id, context_switch, throttle, unthrottle */ + /* - no virtual addresses. */ + aslr->tool.tool.ksymbol = aslr_tool__process_ksymbol; + /* bpf - no virtual address. */ + aslr->tool.tool.text_poke = aslr_tool__process_text_poke; + /* + * event_update, tracing_data, finished_round, build_id, id_index, + * auxtrace_info, auxtrace_error, time_conv, thread_map, cpu_map, + * stat_config, stat, feature, finished_init, bpf_metadata, compressed, + * auxtrace - no virtual addresses. + */ + aslr->tool.tool.auxtrace = aslr_tool__process_auxtrace; + aslr->tool.tool.auxtrace_info = aslr_tool__process_auxtrace_info; + aslr->tool.tool.auxtrace_error = aslr_tool__process_auxtrace_error; + + return 0; +} + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate) +{ + struct aslr_tool *aslr = zalloc(sizeof(*aslr)); + + if (!aslr) + return NULL; + + if (aslr_tool__init(aslr, delegate)) { + free(aslr); + return NULL; + } + return &aslr->tool.tool; +} + +void aslr_tool__delete(struct perf_tool *tool) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct hashmap_entry *cur; + size_t bkt; + struct rb_node *nd; + + if (!tool) + return; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + + hashmap__for_each_entry(&aslr->remap_addresses, cur, bkt) { + struct remap_addresses_key *key = (struct remap_addresses_key *)cur->pkey; + + if (key) + dso__put(key->dso); + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->top_addresses, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->evsel_orig_attrs, cur, bkt) { + zfree(&cur->pvalue); + } + + hashmap__clear(&aslr->remap_addresses); + hashmap__clear(&aslr->top_addresses); + hashmap__clear(&aslr->evsel_orig_attrs); + aslr_tool__destroy_machines_priv(&aslr->machines); + machines__destroy_kernel_maps(&aslr->machines); + + while ((nd = rb_first_cached(&aslr->machines.guests)) != NULL) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + rb_erase_cached(nd, &aslr->machines.guests); + machine__delete(machine); + } + + machines__exit(&aslr->machines); + free(aslr); +} + +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel) +{ + struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); + struct aslr_tool *aslr = container_of(del_tool, struct aslr_tool, tool); + struct aslr_evsel_priv *priv = zalloc(sizeof(*priv)); + int err; + + if (!priv) + return -ENOMEM; + + priv->orig_sample_type = evsel->core.attr.sample_type; + priv->orig_sample_regs_user = evsel->core.attr.sample_regs_user; + priv->orig_sample_regs_intr = evsel->core.attr.sample_regs_intr; + priv->orig_sample_size = evsel->sample_size; + + err = hashmap__add(&aslr->evsel_orig_attrs, evsel, priv); + if (err) { + free(priv); + return err; + } + return 0; +} + +void aslr_tool__strip_evlist(const struct perf_tool *tool __maybe_unused, struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + evsel->core.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + evsel->sample_size = __evsel__sample_size(evsel->core.attr.sample_type); + evsel__calc_id_pos(evsel); + + if (evsel->core.attr.type == PERF_TYPE_BREAKPOINT) { + evsel->core.attr.bp_addr = 0; + } else if (evsel->core.attr.type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->core.attr.type); + + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + evsel->core.attr.config1 = 0; + evsel->core.attr.config2 = 0; + } + } + } +} + +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist) +{ + const struct delegate_tool *del_tool = container_of(tool, const struct delegate_tool, tool); + const struct aslr_tool *aslr = container_of(del_tool, const struct aslr_tool, tool); + struct evsel *evsel; + struct aslr_evsel_priv *priv; + + evlist__for_each_entry(evlist, evsel) { + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + evsel->core.attr.sample_type = priv->orig_sample_type; + evsel->core.attr.sample_regs_user = priv->orig_sample_regs_user; + evsel->core.attr.sample_regs_intr = priv->orig_sample_regs_intr; + evsel->sample_size = priv->orig_sample_size; + evsel__calc_id_pos(evsel); + } + } +} diff --git a/tools/perf/util/aslr.h b/tools/perf/util/aslr.h new file mode 100644 index 000000000000..522e31c8e2c0 --- /dev/null +++ b/tools/perf/util/aslr.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ASLR_H +#define __PERF_ASLR_H + +#include + +#define ASLR_SUPPORTED_SAMPLE_TYPE ( \ + PERF_SAMPLE_IDENTIFIER | \ + PERF_SAMPLE_IP | \ + PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | \ + PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | \ + PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | \ + PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_READ | \ + PERF_SAMPLE_CALLCHAIN | \ + PERF_SAMPLE_RAW | \ + PERF_SAMPLE_BRANCH_STACK | \ + PERF_SAMPLE_STACK_USER | \ + PERF_SAMPLE_WEIGHT_TYPE | \ + PERF_SAMPLE_DATA_SRC | \ + PERF_SAMPLE_TRANSACTION | \ + PERF_SAMPLE_PHYS_ADDR | \ + PERF_SAMPLE_CGROUP | \ + PERF_SAMPLE_DATA_PAGE_SIZE | \ + PERF_SAMPLE_CODE_PAGE_SIZE | \ + PERF_SAMPLE_AUX) + +struct perf_tool; +struct evsel; +struct evlist; +union perf_event; + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate); +void aslr_tool__delete(struct perf_tool *tool); + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist); +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel); +void aslr_tool__strip_evlist(const struct perf_tool *tool, struct evlist *evlist); +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist); + +#endif /* __PERF_ASLR_H */ diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index a224687ffbc1..0b851f32e98c 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -191,7 +191,7 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); mp->mmap_needed = evsel->needs_auxtrace_mmap; @@ -201,11 +201,11 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, mp->idx = idx; if (per_cpu) { - mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); - mp->tid = perf_thread_map__pid(evlist->core.threads, 0); + mp->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); } else { mp->cpu.cpu = -1; - mp->tid = perf_thread_map__pid(evlist->core.threads, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, idx); } } @@ -372,7 +372,8 @@ static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu) { unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; - return cpu_bitmap && cpu.cpu != -1 && !test_bit(cpu.cpu, cpu_bitmap); + return cpu_bitmap && cpu.cpu >= 0 && cpu.cpu < MAX_NR_CPUS && + !test_bit(cpu.cpu, cpu_bitmap); } static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, @@ -667,10 +668,10 @@ int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) { - bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); if (per_cpu_mmaps) { - struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu); if (cpu_map_idx == -1) @@ -896,6 +897,21 @@ int auxtrace_parse_aux_action(struct evlist *evlist) return 0; } +/** + * auxtrace_record__init - Initialize an AUX area tracing record. + * @evlist: The list of events to check for AUX area tracing event. + * @err: Pointer to an integer to store return code. + * + * This function looks through the @evlist to determine which AUX area + * tracing hardware is being used and initializes the auxtrace_record + * structure. + * + * Return: + * a) A pointer to the struct auxtrace_record with @err = 0 on success. + * b) NULL with @err = 0 if no AUX area tracing event is found/supported + * (not considered an error). + * c) NULL with non-zero @err on actual auxtrace_record__init failure. + */ struct auxtrace_record *__weak auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err) { @@ -1759,7 +1775,7 @@ static const char * const auxtrace_error_type_name[] = { [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", }; -static const char *auxtrace_error_name(int type) +static const char *auxtrace_error_name(unsigned int type) { const char *error_type_name = NULL; @@ -1775,6 +1791,7 @@ size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) struct perf_record_auxtrace_error *e = &event->auxtrace_error; unsigned long long nsecs = e->time; const char *msg = e->msg; + int msg_max; int ret; ret = fprintf(fp, " %s error type %u", @@ -1792,11 +1809,26 @@ size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) if (!e->fmt) msg = (const char *)&e->time; - if (e->fmt >= 2 && e->machine_pid) + /* Bound msg to the bytes actually within the event, capped at the array size */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max < 0) + msg_max = 0; + if (msg_max > (int)sizeof(e->msg)) + msg_max = sizeof(e->msg); + + /* + * Unlike the swap path which downgrades fmt in place, + * native-endian events are mmap'd read-only — check size + * instead to avoid accessing machine_pid/vcpu OOB. + */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu) && + e->machine_pid) ret += fprintf(fp, " machine_pid %d vcpu %d", e->machine_pid, e->vcpu); - ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n", - e->cpu, e->pid, e->tid, e->ip, e->code, msg); + ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %.*s\n", + e->cpu, e->pid, e->tid, e->ip, e->code, msg_max, msg); return ret; } @@ -1806,7 +1838,7 @@ void perf_session__auxtrace_error_inc(struct perf_session *session, struct perf_record_auxtrace_error *e = &event->auxtrace_error; if (e->type < PERF_AUXTRACE_ERROR_MAX) - session->evlist->stats.nr_auxtrace_errors[e->type] += 1; + evlist__stats(session->evlist)->nr_auxtrace_errors[e->type] += 1; } void events_stats__auxtrace_error_warn(const struct events_stats *stats) @@ -2663,7 +2695,7 @@ static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt, { /* Same name, and global or the n'th found or any */ return !arch__compare_symbol_names(name, sym->name) && - ((!idx && sym->binding == STB_GLOBAL) || + ((!idx && symbol__binding(sym) == STB_GLOBAL) || (idx > 0 && ++*cnt == idx) || idx < 0); } @@ -2681,8 +2713,8 @@ static void print_duplicate_syms(struct dso *dso, const char *sym_name) if (dso_sym_match(sym, sym_name, &cnt, -1)) { pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", ++cnt, sym->start, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w', + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', sym->name); near = true; } else if (near) { diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c index 8d3a9a661f26..1135e54f4c7f 100644 --- a/tools/perf/util/block-info.c +++ b/tools/perf/util/block-info.c @@ -472,7 +472,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, int *nr_reps) { struct block_report *block_reports; - int nr_hists = evlist->core.nr_entries, i = 0; + int nr_hists = evlist__nr_entries(evlist), i = 0; struct evsel *pos; block_reports = calloc(nr_hists, sizeof(struct block_report)); @@ -483,7 +483,7 @@ struct block_report *block_info__create_report(struct evlist *evlist, struct hists *hists = evsel__hists(pos); process_block_report(hists, &block_reports[i], total_cycles, - block_hpps, nr_hpps, evlist->nr_br_cntr); + block_hpps, nr_hpps, evlist__nr_br_cntr(evlist)); i++; } diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index a27945c279ef..fa3ebc8ea7f0 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -36,7 +36,7 @@ static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) size_t i; for (i = 0; i < len; i++) - ret += snprintf(buf + ret, size - ret, "%02x", data[i]); + ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); return ret; } @@ -59,6 +59,10 @@ static int machine__process_bpf_event_load(struct machine *machine, return 0; info_linear = info_node->info_linear; + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) + return 0; + for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); u64 addr = addrs[i]; @@ -140,22 +144,26 @@ static int synthesize_bpf_prog_name(char *buf, int size, const struct btf_type *t; int name_len; - name_len = snprintf(buf, size, "bpf_prog_"); + name_len = scnprintf(buf, size, "bpf_prog_"); name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE); - if (btf) { + if (btf && + info->func_info_rec_size >= sizeof(*finfo) && + sub_id < info->nr_func_info) { finfo = func_infos + sub_id * info->func_info_rec_size; t = btf__type_by_id(btf, finfo->type_id); - short_name = btf__name_by_offset(btf, t->name_off); + if (t) + short_name = btf__name_by_offset(btf, t->name_off); } else if (sub_id == 0 && sub_prog_cnt == 1) { /* no subprog */ if (info->name[0]) short_name = info->name; } else short_name = "F"; - if (short_name) - name_len += snprintf(buf + name_len, size - name_len, - "_%s", short_name); + if (short_name) { + name_len += scnprintf(buf + name_len, size - name_len, + "_%s", short_name); + } return name_len; } @@ -365,6 +373,15 @@ static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, event_size = sizeof(metadata->event->bpf_metadata) + nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); + /* + * header.size is __u16. synthesize_perf_record_bpf_metadata() + * adds machine->id_hdr_size (up to ~64 bytes) after this, so + * leave headroom to prevent the final size from wrapping. + */ + if (event_size > UINT16_MAX - 256) { + bpf_metadata_free(metadata); + return NULL; + } metadata->event = zalloc(event_size); if (!metadata->event) { bpf_metadata_free(metadata); @@ -393,8 +410,10 @@ static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) continue; metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); - if (!metadata) + if (!metadata) { + bpf_metadata_free_map_data(&map); continue; + } bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); @@ -869,6 +888,7 @@ static int perf_env__add_bpf_info(struct perf_env *env, u32 id) if (!perf_env__insert_bpf_prog_info(env, info_node)) { pr_debug("%s: duplicate add bpf info request for id %u\n", __func__, btf_id); + bpf_metadata_free(info_node->metadata); free(info_linear); free(info_node); goto out; @@ -943,12 +963,15 @@ int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); } -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp) { - __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); - __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + struct bpf_prog_info *info = &info_linear->info; + __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | + (1UL << PERF_BPIL_JITED_FUNC_LENS); + __u32 *prog_lens; + __u64 *prog_addrs; char name[KSYM_NAME_LEN]; struct btf *btf = NULL; u32 sub_prog_cnt, i; @@ -958,6 +981,13 @@ void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, sub_prog_cnt != info->nr_jited_func_lens) return; + /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ + if ((info_linear->arrays & required_arrays) != required_arrays) + return; + + prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + if (info->btf_id) { struct btf_node *node; diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h index 60d2c6637af5..da4eeb4a1a73 100644 --- a/tools/perf/util/bpf-event.h +++ b/tools/perf/util/bpf-event.h @@ -40,7 +40,7 @@ struct btf_node { int machine__process_bpf(struct machine *machine, union perf_event *event, struct perf_sample *sample); int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp); void bpf_metadata_free(struct bpf_metadata *metadata); @@ -58,7 +58,7 @@ static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, return 0; } -static inline void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info __maybe_unused, +static inline void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear __maybe_unused, struct perf_env *env __maybe_unused, FILE *fp __maybe_unused) { diff --git a/tools/perf/util/bpf-trace-summary.c b/tools/perf/util/bpf-trace-summary.c index cf6e1e4402d5..9a31dbf06cbb 100644 --- a/tools/perf/util/bpf-trace-summary.c +++ b/tools/perf/util/bpf-trace-summary.c @@ -6,7 +6,7 @@ #include #include "dwarf-regs.h" /* for EM_HOST */ -#include "syscalltbl.h" +#include "trace/beauty/syscalltbl.h" #include "util/cgroup.h" #include "util/hashmap.h" #include "util/trace.h" diff --git a/tools/perf/util/bpf-utils.c b/tools/perf/util/bpf-utils.c index d6d2c9c190f7..69148197b1ef 100644 --- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -264,12 +264,28 @@ void bpil_offs_to_addr(struct perf_bpil *info_linear) for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { const struct bpil_array_desc *desc = &bpil_array_desc[i]; __u64 addr, offs; + __u32 count, size; if ((info_linear->arrays & (1UL << i)) == 0) continue; offs = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); + count = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + /* offset and extent from perf.data are untrusted — keep within data[] */ + if (offs >= info_linear->data_len || + (u64)count * size > info_linear->data_len - offs) { + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, 0); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, 0); + /* clear the bit so bpil_addr_to_offs() won't reverse a zeroed address */ + info_linear->arrays &= ~(1UL << i); + continue; + } addr = offs + ptr_to_u64(info_linear->data); bpf_prog_info_set_offset_u64(&info_linear->info, desc->array_offset, addr); diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index 34b6b0da18b7..9362e45e17ce 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -443,7 +443,7 @@ static int bperf_check_target(struct evsel *evsel, } else if (target->tid) { *filter_type = BPERF_FILTER_PID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); - } else if (target->pid || evsel->evlist->workload.pid != -1) { + } else if (target->pid || evlist__workload_pid(evsel->evlist) != -1) { *filter_type = BPERF_FILTER_TGID; *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); } else { diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c index 519fee3dc3d0..4e5f4b9dd442 100644 --- a/tools/perf/util/bpf_counter_cgroup.c +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -104,6 +104,11 @@ static int bperf_load_program(struct evlist *evlist) set_max_rlimit(); + if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) { + pr_err("Invalid cgroup or event count\n"); + return -EINVAL; + } + test_max_events_program_load(); skel = bperf_cgroup_bpf__open(); @@ -111,7 +116,7 @@ static int bperf_load_program(struct evlist *evlist) pr_err("Failed to open cgroup skeleton\n"); return -1; } - setup_rodata(skel, evlist->core.nr_entries); + setup_rodata(skel, evlist__nr_entries(evlist)); err = bperf_cgroup_bpf__load(skel); if (err) { @@ -122,12 +127,12 @@ static int bperf_load_program(struct evlist *evlist) err = -1; cgrp_switch = evsel__new(&cgrp_switch_attr); - if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) { + if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) { pr_err("Failed to open cgroup switches event\n"); goto out; } - perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { + perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) { link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, FD(cgrp_switch, i)); if (IS_ERR(link)) { @@ -186,6 +191,21 @@ static int bperf_load_program(struct evlist *evlist) i++; } + /* + * Propagate supported flag from leaders to followers. Follower events + * are not opened, so their supported flag remains false. + */ + { + struct evsel *leader; + int num_events = evlist__nr_entries(evlist) / nr_cgroups; + + evlist__for_each_entry(evlist, evsel) { + leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); + if (leader) + evsel->supported = leader->supported; + } + } + /* * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check * whether the kernel support it @@ -238,7 +258,7 @@ static int bperf_cgrp__sync_counters(struct evlist *evlist) unsigned int idx; int prog_fd = bpf_program__fd(skel->progs.trigger_read); - perf_cpu_map__for_each_cpu(cpu, idx, evlist->core.all_cpus) + perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus) bperf_trigger_reading(prog_fd, cpu.cpu); return 0; @@ -316,7 +336,7 @@ static int bperf_cgrp__destroy(struct evsel *evsel) return 0; bperf_cgroup_bpf__destroy(skel); - evsel__delete(cgrp_switch); // it'll destroy on_switch progs too + evsel__put(cgrp_switch); // it'll destroy on_switch progs too return 0; } diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c index c456d24efa30..abeafd406e8e 100644 --- a/tools/perf/util/bpf_ftrace.c +++ b/tools/perf/util/bpf_ftrace.c @@ -59,13 +59,13 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) /* don't need to set cpu filter for system-wide mode */ if (ftrace->target.cpu_list) { - ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(ftrace->evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { - ntasks = perf_thread_map__nr(ftrace->evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } @@ -87,7 +87,8 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu( + evlist__core(ftrace->evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -99,7 +100,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(ftrace->evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_kwork.c b/tools/perf/util/bpf_kwork.c index d3a2e548f2b6..2248f462a847 100644 --- a/tools/perf/util/bpf_kwork.c +++ b/tools/perf/util/bpf_kwork.c @@ -273,6 +273,7 @@ static int add_work(struct perf_kwork *kwork, .cpu = key->cpu, }; enum kwork_class_type type = key->type; + int ret = 0; if (!valid_kwork_class_type(type)) { pr_debug("Invalid class type %d to add work\n", type); @@ -287,8 +288,10 @@ static int add_work(struct perf_kwork *kwork, return -1; work = kwork->add_work(kwork, tmp.class, &tmp); - if (work == NULL) - return -1; + if (work == NULL) { + ret = -1; + goto out; + } if (kwork->report == KWORK_REPORT_RUNTIME) { work->nr_atoms = data->nr; @@ -304,13 +307,16 @@ static int add_work(struct perf_kwork *kwork, work->max_latency_end = data->max_time_end; } else { pr_debug("Invalid bpf report type %d\n", kwork->report); - return -1; + ret = -1; + goto out; } kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; - return 0; +out: + work_exit(&tmp); + return ret; } int perf_kwork__report_read_bpf(struct perf_kwork *kwork) diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index cbd7435579fe..c20bd075664e 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -186,6 +186,7 @@ int lock_contention_prepare(struct lock_contention *con) int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1, nslabs = 1; struct evlist *evlist = con->evlist; struct target *target = con->target; + bool has_mmap_lock = false; /* make sure it loads the kernel map before lookup */ map__load(machine__kernel_map(con->machine)); @@ -222,11 +223,11 @@ int lock_contention_prepare(struct lock_contention *con) if (target__has_cpu(target)) { skel->rodata->has_cpu = 1; - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); } if (target__has_task(target)) { skel->rodata->has_task = 1; - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); } if (con->filters->nr_types) { skel->rodata->has_type = 1; @@ -244,6 +245,11 @@ int lock_contention_prepare(struct lock_contention *con) unsigned long *addrs; for (i = 0; i < con->filters->nr_syms; i++) { + if (!strcmp(con->filters->syms[i], "mmap_lock")) { + has_mmap_lock = true; + continue; + } + sym = machine__find_kernel_symbol_by_name(con->machine, con->filters->syms[i], &kmap); @@ -263,7 +269,7 @@ int lock_contention_prepare(struct lock_contention *con) addrs[con->filters->nr_addrs++] = map__unmap_ip(kmap, sym->start); con->filters->addrs = addrs; } - naddrs = con->filters->nr_addrs; + naddrs = con->filters->nr_addrs ?: has_mmap_lock; skel->rodata->has_addr = 1; } @@ -298,6 +304,7 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->aggr_mode = con->aggr_mode; skel->rodata->needs_callstack = con->save_callstack; skel->rodata->lock_owner = con->owner; + skel->rodata->has_mmap_lock = has_mmap_lock; if (con->aggr_mode == LOCK_AGGR_CGROUP || con->filters->nr_cgrps) { if (cgroup_is_v2("perf_event")) @@ -327,7 +334,7 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -339,13 +346,13 @@ int lock_contention_prepare(struct lock_contention *con) fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } - if (target__none(target) && evlist->workload.pid > 0) { - u32 pid = evlist->workload.pid; + if (target__none(target) && evlist__workload_pid(evlist) > 0) { + u32 pid = evlist__workload_pid(evlist); u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); @@ -463,8 +470,8 @@ static void update_lock_stat(int map_fd, int pid, u64 end_ts, stat_key.lock_addr_or_cgroup = ts_data->lock; break; case LOCK_AGGR_CGROUP: - /* TODO */ - return; + stat_key.lock_addr_or_cgroup = ts_data->cgroup_id; + break; default: return; } diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c index a3b699a5322f..c4639f6a5776 100644 --- a/tools/perf/util/bpf_off_cpu.c +++ b/tools/perf/util/bpf_off_cpu.c @@ -1,23 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 -#include "util/bpf_counter.h" -#include "util/debug.h" -#include "util/evsel.h" -#include "util/evlist.h" -#include "util/off_cpu.h" -#include "util/perf-hooks.h" -#include "util/record.h" -#include "util/session.h" -#include "util/target.h" -#include "util/cpumap.h" -#include "util/thread_map.h" -#include "util/cgroup.h" -#include "util/strlist.h" +#include + #include #include #include -#include +#include "bpf_counter.h" #include "bpf_skel/off_cpu.skel.h" +#include "cgroup.h" +#include "cpumap.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "off_cpu.h" +#include "parse-events.h" +#include "perf-hooks.h" +#include "record.h" +#include "session.h" +#include "strlist.h" +#include "target.h" +#include "thread_map.h" #define MAX_STACKS 32 #define MAX_PROC 4096 @@ -71,13 +73,13 @@ static void off_cpu_start(void *arg) /* update task filter for the given workload */ if (skel->rodata->has_task && skel->rodata->uses_tgid && - perf_thread_map__pid(evlist->core.threads, 0) != -1) { + perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) { int fd; u32 pid; u8 val = 1; fd = bpf_map__fd(skel->maps.task_filter); - pid = perf_thread_map__pid(evlist->core.threads, 0); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } @@ -166,7 +168,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, /* don't need to set cpu filter for system-wide mode */ if (target->cpu_list) { - ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); skel->rodata->has_cpu = 1; } @@ -197,7 +199,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, skel->rodata->has_task = 1; skel->rodata->uses_tgid = 1; } else if (target__has_task(target)) { - ntasks = perf_thread_map__nr(evlist->core.threads); + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); bpf_map__set_max_entries(skel->maps.task_filter, ntasks); skel->rodata->has_task = 1; } else if (target__none(target)) { @@ -207,7 +209,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, } if (evlist__first(evlist)->cgrp) { - ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */ + ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */ bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); if (!cgroup_is_v2("perf_event")) @@ -238,7 +240,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.cpu_filter); for (i = 0; i < ncpus; i++) { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); } } @@ -267,7 +269,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target, fd = bpf_map__fd(skel->maps.task_filter); for (i = 0; i < ntasks; i++) { - pid = perf_thread_map__pid(evlist->core.threads, i); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } } diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 96e7d853b9ed..0d9c6f55050e 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -175,6 +175,13 @@ struct mm_struct___new { struct rw_semaphore mmap_lock; } __attribute__((preserve_access_index)); +struct cas_ctx { + struct contention_data *data; + u64 duration; + int max_done; + int min_done; +}; + extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym __weak; /* control flags */ @@ -184,6 +191,7 @@ const volatile int has_type; const volatile int has_addr; const volatile int has_cgroup; const volatile int has_slab; +const volatile int has_mmap_lock; const volatile int needs_callstack; const volatile int stack_skip; const volatile int lock_owner; @@ -214,6 +222,8 @@ int data_map_full; struct task_struct *bpf_task_from_pid(s32 pid) __ksym __weak; void bpf_task_release(struct task_struct *p) __ksym __weak; +static inline __u32 check_lock_type(__u64 lock, __u32 flags); + static inline __u64 get_current_cgroup_id(void) { struct task_struct *task; @@ -239,6 +249,8 @@ static inline __u64 get_current_cgroup_id(void) static inline int can_record(u64 *ctx) { + bool is_addr_ok = false; + if (has_cpu) { __u32 cpu = bpf_get_smp_processor_id(); __u8 *ok; @@ -271,8 +283,10 @@ static inline int can_record(u64 *ctx) __u64 addr = ctx[0]; ok = bpf_map_lookup_elem(&addr_filter, &addr); - if (!ok && !has_slab) + if (!ok && !has_slab && !has_mmap_lock) return 0; + + is_addr_ok = !!ok; } if (has_cgroup) { @@ -284,6 +298,10 @@ static inline int can_record(u64 *ctx) return 0; } + if (is_addr_ok) + return 1; + + /* slab and mmap_lock are part of the addr_filter */ if (has_slab && bpf_get_kmem_cache) { __u8 *ok; __u64 addr = ctx[0]; @@ -291,7 +309,17 @@ static inline int can_record(u64 *ctx) kmem_cache_addr = (long)bpf_get_kmem_cache(addr); ok = bpf_map_lookup_elem(&slab_filter, &kmem_cache_addr); - if (!ok) + if (ok) + return 1; + else if (!has_mmap_lock) + return 0; + } + + if (has_mmap_lock) { + __u64 lock = ctx[0]; + __u32 flag = ctx[1]; + + if (check_lock_type(lock, flag) != LCD_F_MMAP_LOCK) return 0; } @@ -486,16 +514,49 @@ static inline s32 get_owner_stack_id(u64 *stacktrace) return -1; } +static long cas_min_max_cb(u64 idx, void *arg) +{ + struct cas_ctx *ctx = arg; + + if (!ctx->max_done) { + u64 old_max = ctx->data->max_time; + if (old_max >= ctx->duration) { + ctx->max_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->max_time, old_max, ctx->duration); + if (r == old_max) + ctx->max_done = 1; + } + } + + if (!ctx->min_done) { + u64 old_min = ctx->data->min_time; + if (old_min <= ctx->duration) { + ctx->min_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->min_time, old_min, ctx->duration); + if (r == old_min) + ctx->min_done = 1; + } + } + + return (ctx->max_done && ctx->min_done) ? 1 : 0; +} + static inline void update_contention_data(struct contention_data *data, u64 duration, u32 count) { __sync_fetch_and_add(&data->total_time, duration); __sync_fetch_and_add(&data->count, count); - /* FIXME: need atomic operations */ - if (data->max_time < duration) - data->max_time = duration; - if (data->min_time > duration) - data->min_time = duration; + struct cas_ctx ctx = { + .data = data, + .duration = duration, + .max_done = 0, + .min_done = 0, + }; + bpf_loop(64, cas_min_max_cb, &ctx, 0); } static inline void update_owner_stat(u32 id, u64 duration, u32 flags) @@ -536,6 +597,8 @@ int contention_begin(u64 *ctx) pelem->timestamp = bpf_ktime_get_ns(); pelem->lock = (__u64)ctx[0]; pelem->flags = (__u32)ctx[1]; + if (aggr_mode == LOCK_AGGR_CGROUP) + pelem->cgroup_id = get_current_cgroup_id(); if (needs_callstack) { u32 i = 0; @@ -771,7 +834,7 @@ int contention_end(u64 *ctx) key.stack_id = pelem->stack_id; break; case LOCK_AGGR_CGROUP: - key.lock_addr_or_cgroup = get_current_cgroup_id(); + key.lock_addr_or_cgroup = pelem->cgroup_id; break; default: /* should not happen */ diff --git a/tools/perf/util/bpf_skel/lock_data.h b/tools/perf/util/bpf_skel/lock_data.h index 28c5e5aced7f..652e114e6b87 100644 --- a/tools/perf/util/bpf_skel/lock_data.h +++ b/tools/perf/util/bpf_skel/lock_data.h @@ -13,6 +13,7 @@ struct owner_tracing_data { struct tstamp_data { u64 timestamp; u64 lock; + u64 cgroup_id; u32 flags; s32 stack_id; }; diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c index 9e706f0fa53d..a9cf2a77ded1 100644 --- a/tools/perf/util/bpf_trace_augment.c +++ b/tools/perf/util/bpf_trace_augment.c @@ -1,11 +1,11 @@ #include #include -#include "util/debug.h" -#include "util/evlist.h" -#include "util/trace_augment.h" - #include "bpf_skel/augmented_raw_syscalls.skel.h" +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "trace_augment.h" static struct augmented_raw_syscalls_bpf *skel; static struct evsel *bpf_output; diff --git a/tools/perf/util/btf.c b/tools/perf/util/btf.c index bb163fe87767..50d98f3e83bf 100644 --- a/tools/perf/util/btf.c +++ b/tools/perf/util/btf.c @@ -14,7 +14,7 @@ const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, { const struct btf_type *t = btf__type_by_id(btf, type_id); const struct btf_member *m; - int i; + u32 i; for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { const char *current_member_name = btf__name_by_offset(btf, m->name_off); diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c index fdb35133fde4..eb95ab90f974 100644 --- a/tools/perf/util/build-id.c +++ b/tools/perf/util/build-id.c @@ -10,6 +10,7 @@ #include "util.h" // lsdir(), mkdir_p(), rm_rf() #include #include +#include #include #include #include @@ -55,7 +56,6 @@ static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct addr_location al; @@ -63,8 +63,8 @@ int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, sample->tid); if (thread == NULL) { - pr_err("problem processing %d event, skipping it.\n", - event->header.type); + pr_err("problem processing %s event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), sample->file_offset); return -1; } @@ -74,7 +74,7 @@ int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, addr_location__exit(&al); - sample__for_each_callchain_node(thread, evsel, sample, PERF_MAX_STACK_DEPTH, + sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, /*symbols=*/false, mark_dso_hit_callback, /*data=*/NULL); @@ -93,8 +93,11 @@ int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size return 0; } - for (size_t i = 0; i < build_id->size && offs < bf_size; ++i) - offs += snprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); + if (bf_size > 0) + bf[0] = '\0'; + + for (size_t i = 0; i < build_id->size && offs + 1 < bf_size; ++i) + offs += scnprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); return offs; } diff --git a/tools/perf/util/build-id.h b/tools/perf/util/build-id.h index 47e621cebe1b..73bad90b06f9 100644 --- a/tools/perf/util/build-id.h +++ b/tools/perf/util/build-id.h @@ -34,12 +34,7 @@ char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, bool is_debug, bool is_kallsyms); int build_id__mark_dso_hit(const struct perf_tool *tool, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, - struct machine *machine); - -int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, - struct machine *machine); + struct perf_sample *sample, struct machine *machine); bool perf_session__read_build_ids(struct perf_session *session, bool with_hits); int perf_session__write_buildid_table(struct perf_session *session, diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index f031cbbeeba8..31c675cbab63 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -801,7 +801,7 @@ static enum match_result match_chain(struct callchain_cursor_node *node, * symbol start. Otherwise do a faster comparison based * on the symbol start address. */ - if (cnode->ms.sym->inlined || node->ms.sym->inlined) { + if (symbol__inlined(cnode->ms.sym) || symbol__inlined(node->ms.sym)) { match = match_chain_strings(cnode->ms.sym->name, node->ms.sym->name); if (match != MATCH_ERROR) @@ -1170,7 +1170,7 @@ int callchain_cursor_append(struct callchain_cursor *cursor, int sample__resolve_callchain(struct perf_sample *sample, struct callchain_cursor *cursor, struct symbol **parent, - struct evsel *evsel, struct addr_location *al, + struct addr_location *al, int max_stack) { if (sample->callchain == NULL && !symbol_conf.show_branchflag_count) @@ -1178,7 +1178,7 @@ int sample__resolve_callchain(struct perf_sample *sample, if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain || perf_hpp_list.parent || symbol_conf.show_branchflag_count) { - return thread__resolve_callchain(al->thread, cursor, evsel, sample, + return thread__resolve_callchain(al->thread, cursor, sample, parent, al, max_stack); } return 0; @@ -1245,7 +1245,7 @@ char *callchain_list__sym_name(struct callchain_list *cl, int printed; if (cl->ms.sym) { - const char *inlined = cl->ms.sym->inlined ? " (inlined)" : ""; + const char *inlined = symbol__inlined(cl->ms.sym) ? " (inlined)" : ""; if (show_srcline && cl->srcline) printed = scnprintf(bf, bfsize, "%s %s%s", @@ -1578,6 +1578,21 @@ void free_callchain(struct callchain_root *root) free_callchain_node(&root->node); } +void callchain_cursor_cleanup(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + + for (node = cursor->first; node; node = next) { + next = node->next; + free(node); + } + cursor->first = NULL; + cursor->last = &cursor->first; + cursor->curr = NULL; +} + static u64 decay_callchain_node(struct callchain_node *node) { struct callchain_node *child; @@ -1853,7 +1868,7 @@ s64 callchain_avg_cycles(struct callchain_node *cnode) return cycles; } -int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, +int sample__for_each_callchain_node(struct thread *thread, struct perf_sample *sample, int max_stack, bool symbols, callchain_iter_fn cb, void *data) { @@ -1864,7 +1879,7 @@ int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, return -ENOMEM; /* Fill in the callchain. */ - ret = __thread__resolve_callchain(thread, cursor, evsel, sample, + ret = __thread__resolve_callchain(thread, cursor, sample, /*parent=*/NULL, /*root_al=*/NULL, max_stack, symbols); if (ret) diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index 06d463ccc7a0..8b1e405d1cdf 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -8,11 +8,9 @@ #include "branch.h" struct addr_location; -struct evsel; struct hist_entry; struct hists; struct ip_callchain; -struct map; struct perf_sample; struct record_opts; struct thread; @@ -245,7 +243,7 @@ int record_opts__parse_callchain(struct record_opts *record, int sample__resolve_callchain(struct perf_sample *sample, struct callchain_cursor *cursor, struct symbol **parent, - struct evsel *evsel, struct addr_location *al, + struct addr_location *al, int max_stack); int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample); int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, @@ -277,8 +275,6 @@ static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused, } #endif -void arch__add_leaf_frame_record_opts(struct record_opts *opts); - char *callchain_list__sym_name(struct callchain_list *cl, char *bf, size_t bfsize, bool show_dso); char *callchain_node__scnprintf_value(struct callchain_node *node, @@ -290,6 +286,7 @@ int callchain_list_counts__printf_value(struct callchain_list *clist, FILE *fp, char *bf, int bfsize); void free_callchain(struct callchain_root *root); +void callchain_cursor_cleanup(struct callchain_cursor *cursor); void decay_callchain(struct callchain_root *root); int callchain_node__make_parent_list(struct callchain_node *node); @@ -308,7 +305,7 @@ s64 callchain_avg_cycles(struct callchain_node *cnode); typedef int (*callchain_iter_fn)(struct callchain_cursor_node *node, void *data); -int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, +int sample__for_each_callchain_node(struct thread *thread, struct perf_sample *sample, int max_stack, bool symbols, callchain_iter_fn cb, void *data); diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index 25cf6e15ec27..5ad537fea436 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -1,7 +1,19 @@ // SPDX-License-Identifier: GPL-2.0 #include "capstone.h" -#include "annotate.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include + #include "addr_location.h" +#include "annotate.h" #include "debug.h" #include "disasm.h" #include "dso.h" @@ -11,13 +23,6 @@ #include "print_insn.h" #include "symbol.h" #include "thread.h" -#include -#include -#include -#include -#include - -#include #ifdef LIBCAPSTONE_DLOPEN static void *perf_cs_dll_handle(void) @@ -137,37 +142,70 @@ static enum cs_err perf_cs_close(csh *handle) #endif } -static int capstone_init(struct machine *machine, csh *cs_handle, bool is64, +static bool e_machine_to_capstone(uint16_t e_machine, bool is64, bool is_big_endian, + enum cs_arch *arch, enum cs_mode *mode) +{ + *mode = is_big_endian ? CS_MODE_BIG_ENDIAN : CS_MODE_LITTLE_ENDIAN; + + switch (e_machine) { + case EM_X86_64: + case EM_386: + *arch = CS_ARCH_X86; + *mode |= is64 ? CS_MODE_64 : CS_MODE_32; + return true; + case EM_AARCH64: + *arch = CS_ARCH_ARM64; + *mode |= CS_MODE_ARM; + return true; + case EM_ARM: + *arch = CS_ARCH_ARM; + *mode |= CS_MODE_ARM | CS_MODE_V8; + return true; + case EM_S390: + *arch = CS_ARCH_SYSZ; + return true; + case EM_MIPS: + *arch = CS_ARCH_MIPS; + *mode |= is64 ? CS_MODE_MIPS64 : CS_MODE_MIPS32; + return true; + case EM_PPC: + *arch = CS_ARCH_PPC; + return true; + case EM_PPC64: + *arch = CS_ARCH_PPC; + *mode |= CS_MODE_64; + return true; + case EM_SPARC: + *arch = CS_ARCH_SPARC; + return true; + case EM_SPARCV9: + *arch = CS_ARCH_SPARC; + *mode |= CS_MODE_V9; + return true; + case EM_RISCV: + *arch = CS_ARCH_RISCV; + *mode |= (is64 ? CS_MODE_RISCV64 : CS_MODE_RISCV32) | CS_MODE_RISCVC; + return true; + default: + return false; + } +} + +static int capstone_init(uint16_t e_machine, csh *cs_handle, bool is64, bool is_big_endian, bool disassembler_style) { enum cs_arch arch; enum cs_mode mode; - if (machine__is(machine, "x86_64") && is64) { - arch = CS_ARCH_X86; - mode = CS_MODE_64; - } else if (machine__normalized_is(machine, "x86")) { - arch = CS_ARCH_X86; - mode = CS_MODE_32; - } else if (machine__normalized_is(machine, "arm64")) { - arch = CS_ARCH_ARM64; - mode = CS_MODE_ARM; - } else if (machine__normalized_is(machine, "arm")) { - arch = CS_ARCH_ARM; - mode = CS_MODE_ARM + CS_MODE_V8; - } else if (machine__normalized_is(machine, "s390")) { - arch = CS_ARCH_SYSZ; - mode = CS_MODE_BIG_ENDIAN; - } else { + if (!e_machine_to_capstone(e_machine, is64, is_big_endian, &arch, &mode)) return -1; - } if (perf_cs_open(arch, mode, cs_handle) != CS_ERR_OK) { pr_warning_once("cs_open failed\n"); return -1; } - if (machine__normalized_is(machine, "x86")) { + if (arch == CS_ARCH_X86) { /* * In case of using capstone_init while symbol__disassemble * setting CS_OPT_SYNTAX_ATT depends if disassembler_style opts @@ -211,29 +249,28 @@ static size_t print_insn_x86(struct thread *thread, u8 cpumode, struct cs_insn * return printed; } - -ssize_t capstone__fprintf_insn_asm(struct machine *machine __maybe_unused, - struct thread *thread __maybe_unused, - u8 cpumode __maybe_unused, bool is64bit __maybe_unused, - const uint8_t *code __maybe_unused, - size_t code_size __maybe_unused, - uint64_t ip __maybe_unused, int *lenp __maybe_unused, - int print_opts __maybe_unused, FILE *fp __maybe_unused) +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, uint64_t ip, + int *lenp, int print_opts, FILE *fp) { size_t printed; struct cs_insn *insn; csh cs_handle; size_t count; + bool is_big_endian = false; + uint16_t e_machine = thread__e_machine_endian(thread, machine, + /*e_flags=*/NULL, &is_big_endian); int ret; /* TODO: Try to initiate capstone only once but need a proper place. */ - ret = capstone_init(machine, &cs_handle, is64bit, true); + ret = capstone_init(e_machine, &cs_handle, is64bit, is_big_endian, + /*disassembler_style=*/true); if (ret < 0) return ret; count = perf_cs_disasm(cs_handle, code, code_size, ip, 1, &insn); if (count > 0) { - if (machine__normalized_is(machine, "x86")) + if (e_machine == EM_X86_64 || e_machine == EM_386) printed = print_insn_x86(thread, cpumode, &insn[0], print_opts, fp); else printed = fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); @@ -322,9 +359,8 @@ static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) return 0; } -int symbol__disassemble_capstone(const char *filename __maybe_unused, - struct symbol *sym __maybe_unused, - struct annotate_args *args __maybe_unused) +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args) { struct annotation *notes = symbol__annotation(sym); struct map *map = args->ms->map; @@ -344,6 +380,8 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, char disasm_buf[512]; struct disasm_line *dl; bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; if (args->options->objdump_path) return -1; @@ -373,8 +411,10 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(thread__maps(args->ms->thread)), &handle, is_64bit, - disassembler_style) < 0) + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -466,6 +506,8 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, struct disasm_line *dl; u32 *line; bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; if (args->options->objdump_path) return -1; @@ -484,8 +526,10 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(thread__maps(args->ms->thread)), &handle, is_64bit, - disassembler_style) < 0) + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; needs_cs_close = true; diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index 1b5664d1481f..c7be16a7915e 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -367,7 +367,7 @@ int parse_cgroups(const struct option *opt, const char *str, char *s; int ret, i; - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "must define events before cgroups\n"); return -1; } @@ -423,7 +423,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro int ret = -1; int prefix_len; - if (evlist->core.nr_entries == 0) { + if (evlist__nr_entries(evlist) == 0) { fprintf(stderr, "must define events before cgroups\n"); return -EINVAL; } @@ -436,11 +436,11 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro } /* save original events and init evlist */ - evlist__splice_list_tail(orig_list, &evlist->core.entries); - evlist->core.nr_entries = 0; + evlist__splice_list_tail(orig_list, &evlist__core(evlist)->entries); + evlist__core(evlist)->nr_entries = 0; - orig_metric_events = evlist->metric_events; - metricgroup__rblist_init(&evlist->metric_events); + orig_metric_events = *evlist__metric_events(evlist); + metricgroup__rblist_init(evlist__metric_events(evlist)); if (has_pattern_string(str)) prefix_len = match_cgroups(str); @@ -469,7 +469,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro /* copy the list and set to the new cgroup. */ evlist__for_each_entry(orig_list, pos) { - struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos); + struct evsel *evsel = evsel__clone(pos); if (evsel == NULL) goto out_err; @@ -503,15 +503,15 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro nr_cgroups++; if (metricgroup__copy_metric_events(tmp_list, cgrp, - &evlist->metric_events, + evlist__metric_events(evlist), &orig_metric_events) < 0) goto out_err; - evlist__splice_list_tail(evlist, &tmp_list->core.entries); - tmp_list->core.nr_entries = 0; + evlist__splice_list_tail(evlist, &evlist__core(tmp_list)->entries); + evlist__core(tmp_list)->nr_entries = 0; } - if (list_empty(&evlist->core.entries)) { + if (list_empty(&evlist__core(evlist)->entries)) { fprintf(stderr, "no cgroup matched: %s\n", str); goto out_err; } @@ -520,8 +520,8 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro cgrp_event_expanded = true; out_err: - evlist__delete(orig_list); - evlist__delete(tmp_list); + evlist__put(orig_list); + evlist__put(tmp_list); metricgroup__rblist_exit(&orig_metric_events); release_cgroup_list(); diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c index 087002fb1b9b..7988149dc7ed 100644 --- a/tools/perf/util/config.c +++ b/tools/perf/util/config.c @@ -23,6 +23,7 @@ #include "build-id.h" #include "debug.h" #include "config.h" +#include "unwind.h" #include #include #include @@ -525,6 +526,9 @@ int perf_default_config(const char *var, const char *value, if (strstarts(var, "addr2line.")) return addr2line_configure(var, value, dummy); + if (strstarts(var, "unwind.")) + return unwind__configure(var, value, dummy); + /* Add other config variables here. */ return 0; } diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c index 11922e1ded84..23ebe9b97f8e 100644 --- a/tools/perf/util/cpumap.c +++ b/tools/perf/util/cpumap.c @@ -10,6 +10,7 @@ #include #include "asm/bug.h" +#include #include #include #include @@ -40,15 +41,16 @@ bool perf_record_cpu_map_data__test_bit(int i, /* Read ith mask value from data into the given 64-bit sized bitmap */ static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, - int i, unsigned long *bitmap) + int i, unsigned long *bitmap, + u16 long_size) { #if __SIZEOF_LONG__ == 8 - if (data->mask32_data.long_size == 4) + if (long_size == 4) bitmap[0] = data->mask32_data.mask[i]; else bitmap[0] = data->mask64_data.mask[i]; #else - if (data->mask32_data.long_size == 4) { + if (long_size == 4) { bitmap[0] = data->mask32_data.mask[i]; bitmap[1] = 0; } else { @@ -64,24 +66,27 @@ static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu } static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) { + /* Snapshot nr — data is mmap'd and could change between reads */ + u16 nr = READ_ONCE(data->cpus_data.nr); struct perf_cpu_map *map; - map = perf_cpu_map__empty_new(data->cpus_data.nr); + map = perf_cpu_map__empty_new(nr); if (!map) return NULL; - for (unsigned int i = 0; i < data->cpus_data.nr; i++) { + for (unsigned int i = 0; i < nr; i++) { + u16 cpu = READ_ONCE(data->cpus_data.cpu[i]); /* * Special treatment for -1, which is not real cpu number, * and we need to use (int) -1 to initialize map[i], * otherwise it would become 65535. */ - if (data->cpus_data.cpu[i] == (u16) -1) { + if (cpu == (u16) -1) { RC_CHK_ACCESS(map)->map[i].cpu = -1; - } else if (data->cpus_data.cpu[i] < INT16_MAX) { - RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) data->cpus_data.cpu[i]; + } else if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) cpu; } else { - pr_err("Invalid cpumap entry %u\n", data->cpus_data.cpu[i]); + pr_err("Invalid cpumap entry %u\n", cpu); perf_cpu_map__put(map); return NULL; } @@ -93,11 +98,21 @@ static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_m static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) { DECLARE_BITMAP(local_copy, 64); - int weight = 0, mask_nr = data->mask32_data.nr; + int weight = 0, mask_nr; + /* Snapshot before validation — data is mmap'd and could change */ + u16 long_size = READ_ONCE(data->mask32_data.long_size); struct perf_cpu_map *map; + /* long_size must be 4 or 8; other values overflow cpus_per_i below */ + if (long_size != 4 && long_size != 8) { + pr_warning("WARNING: cpu_map mask: unsupported long_size %u\n", long_size); + return NULL; + } + + mask_nr = READ_ONCE(data->mask32_data.nr); + for (int i = 0; i < mask_nr; i++) { - perf_record_cpu_map_data__read_one_mask(data, i, local_copy); + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); weight += bitmap_weight(local_copy, 64); } @@ -106,11 +121,14 @@ static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_ return NULL; for (int i = 0, j = 0; i < mask_nr; i++) { - int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); + int cpus_per_i = (i * long_size * BITS_PER_BYTE); int cpu; - perf_record_cpu_map_data__read_one_mask(data, i, local_copy); + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); for_each_set_bit(cpu, local_copy, 64) { + /* Guard against more set bits than the first pass counted */ + if (j >= weight) + break; if (cpu + cpus_per_i < INT16_MAX) { RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; } else { @@ -126,18 +144,28 @@ static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_ static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) { + /* Snapshot fields — data is mmap'd and could change between reads */ + u16 start_cpu = READ_ONCE(data->range_cpu_data.start_cpu); + u16 end_cpu = READ_ONCE(data->range_cpu_data.end_cpu); + u16 any_cpu = READ_ONCE(data->range_cpu_data.any_cpu); struct perf_cpu_map *map; unsigned int i = 0; - map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - - data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); + if (end_cpu < start_cpu) { + pr_warning("WARNING: cpu_map range: end_cpu %u < start_cpu %u\n", + end_cpu, start_cpu); + return NULL; + } + + /* any_cpu is boolean (0 or 1), not a count — clamp to avoid inflated nr */ + map = perf_cpu_map__empty_new(end_cpu - start_cpu + 1 + !!any_cpu); if (!map) return NULL; - if (data->range_cpu_data.any_cpu) + if (any_cpu) RC_CHK_ACCESS(map)->map[i++].cpu = -1; - for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; + for (int cpu = start_cpu; cpu <= end_cpu; i++, cpu++) { if (cpu < INT16_MAX) { RC_CHK_ACCESS(map)->map[i].cpu = cpu; @@ -420,6 +448,12 @@ static int get_max_num(char *path, int *max) buf[num] = '\0'; + /* empty file — nothing to parse */ + if (num == 0) { + err = -1; + goto out; + } + /* start on the right, to find highest node num */ while (--num) { if ((buf[num] == ',') || (buf[num] == '-')) { @@ -466,6 +500,16 @@ static void set_max_cpu_num(void) if (ret) goto out; + /* + * struct perf_cpu.cpu is int16_t (libperf ABI) — clamp to avoid + * truncation to negative. See tools/lib/perf/TODO for the ABI + * widening plan. + */ + if (max > INT16_MAX) { + pr_warning("WARNING: max possible cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } max_cpu_num.cpu = max; /* get the highest present cpu number for a sparse allocation */ @@ -478,11 +522,12 @@ static void set_max_cpu_num(void) ret = get_max_num(path, &max); if (!ret && max > INT16_MAX) { - pr_err("Read out of bounds max cpus of %d\n", max); - ret = -1; + pr_warning("WARNING: max present cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; } if (!ret) - max_present_cpu_num.cpu = (int16_t)max; + max_present_cpu_num.cpu = max; out: if (ret) pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); @@ -548,6 +593,10 @@ int cpu__get_node(struct perf_cpu cpu) return -1; } + /* cpunode_map allocated for max_cpu_num entries; input may be untrusted */ + if (cpu.cpu < 0 || cpu.cpu >= max_cpu_num.cpu) + return -1; + return cpunode_map[cpu.cpu]; } @@ -615,7 +664,9 @@ int cpu__setup_cpunode_map(void) while ((dent2 = readdir(dir2)) != NULL) { if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) continue; - cpunode_map[cpu] = mem; + /* cpunode_map allocated for max_cpu_num entries */ + if (cpu < (unsigned int)max_cpu_num.cpu) + cpunode_map[cpu] = mem; } closedir(dir2); } @@ -641,21 +692,21 @@ size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) if (start == -1) { start = i; if (last) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, i).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, i).cpu); } } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { int end = i - 1; if (start == end) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, start).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, start).cpu); } else { - ret += snprintf(buf + ret, size - ret, - "%s%d-%d", COMMA, - perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d-%d", COMMA, + perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); } first = false; start = i; diff --git a/tools/perf/util/cs-etm-base.c b/tools/perf/util/cs-etm-base.c index 4abe416e3feb..aebef71d3a0a 100644 --- a/tools/perf/util/cs-etm-base.c +++ b/tools/perf/util/cs-etm-base.c @@ -170,7 +170,9 @@ int cs_etm__process_auxtrace_info(union perf_event *event, u64 *ptr = NULL; u64 hdr_version; - if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE)) + /* Ensure priv[] is large enough for the global header entries */ + if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE + + CS_ETM_HEADER_SIZE)) return -EINVAL; /* First the global part */ diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index dee3020ceaa9..26940f1f1b0b 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -402,6 +402,8 @@ cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq, packet_queue->packet_buffer[et].flags = 0; packet_queue->packet_buffer[et].exception_number = UINT32_MAX; packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id; + packet_queue->packet_buffer[et].el = ocsd_EL_unknown; + packet_queue->packet_buffer[et].tid = -1; if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1) return OCSD_RESP_WAIT; @@ -449,6 +451,7 @@ cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, packet->last_instr_type = elem->last_i_type; packet->last_instr_subtype = elem->last_i_subtype; packet->last_instr_cond = elem->last_instr_cond; + packet->el = elem->context.exception_level; if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) packet->last_instr_taken_branch = elem->last_instr_exec; @@ -525,7 +528,9 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, const ocsd_generic_trace_elem *elem, const uint8_t trace_chan_id) { + struct cs_etm_packet *packet; pid_t tid = -1; + int ret; /* * Process the PE_CONTEXT packets if we have a valid contextID or VMID. @@ -546,12 +551,18 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, break; } - if (cs_etm__etmq_set_tid_el(etmq, tid, trace_chan_id, - elem->context.exception_level)) + if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id, + elem->context.exception_level, tid)) return OCSD_RESP_FATAL_SYS_ERR; - if (tid == -1) - return OCSD_RESP_CONT; + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_CONTEXT); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + packet->tid = tid; + packet->el = elem->context.exception_level; /* * A timestamp is generated after a PE_CONTEXT element so make sure @@ -559,7 +570,7 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, */ cs_etm_decoder__reset_timestamp(packet_queue); - return OCSD_RESP_CONT; + return ret; } static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 8a639d2e51a4..95e3ec1171ac 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -6,6 +6,7 @@ * Author: Mathieu Poirier */ +#include #include #include #include @@ -17,6 +18,7 @@ #include #include "auxtrace.h" +#include "callchain.h" #include "color.h" #include "cs-etm.h" #include "cs-etm-decoder/cs-etm-decoder.h" @@ -66,10 +68,13 @@ struct cs_etm_auxtrace { bool snapshot_mode; bool data_queued; bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */ + bool use_thread_stack; + bool use_callchain; int num_cpu; u64 latest_kernel_timestamp; u32 auxtrace_type; + u32 branches_filter; u64 branches_sample_type; u64 branches_id; u64 instructions_sample_type; @@ -83,17 +88,25 @@ struct cs_etm_auxtrace { struct cs_etm_traceid_queue { u8 trace_chan_id; u64 period_instructions; - size_t last_branch_pos; + u64 kernel_start; union perf_event *event_buf; - struct thread *thread; - struct thread *prev_packet_thread; - ocsd_ex_level prev_packet_el; - ocsd_ex_level el; + unsigned int br_stack_sz; struct branch_stack *last_branch; - struct branch_stack *last_branch_rb; + struct ip_callchain *callchain; struct cs_etm_packet *prev_packet; struct cs_etm_packet *packet; struct cs_etm_packet_queue packet_queue; + + struct thread *decode_thread; + ocsd_ex_level decode_el; + + /* + * The frontend accesses the EL from '[prev_]packet' because it needs + * previous EL for branch and current EL for instruction samples. It's + * not possible to change thread in a single branch sample so no need to + * store or access the thread through the packet. + */ + struct thread *frontend_thread; }; enum cs_etm_format { @@ -284,8 +297,11 @@ static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int c { if (etm->per_thread_decoding) return etm->queues.queue_array[0].priv; - else - return etm->queues.queue_array[cpu].priv; + + if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) + return NULL; + + return etm->queues.queue_array[cpu].priv; } static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, @@ -298,6 +314,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id * queue associated with that CPU so only one decoder is made. */ etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); + if (!etmq) + return -EINVAL; + if (etmq->format == UNFORMATTED) return cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); @@ -310,6 +329,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id int ret; etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); if (ret) @@ -350,6 +372,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + if (!etmq) + return -EINVAL; + /* * Check sink id hasn't changed in per-cpu mode. In per-thread mode, * let it pass for now until an actual overlapping trace ID is hit. In @@ -367,6 +392,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; + if (!other_etmq) + continue; + /* Different sinks, skip */ if (other_etmq->sink_id != etmq->sink_id) continue; @@ -388,6 +416,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, } cpu_data = get_cpu_data(etm, cpu); + if (!cpu_data) + return -EINVAL; + ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); if (ret) return ret; @@ -614,10 +645,13 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; tidq->trace_chan_id = trace_chan_id; - tidq->el = tidq->prev_packet_el = ocsd_EL_unknown; - tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1, + tidq->decode_el = ocsd_EL_unknown; + tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); - tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host); + if (!tidq->frontend_thread || !tidq->decode_thread) + goto out; tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -627,7 +661,7 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, if (!tidq->prev_packet) goto out_free; - if (etm->synth_opts.last_branch) { + if (etm->use_thread_stack) { size_t sz = sizeof(struct branch_stack); sz += etm->synth_opts.last_branch_sz * @@ -635,8 +669,16 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, tidq->last_branch = zalloc(sz); if (!tidq->last_branch) goto out_free; - tidq->last_branch_rb = zalloc(sz); - if (!tidq->last_branch_rb) + + tidq->br_stack_sz = etm->synth_opts.last_branch_sz; + } + + if (etm->synth_opts.callchain) { + /* Add 1 to callchain_sz for callchain context */ + tidq->callchain = + zalloc(struct_size(tidq->callchain, ips, + etm->synth_opts.callchain_sz + 1)); + if (!tidq->callchain) goto out_free; } @@ -647,11 +689,13 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, return 0; out_free: - zfree(&tidq->last_branch_rb); + zfree(&tidq->callchain); zfree(&tidq->last_branch); zfree(&tidq->prev_packet); zfree(&tidq->packet); out: + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); return rc; } @@ -750,21 +794,10 @@ static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, /* * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for * the next incoming packet. - * - * Threads and exception levels are also tracked for both the - * previous and current packets. This is because the previous - * packet is used for the 'from' IP for branch samples, so the - * thread at that time must also be assigned to that sample. - * Across discontinuity packets the thread can change, so by - * tracking the thread for the previous packet the branch sample - * will have the correct info. */ tmp = tidq->packet; tidq->packet = tidq->prev_packet; tidq->prev_packet = tmp; - tidq->prev_packet_el = tidq->el; - thread__put(tidq->prev_packet_thread); - tidq->prev_packet_thread = thread__get(tidq->thread); } } @@ -937,11 +970,11 @@ static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) /* Free this traceid_queue from the array */ tidq = etmq->traceid_queues[idx]; - thread__zput(tidq->thread); - thread__zput(tidq->prev_packet_thread); + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); zfree(&tidq->event_buf); + zfree(&tidq->callchain); zfree(&tidq->last_branch); - zfree(&tidq->last_branch_rb); zfree(&tidq->prev_packet); zfree(&tidq->packet); zfree(&tidq); @@ -1083,47 +1116,43 @@ static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, } } -static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, - u64 address, size_t size, u8 *buffer, - const ocsd_mem_space_acc_t mem_space) +static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space, + ocsd_ex_level el, struct thread *thread) { u8 cpumode; u64 offset; int len; struct addr_location al; struct dso *dso; - struct cs_etm_traceid_queue *tidq; int ret = 0; if (!etmq) return 0; addr_location__init(&al); - tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); - if (!tidq) - goto out; /* - * We've already tracked EL along side the PID in cs_etm__set_thread() - * so double check that it matches what OpenCSD thinks as well. It - * doesn't distinguish between EL0 and EL1 for this mem access callback - * so we had to do the extra tracking. Skip validation if it's any of - * the 'any' values. + * We track EL for the frontend and the backend when receiving context + * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 + * for this mem access callback so we had to do the extra tracking. Skip + * validation if it's any of the 'any' values. */ if (!(mem_space == OCSD_MEM_SPACE_ANY || mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { if (mem_space & OCSD_MEM_SPACE_EL1N) { /* Includes both non secure EL1 and EL0 */ - assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0); + assert(el == ocsd_EL1 || el == ocsd_EL0); } else if (mem_space & OCSD_MEM_SPACE_EL2) - assert(tidq->el == ocsd_EL2); + assert(el == ocsd_EL2); else if (mem_space & OCSD_MEM_SPACE_EL3) - assert(tidq->el == ocsd_EL3); + assert(el == ocsd_EL3); } - cpumode = cs_etm__cpu_mode(etmq, address, tidq->el); + cpumode = cs_etm__cpu_mode(etmq, address, el); - if (!thread__find_map(tidq->thread, cpumode, address, &al)) + if (!thread__find_map(thread, cpumode, address, &al)) goto out; dso = map__dso(al.map); @@ -1138,7 +1167,7 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, map__load(al.map); - len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)), + len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), offset, buffer, size); if (len <= 0) { @@ -1158,6 +1187,30 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, return ret; } +static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 address, size_t size, u8 *buffer) +{ + return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, + tidq->frontend_thread); +} + +static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return 0; + + return __cs_etm__mem_access(etmq, address, size, buffer, + mem_space, tidq->decode_el, + tidq->decode_thread); +} + static struct cs_etm_queue *cs_etm__alloc_queue(void) { struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); @@ -1281,64 +1334,14 @@ static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm, return ret; } -static inline -void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq, - struct cs_etm_traceid_queue *tidq) -{ - struct branch_stack *bs_src = tidq->last_branch_rb; - struct branch_stack *bs_dst = tidq->last_branch; - size_t nr = 0; - - /* - * Set the number of records before early exit: ->nr is used to - * determine how many branches to copy from ->entries. - */ - bs_dst->nr = bs_src->nr; - - /* - * Early exit when there is nothing to copy. - */ - if (!bs_src->nr) - return; - - /* - * As bs_src->entries is a circular buffer, we need to copy from it in - * two steps. First, copy the branches from the most recently inserted - * branch ->last_branch_pos until the end of bs_src->entries buffer. - */ - nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos; - memcpy(&bs_dst->entries[0], - &bs_src->entries[tidq->last_branch_pos], - sizeof(struct branch_entry) * nr); - - /* - * If we wrapped around at least once, the branches from the beginning - * of the bs_src->entries buffer and until the ->last_branch_pos element - * are older valid branches: copy them over. The total number of - * branches copied over will be equal to the number of branches asked by - * the user in last_branch_sz. - */ - if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) { - memcpy(&bs_dst->entries[nr], - &bs_src->entries[0], - sizeof(struct branch_entry) * tidq->last_branch_pos); - } -} - -static inline -void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) -{ - tidq->last_branch_pos = 0; - tidq->last_branch_rb->nr = 0; -} - static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, - u8 trace_chan_id, u64 addr) + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr) { u8 instrBytes[2]; - cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes), - instrBytes, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + ARRAY_SIZE(instrBytes), instrBytes); /* * T32 instruction size is indicated by bits[15:11] of the first * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 @@ -1347,6 +1350,18 @@ static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; } +static inline int cs_etm__instr_size(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 addr) +{ + if (packet->isa == CS_ETM_ISA_T32) + return cs_etm__t32_instr_size(etmq, tidq, packet, addr); + + /* Otherwise, 4-byte instruction size for A32/A64 */ + return 4; +} + static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) { /* @@ -1371,62 +1386,46 @@ u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) } static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 offset) { - if (packet->isa == CS_ETM_ISA_T32) { - u64 addr = packet->start_addr; + u64 addr = packet->start_addr; - while (offset) { - addr += cs_etm__t32_instr_size(etmq, - trace_chan_id, addr); - offset--; - } - return addr; - } + /* 4-byte instruction size for A32/A64 */ + if (packet->isa == CS_ETM_ISA_A64 || packet->isa == CS_ETM_ISA_A32) + return addr + offset * 4; - /* Assume a 4 byte instruction size (A32/A64) */ - return packet->start_addr + offset * 4; + while (offset) { + addr += cs_etm__instr_size(etmq, tidq, packet, addr); + offset--; + } + return addr; } -static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, - struct cs_etm_traceid_queue *tidq) +static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, + struct perf_sample *sample, u64 type) { - struct branch_stack *bs = tidq->last_branch_rb; - struct branch_entry *be; - - /* - * The branches are recorded in a circular buffer in reverse - * chronological order: we start recording from the last element of the - * buffer down. After writing the first element of the stack, move the - * insert position back to the end of the buffer. - */ - if (!tidq->last_branch_pos) - tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz; + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; - tidq->last_branch_pos -= 1; + if (!evsel && etm->session && etm->session->evlist) + evsel = evlist__id2evsel(etm->session->evlist, sample->id); - be = &bs->entries[tidq->last_branch_pos]; - be->from = cs_etm__last_executed_instr(tidq->prev_packet); - be->to = cs_etm__first_executed_instr(tidq->packet); - /* No support for mispredict */ - be->flags.mispred = 0; - be->flags.predicted = 1; + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; - /* - * Increment bs->nr until reaching the number of last branches asked by - * the user on the command line. - */ - if (bs->nr < etmq->etm->synth_opts.last_branch_sz) - bs->nr += 1; -} + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; -static int cs_etm__inject_event(union perf_event *event, - struct perf_sample *sample, u64 type) -{ - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } @@ -1472,34 +1471,51 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) return etmq->buf_len; } -static void cs_etm__set_thread(struct cs_etm_queue *etmq, - struct cs_etm_traceid_queue *tidq, pid_t tid, - ocsd_ex_level el) +/* + * Convert a raw thread number to a thread struct and assign it to **thread. + */ +static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, + ocsd_ex_level el, pid_t tid, + struct thread **thread) { struct machine *machine = cs_etm__get_machine(etmq, el); + if (!machine || !*thread) + return -EINVAL; + if (tid != -1) { - thread__zput(tidq->thread); - tidq->thread = machine__find_thread(machine, -1, tid); + thread__zput(*thread); + *thread = machine__find_thread(machine, -1, tid); } /* Couldn't find a known thread */ - if (!tidq->thread) - tidq->thread = machine__idle_thread(machine); + if (!*thread) + *thread = machine__idle_thread(machine); - tidq->el = el; + return 0; } -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el) +/* + * Set the thread and EL of the decode context which is ahead in time of the + * frontend context. + */ +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, + ocsd_ex_level el, pid_t tid) { struct cs_etm_traceid_queue *tidq; + int ret; tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); if (!tidq) return -EINVAL; - cs_etm__set_thread(etmq, tidq, tid, el); + ret = cs_etm__etmq_update_thread(etmq, el, tid, + &tidq->decode_thread); + if (ret) + return ret; + + tidq->decode_el = el; return 0; } @@ -1509,8 +1525,8 @@ bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) } static void cs_etm__copy_insn(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, struct perf_sample *sample) { /* @@ -1522,19 +1538,10 @@ static void cs_etm__copy_insn(struct cs_etm_queue *etmq, return; } - /* - * T32 instruction size might be 32-bit or 16-bit, decide by calling - * cs_etm__t32_instr_size(). - */ - if (packet->isa == CS_ETM_ISA_T32) - sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id, - sample->ip); - /* Otherwise, A64 and A32 instruction size are always 32-bit. */ - else - sample->insn_len = 4; + sample->insn_len = cs_etm__instr_size(etmq, tidq, packet, sample->ip); - cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len, - (void *)sample->insn, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, + sample->insn_len, (void *)sample->insn); } u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) @@ -1559,8 +1566,81 @@ static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, return etm->latest_kernel_timestamp; } +static bool cs_etm__packet_has_taken_branch(struct cs_etm_packet *packet) +{ + if (packet->sample_type == CS_ETM_RANGE && + packet->last_instr_taken_branch) + return true; + + return false; +} + +static void cs_etm__add_stack_event(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + u64 from, to; + int size; + + if (!etm->synth_opts.branches && !etm->synth_opts.instructions) + return; + + if (!cs_etm__packet_has_taken_branch(tidq->prev_packet)) + return; + + if (etmq->etm->use_thread_stack) { + from = cs_etm__last_executed_instr(tidq->prev_packet); + to = cs_etm__first_executed_instr(tidq->packet); + + size = cs_etm__instr_size(etmq, tidq, tidq->prev_packet, from); + + /* Enable callchain so thread stack entry can be allocated */ + thread_stack__event(tidq->frontend_thread, tidq->prev_packet->cpu, + tidq->prev_packet->flags, from, to, size, + etmq->buffer->buffer_nr + 1, + etmq->etm->use_callchain, + tidq->br_stack_sz, 0); + } else { + thread_stack__set_trace_nr(tidq->frontend_thread, + tidq->prev_packet->cpu, + etmq->buffer->buffer_nr + 1); + } +} + +static void cs_etm__sample_branch_stack(struct cs_etm_auxtrace *etm, + struct cs_etm_traceid_queue *tidq, + struct perf_sample *sample) +{ + if (etm->synth_opts.last_branch) { + thread_stack__br_sample(tidq->frontend_thread, tidq->packet->cpu, + tidq->last_branch, tidq->br_stack_sz); + sample->branch_stack = tidq->last_branch; + } + + if (etm->synth_opts.callchain) { + if (tidq->kernel_start) + thread_stack__sample(tidq->frontend_thread, + tidq->packet->cpu, + tidq->callchain, + etm->synth_opts.callchain_sz + 1, + sample->ip, tidq->kernel_start); + else + /* + * Clear the callchain when the kernel start address is + * not available yet. The empty callchain can then be + * consumed by cs_etm__inject_event(). + */ + memset(tidq->callchain, 0, + struct_size(tidq->callchain, ips, + etm->synth_opts.callchain_sz + 1)); + + sample->callchain = tidq->callchain; + } +} + static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr, u64 period) { int ret = 0; @@ -1570,29 +1650,27 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, perf_sample__init(&sample, /*all=*/true); event->sample.header.type = PERF_RECORD_SAMPLE; - event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el); + event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = addr; - sample.pid = thread__pid(tidq->thread); - sample.tid = thread__tid(tidq->thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.id = etmq->etm->instructions_id; sample.stream_id = etmq->etm->instructions_id; sample.period = period; - sample.cpu = tidq->packet->cpu; + sample.cpu = packet->cpu; sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample); - - if (etm->synth_opts.last_branch) - sample.branch_stack = tidq->last_branch; + cs_etm__copy_insn(etmq, tidq, packet, &sample); + cs_etm__sample_branch_stack(etm, tidq, &sample); if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->instructions_sample_type); if (ret) return ret; @@ -1618,8 +1696,9 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, { int ret = 0; struct cs_etm_auxtrace *etm = etmq->etm; - struct perf_sample sample = {.ip = 0,}; + struct perf_sample sample; union perf_event *event = tidq->event_buf; + struct dummy_branch_stack { u64 nr; u64 hw_idx; @@ -1627,19 +1706,24 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } dummy_bs; u64 ip; + if (etm->branches_filter && + !(etm->branches_filter & tidq->prev_packet->flags)) + return 0; + + perf_sample__init(&sample, /*all=*/true); ip = cs_etm__last_executed_instr(tidq->prev_packet); event->sample.header.type = PERF_RECORD_SAMPLE; event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, - tidq->prev_packet_el); + tidq->prev_packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = ip; - sample.pid = thread__pid(tidq->prev_packet_thread); - sample.tid = thread__tid(tidq->prev_packet_thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.addr = cs_etm__first_executed_instr(tidq->packet); sample.id = etmq->etm->branches_id; sample.stream_id = etmq->etm->branches_id; @@ -1648,8 +1732,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet, - &sample); + cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); /* * perf report cannot handle events without a branch stack @@ -1667,7 +1750,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->branches_sample_type); if (ret) return ret; @@ -1680,6 +1763,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, "CS ETM Trace: failed to deliver instruction event, error %d\n", ret); + perf_sample__exit(&sample); return ret; } @@ -1750,6 +1834,9 @@ static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; } + if (etm->synth_opts.callchain) + attr.sample_type |= PERF_SAMPLE_CALLCHAIN; + if (etm->synth_opts.instructions) { attr.config = PERF_COUNT_HW_INSTRUCTIONS; attr.sample_period = etm->synth_opts.period; @@ -1770,7 +1857,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, { struct cs_etm_auxtrace *etm = etmq->etm; int ret; - u8 trace_chan_id = tidq->trace_chan_id; u64 instrs_prev; /* Get instructions remainder from previous packet */ @@ -1778,14 +1864,7 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, tidq->period_instructions += tidq->packet->instr_count; - /* - * Record a branch when the last instruction in - * PREV_PACKET is a branch. - */ - if (etm->synth_opts.last_branch && - tidq->prev_packet->sample_type == CS_ETM_RANGE && - tidq->prev_packet->last_instr_taken_branch) - cs_etm__update_last_branch_rb(etmq, tidq); + cs_etm__add_stack_event(etmq, tidq); if (etm->synth_opts.instructions && tidq->period_instructions >= etm->instructions_sample_period) { @@ -1844,10 +1923,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, u64 offset = etm->instructions_sample_period - instrs_prev; u64 addr; - /* Prepare last branches for instruction sample */ - if (etm->synth_opts.last_branch) - cs_etm__copy_last_branch_rb(etmq, tidq); - while (tidq->period_instructions >= etm->instructions_sample_period) { /* @@ -1856,10 +1931,10 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, * been executed, but PC has not advanced to next * instruction) */ - addr = cs_etm__instr_addr(etmq, trace_chan_id, - tidq->packet, offset - 1); + addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, + offset - 1); ret = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->packet, addr, etm->instructions_sample_period); if (ret) return ret; @@ -1878,8 +1953,7 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, generate_sample = true; /* Generate sample for branch taken packet */ - if (tidq->prev_packet->sample_type == CS_ETM_RANGE && - tidq->prev_packet->last_instr_taken_branch) + if (cs_etm__packet_has_taken_branch(tidq->prev_packet)) generate_sample = true; if (generate_sample) { @@ -1894,6 +1968,34 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, return 0; } +static int cs_etm__context(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + ocsd_ex_level el = tidq->packet->el; + struct machine *machine; + int ret; + + machine = cs_etm__get_machine(etmq, el); + if (!machine) { + ret = -EINVAL; + goto err; + } + + tidq->kernel_start = machine__kernel_start(machine); + + ret = cs_etm__etmq_update_thread(etmq, el, tidq->packet->tid, + &tidq->frontend_thread); + if (ret) + goto err; + + return 0; + +err: + thread__zput(tidq->frontend_thread); + tidq->kernel_start = 0; + return ret; +} + static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) { /* @@ -1927,10 +2029,6 @@ static int cs_etm__flush(struct cs_etm_queue *etmq, etmq->etm->synth_opts.instructions && tidq->prev_packet->sample_type == CS_ETM_RANGE) { u64 addr; - - /* Prepare last branches for instruction sample */ - cs_etm__copy_last_branch_rb(etmq, tidq); - /* * Generate a last branch event for the branches left in the * circular buffer at the end of the trace. @@ -1941,7 +2039,7 @@ static int cs_etm__flush(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -1961,8 +2059,8 @@ static int cs_etm__flush(struct cs_etm_queue *etmq, cs_etm__packet_swap(etm, tidq); /* Reset last branches after flush the trace */ - if (etm->synth_opts.last_branch) - cs_etm__reset_last_branch_rb(tidq); + if (etm->use_thread_stack) + thread_stack__flush(tidq->frontend_thread); return err; } @@ -1986,9 +2084,6 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, tidq->prev_packet->sample_type == CS_ETM_RANGE) { u64 addr; - /* Prepare last branches for instruction sample */ - cs_etm__copy_last_branch_rb(etmq, tidq); - /* * Use the address of the end of the last reported execution * range. @@ -1996,7 +2091,7 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -2006,6 +2101,45 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, return 0; } + +static int cs_etm__flush_stack_cb(struct thread *thread, + void *data __maybe_unused) +{ + thread_stack__flush(thread); + return 0; +} + +static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) +{ + struct machine *machine; + + machine = machines__find(&etmq->etm->session->machines, pid); + if (machine) + machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); +} + +static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + if (!etmq->etm->use_thread_stack) + return; + + switch (pid_fmt) { + case CS_ETM_PIDFMT_CTXTID2: + /* Clear the guest stack if virtualization is supported */ + cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); + fallthrough; + case CS_ETM_PIDFMT_CTXTID: + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + + } +} + /* * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue * if need be. @@ -2028,14 +2162,20 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) ret = cs_etm_decoder__reset(etmq->decoder); if (ret) return ret; + + /* + * Since the decoder is reset, this causes a global trace + * discontinuity. Flush all thread stacks. + */ + cs_etm__flush_all_stack(etmq); } return etmq->buf_len; } -static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, - struct cs_etm_packet *packet, - u64 end_addr) +static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 end_addr) { /* Initialise to keep compiler happy */ u16 instr16 = 0; @@ -2057,8 +2197,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * so below only read 2 bytes as instruction size for T32. */ addr = end_addr - 2; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16), - (u8 *)&instr16, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr16), (u8 *)&instr16); if ((instr16 & 0xFF00) == 0xDF00) return true; @@ -2073,8 +2213,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +---------+---------+-------------------------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0x0F000000) == 0x0F000000 && (instr32 & 0xF0000000) != 0xF0000000) return true; @@ -2090,8 +2230,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +-----------------------+---------+-----------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0xFFE0001F) == 0xd4000001) return true; @@ -2107,7 +2247,6 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2120,9 +2259,9 @@ static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, * HVC cases; need to check if it's SVC instruction based on * packet address. */ - if (magic == __perf_cs_etmv4_magic) { + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { if (packet->exception_number == CS_ETMV4_EXC_CALL && - cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; } @@ -2143,7 +2282,7 @@ static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, packet->exception_number == CS_ETMV3_EXC_FIQ) return true; - if (magic == __perf_cs_etmv4_magic) + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) if (packet->exception_number == CS_ETMV4_EXC_RESET || packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || @@ -2160,7 +2299,6 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2174,7 +2312,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, packet->exception_number == CS_ETMV3_EXC_GENERIC) return true; - if (magic == __perf_cs_etmv4_magic) { + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { if (packet->exception_number == CS_ETMV4_EXC_TRAP || packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || packet->exception_number == CS_ETMV4_EXC_INST_FAULT || @@ -2186,7 +2324,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, * (SMC, HVC) are taken as sync exceptions. */ if (packet->exception_number == CS_ETMV4_EXC_CALL && - !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + !cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; @@ -2210,7 +2348,6 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, { struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; - u8 trace_chan_id = tidq->trace_chan_id; u64 magic; int ret; @@ -2291,11 +2428,11 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT) && - cs_etm__is_svc_instr(etmq, trace_chan_id, - packet, packet->start_addr)) + cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { prev_packet->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET; + } break; case CS_ETM_DISCONTINUITY: /* @@ -2376,6 +2513,7 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT; break; + case CS_ETM_CONTEXT: case CS_ETM_EMPTY: default: break; @@ -2451,6 +2589,17 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, */ cs_etm__sample(etmq, tidq); break; + case CS_ETM_CONTEXT: + /* + * Update context but don't swap packet. Keep the + * previous one for branch source address info, if + * tracing the kernel the context packet will be emitted + * between two ranges. + */ + ret = cs_etm__context(etmq, tidq); + if (ret) + goto out; + break; case CS_ETM_EXCEPTION: case CS_ETM_EXCEPTION_RET: /* @@ -2479,6 +2628,7 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, } } +out: return ret; } @@ -2602,7 +2752,7 @@ static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, if (!tidq) continue; - if (tid == -1 || thread__tid(tidq->thread) == tid) + if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) cs_etm__run_per_thread_timeless_decoder(etmq); } else cs_etm__run_per_cpu_timeless_decoder(etmq); @@ -3079,6 +3229,9 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); + if (!etmq) + return -EINVAL; + /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event * based on the sizes of the aux event, and queue that fragment. @@ -3310,7 +3463,7 @@ static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) */ if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 0x0L, ((u64) -1L), - cs_etm__mem_access)) + cs_etm__decoder_mem_access)) goto out_free_decoder; zfree(&t_params); @@ -3366,6 +3519,18 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, /* First the global part */ ptr = (u64 *) auxtrace_info->priv; num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + + /* + * Bound num_cpu by the event size: the global header consumes + * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 + * metadata entry after that. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE - + CS_ETM_HEADER_SIZE; + if (num_cpu <= 0 || priv_size <= 0 || + num_cpu > priv_size / (int)sizeof(u64)) + return -EINVAL; + metadata = zalloc(sizeof(*metadata) * num_cpu); if (!metadata) return -ENOMEM; @@ -3404,7 +3569,13 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, goto err_free_metadata; } - if ((int) metadata[j][CS_ETM_CPU] > max_cpu) + /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ + if (metadata[j][CS_ETM_CPU] >= INT_MAX) { + err = -EINVAL; + goto err_free_metadata; + } + + if ((int)metadata[j][CS_ETM_CPU] > max_cpu) max_cpu = metadata[j][CS_ETM_CPU]; } @@ -3451,6 +3622,25 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, itrace_synth_opts__set_default(&etm->synth_opts, session->itrace_synth_opts->default_no_sample); etm->synth_opts.callchain = false; + etm->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack; + } + + if (etm->synth_opts.calls) + etm->branches_filter |= PERF_IP_FLAG_CALL | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.returns) + etm->branches_filter |= PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.callchain && !symbol_conf.use_callchain) { + symbol_conf.use_callchain = true; + if (callchain_register_param(&callchain_param) < 0) { + symbol_conf.use_callchain = false; + etm->synth_opts.callchain = false; + } } etm->session = session; @@ -3496,12 +3686,20 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->tc.time_shift = tc->time_shift; etm->tc.time_mult = tc->time_mult; etm->tc.time_zero = tc->time_zero; - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { etm->tc.time_cycles = tc->time_cycles; etm->tc.time_mask = tc->time_mask; etm->tc.cap_user_time_zero = tc->cap_user_time_zero; etm->tc.cap_user_time_short = tc->cap_user_time_short; } + + etm->use_thread_stack = etm->synth_opts.thread_stack || + etm->synth_opts.last_branch || + etm->synth_opts.callchain; + + etm->use_callchain = etm->synth_opts.thread_stack || + etm->synth_opts.callchain; + err = cs_etm__synth_events(etm, session); if (err) goto err_free_queues; diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h index aa9bb4a32eca..b81099c2b301 100644 --- a/tools/perf/util/cs-etm.h +++ b/tools/perf/util/cs-etm.h @@ -158,6 +158,7 @@ enum cs_etm_sample_type { CS_ETM_DISCONTINUITY, CS_ETM_EXCEPTION, CS_ETM_EXCEPTION_RET, + CS_ETM_CONTEXT, }; enum cs_etm_isa { @@ -184,6 +185,8 @@ struct cs_etm_packet { u8 last_instr_size; u8 trace_chan_id; int cpu; + int el; + pid_t tid; }; #define CS_ETM_PACKET_MAX_BUFFER 1024 @@ -259,8 +262,9 @@ enum cs_etm_pid_fmt { #include int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu); enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq); -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el); +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, ocsd_ex_level el, + pid_t tid); bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq); void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, u8 trace_chan_id); diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c index 3b8f2df823a9..e222371394c0 100644 --- a/tools/perf/util/data-convert-bt.c +++ b/tools/perf/util/data-convert-bt.c @@ -11,14 +11,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include "asm/bug.h" #include "data-convert.h" #include "session.h" @@ -121,13 +120,13 @@ static int value_set(struct bt_ctf_field_type *type, } if (sign) { - ret = bt_ctf_field_signed_integer_set_value(field, val); + ret = bt_ctf_field_integer_signed_set_value(field, val); if (ret) { pr_err("failed to set field value %s\n", name); goto err; } } else { - ret = bt_ctf_field_unsigned_integer_set_value(field, val); + ret = bt_ctf_field_integer_unsigned_set_value(field, val); if (ret) { pr_err("failed to set field value %s\n", name); goto err; @@ -374,10 +373,10 @@ static int add_tracepoint_field_value(struct ctf_writer *cw, data + offset + i * len, len); if (!(flags & TEP_FIELD_IS_SIGNED)) - ret = bt_ctf_field_unsigned_integer_set_value( + ret = bt_ctf_field_integer_unsigned_set_value( field, value_int); else - ret = bt_ctf_field_signed_integer_set_value( + ret = bt_ctf_field_integer_signed_set_value( field, adjust_signedness(value_int, len)); } @@ -471,7 +470,7 @@ add_bpf_output_values(struct bt_ctf_event_class *event_class, goto put_len_type; } - ret = bt_ctf_field_unsigned_integer_set_value(len_field, nr_elements); + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); if (ret) { pr_err("failed to set field value for raw_len\n"); goto put_len_field; @@ -500,7 +499,7 @@ add_bpf_output_values(struct bt_ctf_event_class *event_class, struct bt_ctf_field *elem_field = bt_ctf_field_sequence_get_field(seq_field, i); - ret = bt_ctf_field_unsigned_integer_set_value(elem_field, + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, ((u32 *)(sample->raw_data))[i]); bt_ctf_field_put(elem_field); @@ -545,7 +544,7 @@ add_callchain_output_values(struct bt_ctf_event_class *event_class, goto put_len_type; } - ret = bt_ctf_field_unsigned_integer_set_value(len_field, nr_elements); + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); if (ret) { pr_err("failed to set field value for perf_callchain_size\n"); goto put_len_field; @@ -575,7 +574,7 @@ add_callchain_output_values(struct bt_ctf_event_class *event_class, struct bt_ctf_field *elem_field = bt_ctf_field_sequence_get_field(seq_field, i); - ret = bt_ctf_field_unsigned_integer_set_value(elem_field, + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, ((u64 *)(callchain->ips))[i]); bt_ctf_field_put(elem_field); @@ -728,7 +727,7 @@ static struct ctf_stream *ctf_stream__create(struct ctf_writer *cw, int cpu) goto out; } - ret = bt_ctf_field_unsigned_integer_set_value(cpu_field, (u32) cpu); + ret = bt_ctf_field_integer_unsigned_set_value(cpu_field, (u32) cpu); if (ret) { pr_err("Failed to update CPU number\n"); goto out; @@ -803,10 +802,10 @@ static bool is_flush_needed(struct ctf_stream *cs) static int process_sample_event(const struct perf_tool *tool, union perf_event *_event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct convert *c = container_of(tool, struct convert, tool); + struct evsel *evsel = sample->evsel; struct evsel_priv *priv = evsel->priv; struct ctf_writer *cw = &c->writer; struct ctf_stream *cs; @@ -1363,7 +1362,7 @@ static void cleanup_events(struct perf_session *session) zfree(&evsel->priv); } - evlist__delete(evlist); + evlist__put(evlist); session->evlist = NULL; } @@ -1414,7 +1413,7 @@ do { \ ADD("host", env->hostname); ADD("sysname", "Linux"); - ADD("release", env->os_release); + ADD("release", perf_env__os_release(env)); ADD("version", env->version); ADD("machine", env->arch); ADD("domain", "kernel"); diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index d526c91312ed..40412c3dbdb2 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -16,6 +16,7 @@ #include "linux/err.h" #include "util/auxtrace.h" #include "util/debug.h" +#include "util/env.h" #include "util/dso.h" #include "util/event.h" #include "util/evsel.h" @@ -159,13 +160,12 @@ static void output_sample_callchain_entry(const struct perf_tool *tool, static int process_sample_event(const struct perf_tool *tool, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel __maybe_unused, struct machine *machine) { struct convert_json *c = container_of(tool, struct convert_json, tool); FILE *out = c->out; struct addr_location al; - u64 sample_type = __evlist__combined_sample_type(evsel->evlist); + u64 sample_type = __evlist__combined_sample_type(sample->evsel->evlist); u8 cpumode = PERF_RECORD_MISC_USER; addr_location__init(&al); @@ -177,6 +177,7 @@ static int process_sample_event(const struct perf_tool *tool, if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { ++c->skipped; + addr_location__exit(&al); return 0; } @@ -245,7 +246,7 @@ static int process_sample_event(const struct perf_tool *tool, #ifdef HAVE_LIBTRACEEVENT if (sample->raw_data) { - struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_event *tp_format = evsel__tp_format(sample->evsel); struct tep_format_field **fields = tp_format ? tep_event_fields(tp_format) : NULL; if (fields) { @@ -273,7 +274,7 @@ static void output_headers(struct perf_session *session, struct convert_json *c) { struct stat st; const struct perf_header *header = &session->header; - const struct perf_env *env = perf_session__env(session); + struct perf_env *env = perf_session__env(session); int ret; int fd = perf_data__fd(session->data); int i; @@ -297,7 +298,8 @@ static void output_headers(struct perf_session *session, struct convert_json *c) output_json_key_format(out, true, 2, "feat-offset", "%" PRIu64, header->feat_offset); output_json_key_string(out, true, 2, "hostname", env->hostname); - output_json_key_string(out, true, 2, "os-release", env->os_release); + output_json_key_string(out, true, 2, "os-release", + perf_env__os_release(env)); output_json_key_string(out, true, 2, "arch", env->arch); if (env->cpu_desc) diff --git a/tools/perf/util/data-convert.h b/tools/perf/util/data-convert.h index ee651fa680a1..a96240f15671 100644 --- a/tools/perf/util/data-convert.h +++ b/tools/perf/util/data-convert.h @@ -11,10 +11,10 @@ struct perf_data_convert_opts { const char *time_str; }; -#ifdef HAVE_LIBBABELTRACE_SUPPORT +#ifdef HAVE_BABELTRACE2_CTF_WRITER_SUPPORT int bt_convert__perf2ctf(const char *input_name, const char *to_ctf, struct perf_data_convert_opts *opts); -#endif /* HAVE_LIBBABELTRACE_SUPPORT */ +#endif /* HAVE_BABELTRACE2_CTF_WRITER_SUPPORT */ int bt_convert__perf2json(const char *input_name, const char *to_ctf, struct perf_data_convert_opts *opts); diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c index 94dc534a7386..33fd1b82019e 100644 --- a/tools/perf/util/data.c +++ b/tools/perf/util/data.c @@ -345,9 +345,10 @@ static int open_dir(struct perf_data *data) if (asprintf(&data->file.path, "%s/data", data->path) < 0) return -1; - if (perf_data__is_write(data) && - mkdir(data->path, S_IRWXU) < 0) + if (perf_data__is_write(data) && mkdir(data->path, 0700) < 0) { + zfree(&data->file.path); return -1; + } ret = open_file(data); @@ -360,9 +361,16 @@ static int open_dir(struct perf_data *data) int perf_data__open(struct perf_data *data) { - if (check_pipe(data)) + int ret; + + if (data->open) return 0; + if (check_pipe(data)) { + data->open = true; + return 0; + } + /* currently it allows stdio for pipe only */ data->file.use_stdio = false; @@ -375,16 +383,24 @@ int perf_data__open(struct perf_data *data) if (perf_data__is_read(data)) data->is_dir = is_dir(data); - return perf_data__is_dir(data) ? - open_dir(data) : open_file_dup(data); + ret = perf_data__is_dir(data) ? open_dir(data) : open_file_dup(data); + + if (!ret) + data->open = true; + + return ret; } void perf_data__close(struct perf_data *data) { + if (!data->open) + return; + if (perf_data__is_dir(data)) perf_data__close_dir(data); perf_data_file__close(&data->file); + data->open = false; } static ssize_t perf_data_file__read(struct perf_data_file *file, void *buf, size_t size) @@ -457,6 +473,7 @@ int perf_data__switch(struct perf_data *data, if (!at_exit) { perf_data_file__close(&data->file); + data->open = false; ret = perf_data__open(data); if (ret < 0) goto out; diff --git a/tools/perf/util/data.h b/tools/perf/util/data.h index 8299fb5fa7da..76f57f60361f 100644 --- a/tools/perf/util/data.h +++ b/tools/perf/util/data.h @@ -50,6 +50,8 @@ struct perf_data { const char *path; /** @file: Underlying file to be used. */ struct perf_data_file file; + /** @open: Has the file or directory been opened. */ + bool open; /** @is_pipe: Underlying file is a pipe. */ bool is_pipe; /** @is_dir: Underlying file is a directory. */ @@ -59,7 +61,7 @@ struct perf_data { /** @in_place_update: A file opened for reading but will be written to. */ bool in_place_update; /** @mode: Read or write mode. */ - enum perf_data_mode mode; + enum perf_data_mode mode:8; struct { /** @version: perf_dir_version. */ diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c index cc2bb1af4243..ba54b1119ee6 100644 --- a/tools/perf/util/db-export.c +++ b/tools/perf/util/db-export.c @@ -208,8 +208,7 @@ static int db_ids_from_al(struct db_export *dbe, struct addr_location *al, static struct call_path *call_path_from_sample(struct db_export *dbe, struct machine *machine, struct thread *thread, - struct perf_sample *sample, - struct evsel *evsel) + struct perf_sample *sample) { u64 kernel_start = machine__kernel_start(machine); struct call_path *current = &dbe->cpr->call_path; @@ -227,7 +226,7 @@ static struct call_path *call_path_from_sample(struct db_export *dbe, */ callchain_param.order = ORDER_CALLER; cursor = get_tls_callchain_cursor(); - err = thread__resolve_callchain(thread, cursor, evsel, + err = thread__resolve_callchain(thread, cursor, sample, NULL, NULL, PERF_MAX_STACK_DEPTH); if (err) { callchain_param.order = saved_order; @@ -345,14 +344,13 @@ static int db_export__threads(struct db_export *dbe, struct thread *thread, } int db_export__sample(struct db_export *dbe, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, + struct perf_sample *sample, struct addr_location *al, struct addr_location *addr_al) { struct thread *thread = al->thread; struct export_sample es = { .event = event, .sample = sample, - .evsel = evsel, .al = al, }; struct thread *main_thread; @@ -365,7 +363,7 @@ int db_export__sample(struct db_export *dbe, union perf_event *event, if (!machine) return -1; - err = db_export__evsel(dbe, evsel); + err = db_export__evsel(dbe, sample->evsel); if (err) return err; @@ -390,8 +388,7 @@ int db_export__sample(struct db_export *dbe, union perf_event *event, if (dbe->cpr) { struct call_path *cp = call_path_from_sample(dbe, machine, - thread, sample, - evsel); + thread, sample); if (cp) { db_export__call_path(dbe, cp); es.call_path_id = cp->db_id; diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h index 23983cb35706..1abbfd398e3a 100644 --- a/tools/perf/util/db-export.h +++ b/tools/perf/util/db-export.h @@ -25,7 +25,6 @@ struct call_return; struct export_sample { union perf_event *event; struct perf_sample *sample; - struct evsel *evsel; struct addr_location *al; u64 db_id; u64 comm_db_id; @@ -96,7 +95,7 @@ int db_export__symbol(struct db_export *dbe, struct symbol *sym, int db_export__branch_type(struct db_export *dbe, u32 branch_type, const char *name); int db_export__sample(struct db_export *dbe, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, + struct perf_sample *sample, struct addr_location *al, struct addr_location *addr_al); int db_export__branch_types(struct db_export *dbe); diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c index 0e35c13abd04..84a78b30ceac 100644 --- a/tools/perf/util/debuginfo.c +++ b/tools/perf/util/debuginfo.c @@ -42,6 +42,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, { GElf_Addr dummy; int fd; + bool fd_consumed = false; fd = open(path, O_RDONLY); if (fd < 0) @@ -55,6 +56,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); if (!dbg->mod) goto error; + fd_consumed = true; dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); if (!dbg->dbg) @@ -62,13 +64,14 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy); - dwfl_report_end(dbg->dwfl, NULL, NULL); + if (dwfl_report_end(dbg->dwfl, NULL, NULL) != 0) + goto error; return 0; error: if (dbg->dwfl) dwfl_end(dbg->dwfl); - else + if (!fd_consumed) close(fd); memset(dbg, 0, sizeof(*dbg)); @@ -167,7 +170,7 @@ int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, /* Search the relocation related .text section */ for (i = 0; i < n; i++) { p = dwfl_module_relocation_info(dbg->mod, i, &shndx); - if (strcmp(p, ".text") == 0) { + if (p && strcmp(p, ".text") == 0) { /* OK, get the section header */ scn = elf_getscn(elf, shndx); if (!scn) diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 59ba88e1f744..0a1a7e9cf3ef 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -1577,8 +1577,11 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) return -1; - decomp = true; - strcpy(symfs_filename, tmp); + /* empty pathname means file wasn't actually compressed */ + if (tmp[0] != '\0') { + decomp = true; + strcpy(symfs_filename, tmp); + } } /* diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c index dc31b5e7149e..e11e144af62b 100644 --- a/tools/perf/util/dlfilter.c +++ b/tools/perf/util/dlfilter.c @@ -56,7 +56,7 @@ static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) d_al->symoff = al->addr - map__start(al->map) - sym->start; else d_al->symoff = 0; - d_al->sym_binding = sym->binding; + d_al->sym_binding = symbol__binding(sym); } else { d_al->sym = NULL; d_al->sym_start = 0; diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index b791e1b6b2cf..2309196d8df3 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -343,10 +343,16 @@ int filename__decompress(const char *name, char *pathname, * To keep this transparent, we detect this and return the file * descriptor to the uncompressed file. */ - if (!compressions[comp].is_compressed(name)) - return open(name, O_RDONLY); + if (!compressions[comp].is_compressed(name)) { + fd = open(name, O_RDONLY | O_CLOEXEC); + if (fd < 0) + *err = errno; + if (pathname && len > 0) + pathname[0] = '\0'; + return fd; + } - fd = mkstemp(tmpbuf); + fd = mkostemp(tmpbuf, O_CLOEXEC); if (fd < 0) { *err = errno; return -1; @@ -594,12 +600,28 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - errno = *dso__load_errno(dso); + /* + * Use a standard errno value, not the negative custom + * DSO_LOAD_ERRNO stored in dso__load_errno(dso): + * __open_dso() computes fd = -errno, so a negative + * errno produces a positive fd that looks valid. + */ + errno = EIO; goto out; } - *decomp = true; - strcpy(name, newpath); + /* empty pathname means file wasn't actually compressed */ + if (newpath[0] != '\0') { + char *tmp = strdup(newpath); + + if (!tmp) { + unlink(newpath); + goto out; + } + free(name); + name = tmp; + *decomp = true; + } } return name; @@ -864,6 +886,12 @@ static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) return -1; } + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(node->info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + len = node->info_linear->info.jited_prog_len; buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; @@ -1220,7 +1248,8 @@ static enum dso_swap_type dso_swap_type__from_elf_data(unsigned char eidata) } /* Reads e_machine from fd, optionally caching data in dso. */ -uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian) { uint16_t e_machine = EM_NONE; unsigned char e_ident[EI_NIDENT]; @@ -1250,6 +1279,9 @@ uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags if (swap_type == DSO_SWAP__UNSET) return EM_NONE; // Bad ELF data encoding. + if (is_big_endian) + *is_big_endian = (e_ident[EI_DATA] == ELFDATA2MSB); + /* Cache the need for swapping. */ if (optional_dso) { assert(dso__needs_swap(optional_dso) == DSO_SWAP__UNSET || @@ -1288,7 +1320,8 @@ uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags return e_machine; } -uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) { uint16_t e_machine = EM_NONE; int fd; @@ -1308,9 +1341,11 @@ uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_fl case DSO_BINARY_TYPE__BPF_IMAGE: case DSO_BINARY_TYPE__OOL: case DSO_BINARY_TYPE__JAVA_JIT: - if (e_flags) - *e_flags = EF_HOST; - return EM_HOST; + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); case DSO_BINARY_TYPE__DEBUGLINK: case DSO_BINARY_TYPE__BUILD_ID_CACHE: case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: @@ -1338,7 +1373,7 @@ uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_fl try_to_open_dso(dso, machine); fd = dso__data(dso)->fd; if (fd >= 0) - e_machine = dso__read_e_machine(dso, fd, e_flags); + e_machine = dso__read_e_machine_endian(dso, fd, e_flags, is_big_endian); else if (e_flags) *e_flags = 0; @@ -1766,7 +1801,7 @@ void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) if (machine__is_default_guest(machine)) return; - sprintf(path, "%s/sys/kernel/notes", machine->root_dir); + snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); sysfs__read_build_id(path, &bid); dso__set_build_id(dso, &bid); } @@ -1904,7 +1939,7 @@ static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, int saved_errno; nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); - fd = open(symfs_filename, O_RDONLY); + fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); saved_errno = errno; nsinfo__mountns_exit(&nsc); if (fd < 0) { @@ -1972,6 +2007,10 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return NULL; } info_linear = info_node->info_linear; + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } assert(len <= info_linear->info.jited_prog_len); *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index ede691e9a249..2916b954a804 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -866,8 +866,18 @@ int dso__data_file_size(struct dso *dso, struct machine *machine); off_t dso__data_size(struct dso *dso, struct machine *machine); ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, u64 offset, u8 *data, ssize_t size); -uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags); -uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags); +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +{ + return dso__read_e_machine_endian(optional_dso, fd, e_flags, NULL); +} +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +{ + return dso__e_machine_endian(dso, machine, e_flags, NULL); +} ssize_t dso__data_read_addr(struct dso *dso, struct map *map, struct machine *machine, u64 addr, u8 *data, ssize_t size); diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c index 92db2fccc788..d7160f87ac7d 100644 --- a/tools/perf/util/dwarf-aux.c +++ b/tools/perf/util/dwarf-aux.c @@ -125,7 +125,8 @@ int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, && die_entrypc(&die_mem, &faddr) == 0 && faddr == addr) { *fname = die_get_decl_file(&die_mem); - dwarf_decl_line(&die_mem, lineno); + if (dwarf_decl_line(&die_mem, lineno) != 0) + return -ENOENT; goto out; } @@ -171,7 +172,6 @@ int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, } return ret; - } /** @@ -460,7 +460,7 @@ int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs) size_t nexpr; int ret; - if (dwarf_attr(mb_die, DW_AT_data_member_location, &attr) == NULL) + if (dwarf_attr_integrate(mb_die, DW_AT_data_member_location, &attr) == NULL) return -ENOENT; if (dwarf_formudata(&attr, offs) != 0) { @@ -620,7 +620,7 @@ Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -659,7 +659,7 @@ Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -796,8 +796,7 @@ static int __die_walk_instances_cb(Dwarf_Die *inst, void *data) /* Ignore redundant instances */ if (dwarf_tag(inst) == DW_TAG_inlined_subroutine) { - dwarf_decl_line(origin, &tmp); - if (die_get_call_lineno(inst) == tmp) { + if (dwarf_decl_line(origin, &tmp) == 0 && die_get_call_lineno(inst) == tmp) { tmp = die_get_decl_fileno(origin); if (die_get_call_fileno(inst) == tmp) return DIE_FIND_CB_CONTINUE; @@ -951,11 +950,6 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) cu_die = dwarf_diecu(rt_die, &die_mem, NULL, NULL); dwarf_decl_line(rt_die, &decl); decf = die_get_decl_file(rt_die); - if (!decf) { - pr_debug2("Failed to get the declared file name of %s\n", - dwarf_diename(rt_die)); - return -EINVAL; - } } else cu_die = rt_die; if (!cu_die) { @@ -999,11 +993,12 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) if (die_find_inlinefunc(rt_die, addr, &die_mem)) { /* Call-site check */ inf = die_get_call_file(&die_mem); - if ((inf && !strcmp(inf, decf)) && + if ((inf == decf || (inf && decf && !strcmp(inf, decf))) && die_get_call_lineno(&die_mem) == lineno) goto found; - dwarf_decl_line(&die_mem, &inl); + if (dwarf_decl_line(&die_mem, &inl) != 0) + inl = 0; if (inl != decl || decf != die_get_decl_file(&die_mem)) continue; @@ -1035,8 +1030,10 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) .data = data, .retval = 0, }; - dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0); - ret = param.retval; + if (dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0) < 0) + ret = -EINVAL; + else + ret = param.retval; } return ret; @@ -1940,10 +1937,12 @@ static bool die_get_postprologue_addr(unsigned long entrypc_idx, break; } - dwarf_lineaddr(line, postprologue_addr); - if (*postprologue_addr >= highpc) - dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), - postprologue_addr); + if (dwarf_lineaddr(line, postprologue_addr) != 0) + return false; + if (*postprologue_addr >= highpc) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), postprologue_addr) != 0) + return false; + } return true; } diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h index a79968a2e573..161f0bf980b6 100644 --- a/tools/perf/util/dwarf-aux.h +++ b/tools/perf/util/dwarf-aux.h @@ -10,6 +10,11 @@ #include #include +static inline const char *die_name(Dwarf_Die *die) +{ + return dwarf_diename(die) ?: ""; +} + struct strbuf; /* Find the realpath of the target file */ diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index 1e54e2c86360..c0e2b9d5f0b2 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #include "cpumap.h" +#include "dwarf-regs.h" #include "debug.h" #include "env.h" #include "util/header.h" #include "util/rwsem.h" #include +#include #include #include #include @@ -248,6 +250,8 @@ void perf_env__exit(struct perf_env *env) { int i, j; + mutex_destroy(&env->lock); + perf_env__purge_bpf(env); perf_env__purge_cgroups(env); zfree(&env->hostname); @@ -305,19 +309,32 @@ void perf_env__init(struct perf_env *env) init_rwsem(&env->bpf_progs.lock); #endif env->kernel_is_64_bit = -1; + mutex_init(&env->lock); } static void perf_env__init_kernel_mode(struct perf_env *env) { - const char *arch = perf_env__raw_arch(env); + const char *arch = env->arch; + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; - if (!strncmp(arch, "x86_64", 6) || !strncmp(arch, "aarch64", 7) || - !strncmp(arch, "arm64", 5) || !strncmp(arch, "mips64", 6) || - !strncmp(arch, "parisc64", 8) || !strncmp(arch, "riscv64", 7) || - !strncmp(arch, "s390x", 5) || !strncmp(arch, "sparc64", 7)) - env->kernel_is_64_bit = 1; - else - env->kernel_is_64_bit = 0; + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + if (arch) { + if (strstr(arch, "64") || strstr(arch, "s390x")) + env->kernel_is_64_bit = 1; + else + env->kernel_is_64_bit = 0; + return; + } + + /* Fallback if completely unresolvable (assume host-bitness) */ + env->kernel_is_64_bit = (sizeof(void *) == 8) ? 1 : 0; } int perf_env__kernel_is_64_bit(struct perf_env *env) @@ -328,6 +345,60 @@ int perf_env__kernel_is_64_bit(struct perf_env *env) return env->kernel_is_64_bit; } +bool perf_arch_is_big_endian(const char *arch) +{ + if (!arch) + return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__; + + if (str_ends_with(arch, "_be") || !strcmp(arch, "sparc") || !strcmp(arch, "sparc64") || + !strcmp(arch, "s390") || !strcmp(arch, "s390x") || !strcmp(arch, "powerpc") || + !strcmp(arch, "ppc") || !strcmp(arch, "ppc64") || + !strcmp(arch, "mips") || !strcmp(arch, "mips64") || !strcmp(arch, "parisc") || + !strcmp(arch, "parisc64") || !strcmp(arch, "m68k") || + !strcmp(arch, "armeb") || !strcmp(arch, "mipseb") || !strcmp(arch, "mips64eb")) + return true; + + return false; +} + +const char *perf_env__os_release(struct perf_env *env) +{ + struct utsname uts; + int ret; + const char *release; + + if (!env) + return perf_version_string; + + mutex_lock(&env->lock); + if (env->os_release) { + release = env->os_release; + goto out; + } + + /* + * If env->arch is set, this is an offline target environment. + * If the os_release is not populated in the file, we do not want + * to poison it with the host's release which would break guest checks. + */ + if (env->arch) { + release = NULL; + goto out; + } + + /* + * The os_release is being accessed but wasn't initialized from a data + * file, assume this is 'live' mode and use the release from uname. If + * uname or strdup fails then use the current perf tool version. + */ + ret = uname(&uts); + env->os_release = strdup(ret < 0 ? perf_version_string : uts.release); + release = env->os_release ?: perf_version_string; +out: + mutex_unlock(&env->lock); + return release; +} + int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]) { int i; @@ -440,19 +511,6 @@ int perf_env__read_cpuid(struct perf_env *env) return 0; } -static int perf_env__read_arch(struct perf_env *env) -{ - struct utsname uts; - - if (env->arch) - return 0; - - if (!uname(&uts)) - env->arch = strdup(uts.machine); - - return env->arch ? 0 : -ENOMEM; -} - static int perf_env__read_nr_cpus_avail(struct perf_env *env) { if (env->nr_cpus_avail == 0) @@ -571,11 +629,6 @@ int perf_env__read_core_pmu_caps(struct perf_env *env) return ret; } -const char *perf_env__raw_arch(struct perf_env *env) -{ - return env && !perf_env__read_arch(env) ? env->arch : "unknown"; -} - int perf_env__nr_cpus_avail(struct perf_env *env) { return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0; @@ -588,67 +641,244 @@ void cpu_cache_level__free(struct cpu_cache_level *cache) zfree(&cache->size); } +struct arch_to_e_machine { + const char *prefix; + uint16_t e_machine; +}; + /* - * Return architecture name in a normalized form. - * The conversion logic comes from the Makefile. + * A mapping from an arch prefix string to an ELF machine that can be used in a + * bsearch. Some arch prefixes are shared an need additional processing as + * marked next to the architecture. The prefixes handle both perf's architecture + * naming and those from uname. */ -static const char *normalize_arch(char *arch) -{ - if (!strcmp(arch, "x86_64")) - return "x86"; - if (arch[0] == 'i' && arch[2] == '8' && arch[3] == '6') - return "x86"; - if (!strcmp(arch, "sun4u") || !strncmp(arch, "sparc", 5)) - return "sparc"; - if (!strncmp(arch, "aarch64", 7) || !strncmp(arch, "arm64", 5)) - return "arm64"; - if (!strncmp(arch, "arm", 3) || !strcmp(arch, "sa110")) - return "arm"; - if (!strncmp(arch, "s390", 4)) - return "s390"; - if (!strncmp(arch, "parisc", 6)) - return "parisc"; - if (!strncmp(arch, "powerpc", 7) || !strncmp(arch, "ppc", 3)) - return "powerpc"; - if (!strncmp(arch, "mips", 4)) - return "mips"; - if (!strncmp(arch, "sh", 2) && isdigit(arch[2])) - return "sh"; - if (!strncmp(arch, "loongarch", 9)) - return "loongarch"; +static const struct arch_to_e_machine prefix_to_e_machine[] = { + {"aarch64", EM_AARCH64}, + {"alpha", EM_ALPHA}, + {"arc", EM_ARC}, + {"arm", EM_ARM}, /* Check also for EM_AARCH64. */ + {"avr", EM_AVR}, /* Check also for EM_AVR32. */ + {"bfin", EM_BLACKFIN}, + {"blackfin", EM_BLACKFIN}, + {"cris", EM_CRIS}, + {"csky", EM_CSKY}, + {"hppa", EM_PARISC}, + {"i386", EM_386}, + {"i486", EM_386}, + {"i586", EM_386}, + {"i686", EM_386}, + {"loongarch", EM_LOONGARCH}, + {"m32r", EM_M32R}, + {"m68k", EM_68K}, + {"microblaze", EM_MICROBLAZE}, + {"mips", EM_MIPS}, + {"msp430", EM_MSP430}, + {"parisc", EM_PARISC}, + {"powerpc", EM_PPC}, /* Check also for EM_PPC64. */ + {"ppc", EM_PPC}, /* Check also for EM_PPC64. */ + {"riscv", EM_RISCV}, + {"s390", EM_S390}, + {"sa110", EM_ARM}, + {"sh", EM_SH}, + {"sparc", EM_SPARC}, /* Check also for EM_SPARCV9. */ + {"sun4u", EM_SPARC}, + {"x86", EM_X86_64}, /* Check also for EM_386. */ + {"xtensa", EM_XTENSA}, +}; + +static int compare_prefix(const void *key, const void *element) +{ + const char *search_key = key; + const struct arch_to_e_machine *map_element = element; + size_t prefix_len = strlen(map_element->prefix); - return arch; + return strncmp(search_key, map_element->prefix, prefix_len); } -const char *perf_env__arch(struct perf_env *env) +static uint16_t perf_arch_to_e_machine(const char *perf_arch, int is_64_bit) +{ + /* Binary search for a matching prefix. */ + const struct arch_to_e_machine *result; + + if (!perf_arch) + return EM_HOST; + + result = bsearch(perf_arch, + prefix_to_e_machine, ARRAY_SIZE(prefix_to_e_machine), + sizeof(prefix_to_e_machine[0]), + compare_prefix); + + if (!result) { + pr_debug("Unknown perf arch for ELF machine mapping: %s\n", perf_arch); + return EM_NONE; + } + + /* + * Handle conflicting prefixes. If the is_64_bit is unknown (-1) then + * assume 64-bit. We can't use perf_env__kernel_is_64_bit as that + * depends on the arch string. + */ + switch (result->e_machine) { + case EM_ARM: + return !strcmp(perf_arch, "arm64") || !strcmp(perf_arch, "aarch64") + ? EM_AARCH64 : EM_ARM; + case EM_AVR: + return !strcmp(perf_arch, "avr32") ? EM_AVR32 : EM_AVR; + case EM_PPC: + if (is_64_bit == 1) + return EM_PPC64; + if (is_64_bit == 0) + return EM_PPC; + return strstarts(perf_arch, "ppc64") ? EM_PPC64 : EM_PPC; + case EM_SPARC: + if (is_64_bit == 1) + return EM_SPARCV9; + if (is_64_bit == 0) + return EM_SPARC; + return !strcmp(perf_arch, "sparc64") || !strcmp(perf_arch, "sun4u") + ? EM_SPARCV9 : EM_SPARC; + case EM_X86_64: + if (is_64_bit == 1) + return EM_X86_64; + if (is_64_bit == 0) + return EM_386; + return !strcmp(perf_arch, "x86_64") || !strcmp(perf_arch, "x86") + ? EM_X86_64 : EM_386; + default: + return result->e_machine; + } +} + +static const char *e_machine_to_perf_arch(uint16_t e_machine) +{ + /* + * Table for if either the perf arch string differs from uname or there + * are >1 ELF machine with the prefix. + */ + static const struct arch_to_e_machine extras[] = { + {"arm64", EM_AARCH64}, + {"avr32", EM_AVR32}, + {"powerpc", EM_PPC}, + {"powerpc", EM_PPC64}, + {"sparc", EM_SPARCV9}, + {"x86", EM_386}, + {"x86", EM_X86_64}, + {"none", EM_NONE}, + }; + + for (size_t i = 0; i < ARRAY_SIZE(extras); i++) { + if (extras[i].e_machine == e_machine) + return extras[i].prefix; + } + + for (size_t i = 0; i < ARRAY_SIZE(prefix_to_e_machine); i++) { + if (prefix_to_e_machine[i].e_machine == e_machine) + return prefix_to_e_machine[i].prefix; + + } + return "unknown"; +} + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags) { - char *arch_name; + uint16_t e_machine = EM_NONE; + const char *arch = NULL; + int is_64_bit = -1; + + if (e_flags) + *e_flags = 0; + + if (env) { + arch = env->arch; + is_64_bit = env->kernel_is_64_bit; + } - if (!env || !env->arch) { /* Assume local operation */ + if (!arch) { static struct utsname uts = { .machine[0] = '\0', }; - if (uts.machine[0] == '\0' && uname(&uts) < 0) - return NULL; - arch_name = uts.machine; - } else - arch_name = env->arch; - return normalize_arch(arch_name); + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + e_machine = perf_arch_to_e_machine(arch, is_64_bit); + + if (e_flags) + *e_flags = (e_machine == EM_HOST) ? EF_HOST : 0; + + return e_machine; } -#if defined(HAVE_LIBTRACEEVENT) -#include "trace/beauty/arch_errno_names.c" -#endif +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine; + uint32_t local_e_flags = 0; + + if (env && env->e_machine != EM_NONE) { + if (e_flags) + *e_flags = env->e_flags; + + return env->e_machine; + } + e_machine = perf_env__e_machine_nocache(env, &local_e_flags); + /* + * Only cache the e_machine in perf_env if env->arch is not NULL. + * If env->arch is NULL, the e_machine is just a fallback to EM_HOST. + * Caching it permanently would prevent dynamic, more accurate + * thread-based session e_machine scanning later in + * perf_session__e_machine(). + */ + if (env && env->arch) { + env->e_machine = e_machine; + env->e_flags = local_e_flags; + } + if (e_flags) + *e_flags = local_e_flags; -const char *perf_env__arch_strerrno(struct perf_env *env __maybe_unused, int err __maybe_unused) + return e_machine; +} + +const char *perf_env__arch(struct perf_env *env) { -#if defined(HAVE_LIBTRACEEVENT) - if (env->arch_strerrno == NULL) - env->arch_strerrno = arch_syscalls__strerrno_function(perf_env__arch(env)); + uint16_t e_machine; + const char *arch; - return env->arch_strerrno ? env->arch_strerrno(err) : "no arch specific strerrno function"; -#else - return "!HAVE_LIBTRACEEVENT"; -#endif + if (!env) { + static struct utsname uts = { .machine[0] = '\0', }; + uint16_t host_e_machine; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') { + host_e_machine = perf_arch_to_e_machine(uts.machine, -1); + return e_machine_to_perf_arch(host_e_machine); + } + return e_machine_to_perf_arch(EM_HOST); + } + + /* + * Lazily compute/allocate arch. The e_machine may have been + * read from a data file and so may not be EM_HOST. + */ + e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + arch = e_machine_to_perf_arch(e_machine); + + if (e_machine == EM_RISCV && perf_env__kernel_is_64_bit(env) == 1) + arch = "riscv64"; + else if (e_machine == EM_MIPS && perf_env__kernel_is_64_bit(env) == 1) + arch = "mips64"; + else if (e_machine == EM_PARISC && perf_env__kernel_is_64_bit(env) == 1) + arch = "parisc64"; + + return arch; +} + +const char *arch_syscalls__strerrno(uint16_t e_machine, int err); + +const char *perf_env__arch_strerrno(uint16_t e_machine, int err) +{ + return arch_syscalls__strerrno(e_machine, err); } const char *perf_env__cpuid(struct perf_env *env) @@ -825,6 +1055,7 @@ bool x86__is_amd_cpu(void) struct perf_env env = { .total_mem = 0, }; bool is_amd; + perf_env__init(&env); perf_env__cpuid(&env); is_amd = perf_env__is_x86_amd_cpu(&env); perf_env__exit(&env); @@ -847,6 +1078,7 @@ bool x86__is_intel_cpu(void) struct perf_env env = { .total_mem = 0, }; bool is_intel; + perf_env__init(&env); perf_env__cpuid(&env); is_intel = perf_env__is_x86_intel_cpu(&env); perf_env__exit(&env); diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index c7052ac1f856..7acca39b42ff 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -6,6 +6,7 @@ #include #include "cpumap.h" #include "rwsem.h" +#include "mutex.h" struct perf_cpu_map; @@ -67,8 +68,6 @@ struct cpu_domain_map { struct domain_info **domains; }; -typedef const char *(arch_syscalls__strerrno_t)(int err); - struct perf_env { char *hostname; char *os_release; @@ -158,7 +157,8 @@ struct perf_env { */ bool enabled; } clock; - arch_syscalls__strerrno_t *arch_strerrno; + /* Protects lazy environment initialization (e.g. os_release, e_machine). */ + struct mutex lock; }; enum perf_compress_type { @@ -175,6 +175,8 @@ void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, void perf_env__exit(struct perf_env *env); int perf_env__kernel_is_64_bit(struct perf_env *env); +bool perf_arch_is_big_endian(const char *arch); +const char *perf_env__os_release(struct perf_env *env); int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]); @@ -185,12 +187,27 @@ const char *perf_env__pmu_mappings(struct perf_env *env); int perf_env__read_cpu_topology_map(struct perf_env *env); +/* + * Safe accessor for env->cpu[] topology array. env->cpu can be NULL when + * reading old-format perf.data that predates topology information — + * process_cpu_topology() in header.c frees it while nr_cpus_avail remains + * set, so callers must not index env->cpu[] without this check. + */ +static inline struct cpu_topology_map * +perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu) +{ + if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail) + return &env->cpu[cpu.cpu]; + return NULL; +} + void cpu_cache_level__free(struct cpu_cache_level *cache); +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags); +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags); const char *perf_env__arch(struct perf_env *env); -const char *perf_env__arch_strerrno(struct perf_env *env, int err); +const char *perf_env__arch_strerrno(uint16_t e_machine, int err); const char *perf_env__cpuid(struct perf_env *env); -const char *perf_env__raw_arch(struct perf_env *env); int perf_env__nr_cpus_avail(struct perf_env *env); void perf_env__init(struct perf_env *env); diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c index 66f4843bb235..ea75816d126a 100644 --- a/tools/perf/util/event.c +++ b/tools/perf/util/event.c @@ -14,6 +14,7 @@ #include #include "cpumap.h" #include "dso.h" +#include "env.h" #include "event.h" #include "debug.h" #include "hist.h" @@ -836,8 +837,18 @@ int machine__resolve(struct machine *machine, struct addr_location *al, if (al->cpu >= 0) { struct perf_env *env = machine->env; - if (env && env->cpu) - al->socket = env->cpu[al->cpu].socket_id; + /* + * Bounds-check al->cpu (s32) before casting to struct perf_cpu + * (int16_t): without this, e.g. 65536 truncates to 0 and silently + * returns CPU 0's topology. Can go once perf_cpu.cpu is widened. + */ + if (env && al->cpu < env->nr_cpus_avail) { + struct cpu_topology_map *topo; + + topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu }); + if (topo) + al->socket = topo->socket_id; + } } /* Account for possible out-of-order switch events. */ diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index ee971d15b3c6..eb7c0d7be064 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -5,67 +5,69 @@ * Parts came from builtin-{top,stat,record}.c, see those files for further * copyright notes. */ -#include +#include "evlist.h" + #include #include -#include -#include "cpumap.h" -#include "util/mmap.h" -#include "thread_map.h" -#include "target.h" -#include "dwarf-regs.h" -#include "evlist.h" -#include "evsel.h" -#include "record.h" -#include "debug.h" -#include "units.h" -#include "bpf_counter.h" -#include // page_size -#include "affinity.h" -#include "../perf.h" -#include "asm/bug.h" -#include "bpf-event.h" -#include "util/event.h" -#include "util/string2.h" -#include "util/perf_api_probe.h" -#include "util/evsel_fprintf.h" -#include "util/pmu.h" -#include "util/sample.h" -#include "util/bpf-filter.h" -#include "util/stat.h" -#include "util/util.h" -#include "util/env.h" -#include "util/intel-tpebs.h" -#include "util/metricgroup.h" -#include "util/strbuf.h" #include -#include -#include #include -#include "parse-events.h" -#include - #include -#include -#include -#include -#include -#include - #include +#include #include #include -#include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // page_size +#include +#include +#include #include #include -#include #include +#include -#include +#include "../perf.h" +#include "affinity.h" +#include "asm/bug.h" +#include "bpf-event.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "cpumap.h" +#include "debug.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evsel.h" +#include "evsel_fprintf.h" +#include "intel-tpebs.h" +#include "metricgroup.h" +#include "mmap.h" +#include "parse-events.h" +#include "perf_api_probe.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "sample.h" +#include "stat.h" +#include "strbuf.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "units.h" +#include "util.h" #ifdef LACKS_SIGQUEUE_PROTOTYPE int sigqueue(pid_t pid, int sig, const union sigval value); @@ -74,29 +76,31 @@ int sigqueue(pid_t pid, int sig, const union sigval value); #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) #define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y) -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads) -{ - perf_evlist__init(&evlist->core); - perf_evlist__set_maps(&evlist->core, cpus, threads); - evlist->workload.pid = -1; - evlist->bkw_mmap_state = BKW_MMAP_NOTREADY; - evlist->ctl_fd.fd = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.pos = -1; - evlist->nr_br_cntr = -1; - metricgroup__rblist_init(&evlist->metric_events); - INIT_LIST_HEAD(&evlist->deferred_samples); -} +static void event_enable_timer__exit(struct event_enable_timer **ep); struct evlist *evlist__new(void) { - struct evlist *evlist = zalloc(sizeof(*evlist)); - - if (evlist != NULL) - evlist__init(evlist, NULL, NULL); - - return evlist; + struct evlist *result; + RC_STRUCT(evlist) *evlist; + + evlist = zalloc(sizeof(*evlist)); + if (ADD_RC_CHK(result, evlist)) { + perf_evlist__init(evlist__core(result)); + perf_evlist__set_maps(evlist__core(result), /*cpus=*/NULL, /*threads=*/NULL); + evlist__set_workload_pid(result, -1); + evlist__set_bkw_mmap_state(result, BKW_MMAP_NOTREADY); + evlist__set_ctl_fd_fd(result, -1); + evlist__set_ctl_fd_ack(result, -1); + evlist__set_ctl_fd_pos(result, -1); + evlist__set_nr_br_cntr(result, -1); + metricgroup__rblist_init(evlist__metric_events(result)); + INIT_LIST_HEAD(&evlist->deferred_samples); + refcount_set(evlist__refcnt(result), 1); + } else { + free(evlist); + result = NULL; + } + return result; } struct evlist *evlist__new_default(const struct target *target, bool sample_callchains) @@ -104,7 +108,6 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call struct evlist *evlist = evlist__new(); bool can_profile_kernel; struct perf_pmu *pmu = NULL; - struct evsel *evsel; char buf[256]; int err; @@ -131,14 +134,16 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call } /* If there is only 1 event a sample identifier isn't necessary. */ - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { + struct evsel *evsel; + evlist__for_each_entry(evlist, evsel) evsel__set_sample_id(evsel, /*can_sample_identifier=*/false); } return evlist; out_err: - evlist__delete(evlist); + evlist__put(evlist); return NULL; } @@ -147,13 +152,23 @@ struct evlist *evlist__new_dummy(void) struct evlist *evlist = evlist__new(); if (evlist && evlist__add_dummy(evlist)) { - evlist__delete(evlist); + evlist__put(evlist); evlist = NULL; } return evlist; } +struct evlist *evlist__get(struct evlist *evlist) +{ + struct evlist *result; + + if (RC_CHK_GET(result, evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return result; +} + /** * evlist__set_id_pos - set the positions of event ids. * @evlist: selected event list @@ -165,8 +180,8 @@ void evlist__set_id_pos(struct evlist *evlist) { struct evsel *first = evlist__first(evlist); - evlist->id_pos = first->id_pos; - evlist->is_pos = first->is_pos; + RC_CHK_ACCESS(evlist)->id_pos = first->id_pos; + RC_CHK_ACCESS(evlist)->is_pos = first->is_pos; } static void evlist__update_id_pos(struct evlist *evlist) @@ -185,49 +200,85 @@ static void evlist__purge(struct evlist *evlist) evlist__for_each_entry_safe(evlist, n, pos) { list_del_init(&pos->core.node); + if (pos->evlist) { + if (!RC_CHK_EQUAL(pos->evlist, evlist)) { + evlist__put(pos->evlist); + } else { + refcount_dec_and_test(evlist__refcnt(pos->evlist)); + RC_CHK_PUT(pos->evlist); + } + } pos->evlist = NULL; - evsel__delete(pos); + evsel__put(pos); } - evlist->core.nr_entries = 0; + evlist__core(evlist)->nr_entries = 0; } -void evlist__exit(struct evlist *evlist) +static void evlist__exit(struct evlist *evlist) { - metricgroup__rblist_exit(&evlist->metric_events); - event_enable_timer__exit(&evlist->eet); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); - perf_evlist__exit(&evlist->core); + metricgroup__rblist_exit(evlist__metric_events(evlist)); + event_enable_timer__exit(&RC_CHK_ACCESS(evlist)->eet); + free(evlist__mmap(evlist)); + free(evlist__overwrite_mmap(evlist)); + perf_evlist__exit(evlist__core(evlist)); } -void evlist__delete(struct evlist *evlist) +void evlist__put(struct evlist *evlist) { + struct evsel *evsel; + unsigned int count, ref_cnt; + if (evlist == NULL) return; + if (refcount_dec_and_test(evlist__refcnt(evlist))) + goto out_delete; + +retry: + count = refcount_read(evlist__refcnt(evlist)); + ref_cnt = count; + evlist__for_each_entry(evlist, evsel) { + if (RC_CHK_EQUAL(evsel->evlist, evlist) && count && + refcount_read(&evsel->refcnt) == 1) + count--; + } + if (refcount_read(evlist__refcnt(evlist)) != ref_cnt) + goto retry; + + if (count != 0) { + /* + * Not the last reference except for back references from + * evsels. + */ + RC_CHK_PUT(evlist); + return; + } +out_delete: evlist__free_stats(evlist); - evlist__munmap(evlist); + evlist__do_munmap(evlist); evlist__close(evlist); evlist__purge(evlist); evlist__exit(evlist); - free(evlist); + RC_CHK_FREE(evlist); } void evlist__add(struct evlist *evlist, struct evsel *entry) { - perf_evlist__add(&evlist->core, &entry->core); - entry->evlist = evlist; + perf_evlist__add(evlist__core(evlist), &entry->core); + evlist__put(entry->evlist); + entry->evlist = evlist__get(evlist); entry->tracking = !entry->core.idx; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) evlist__set_id_pos(evlist); } void evlist__remove(struct evlist *evlist, struct evsel *evsel) { + perf_evlist__remove(evlist__core(evlist), &evsel->core); + evlist__put(evsel->evlist); evsel->evlist = NULL; - perf_evlist__remove(&evlist->core, &evsel->core); } void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) @@ -276,7 +327,7 @@ int __evlist__set_tracepoints_handlers(struct evlist *evlist, static void evlist__set_leader(struct evlist *evlist) { - perf_evlist__set_leader(&evlist->core); + perf_evlist__set_leader(evlist__core(evlist)); } static struct evsel *evlist__dummy_event(struct evlist *evlist) @@ -290,7 +341,7 @@ static struct evsel *evlist__dummy_event(struct evlist *evlist) .sample_period = 1, }; - return evsel__new_idx(&attr, evlist->core.nr_entries); + return evsel__new_idx(&attr, evlist__nr_entries(evlist)); } int evlist__add_dummy(struct evlist *evlist) @@ -379,8 +430,8 @@ static bool evlist__use_affinity(struct evlist *evlist) struct perf_cpu_map *used_cpus = NULL; bool ret = false; - if (evlist->no_affinity || !evlist->core.user_requested_cpus || - cpu_map__is_dummy(evlist->core.user_requested_cpus)) + if (evlist__no_affinity(evlist) || !evlist__core(evlist)->user_requested_cpus || + cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus)) return false; evlist__for_each_entry(evlist, pos) { @@ -435,7 +486,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e .evsel = NULL, .cpu_map_idx = 0, .evlist_cpu_map_idx = 0, - .evlist_cpu_map_nr = perf_cpu_map__nr(evlist->core.all_cpus), + .evlist_cpu_map_nr = perf_cpu_map__nr(evlist__core(evlist)->all_cpus), .cpu = (struct perf_cpu){ .cpu = -1}, .affinity = NULL, }; @@ -451,7 +502,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e itr->affinity = &itr->saved_affinity; } itr->evsel = evlist__first(evlist); - itr->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, 0); + itr->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, 0); if (itr->affinity) affinity__set(itr->affinity, itr->cpu.cpu); itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu); @@ -486,7 +537,7 @@ void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr) if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) { evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container); evlist_cpu_itr->cpu = - perf_cpu_map__cpu(evlist_cpu_itr->container->core.all_cpus, + perf_cpu_map__cpu(evlist__core(evlist_cpu_itr->container)->all_cpus, evlist_cpu_itr->evlist_cpu_map_idx); if (evlist_cpu_itr->affinity) affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu); @@ -513,7 +564,7 @@ static int evsel__strcmp(struct evsel *pos, char *evsel_name) return !evsel__name_is(pos, evsel_name); } -static int evlist__is_enabled(struct evlist *evlist) +static bool evlist__is_enabled(struct evlist *evlist) { struct evsel *pos; @@ -529,7 +580,7 @@ static int evlist__is_enabled(struct evlist *evlist) static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { - struct evsel *pos; + struct evsel *pos, *member; struct evlist_cpu_iterator evlist_cpu_itr; bool has_imm = false; @@ -561,16 +612,16 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl if (excl_dummy && evsel__is_dummy_event(pos)) continue; pos->disabled = true; + + for_each_group_member(member, pos) + member->disabled = true; } /* * If we disabled only single event, we need to check * the enabled state of the evlist manually. */ - if (evsel_name) - evlist->enabled = evlist__is_enabled(evlist); - else - evlist->enabled = false; + evlist__set_enabled(evlist, evsel_name ? evlist__is_enabled(evlist) : false); } void evlist__disable(struct evlist *evlist) @@ -590,7 +641,7 @@ void evlist__disable_evsel(struct evlist *evlist, char *evsel_name) static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { - struct evsel *pos; + struct evsel *pos, *member; struct evlist_cpu_iterator evlist_cpu_itr; evlist__for_each_cpu(evlist_cpu_itr, evlist) { @@ -611,6 +662,9 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_ if (excl_dummy && evsel__is_dummy_event(pos)) continue; pos->disabled = false; + + for_each_group_member(member, pos) + member->disabled = false; } /* @@ -618,7 +672,7 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_ * so the toggle can work properly and toggle to * 'disabled' state. */ - evlist->enabled = true; + evlist__set_enabled(evlist, true); } void evlist__enable(struct evlist *evlist) @@ -638,23 +692,24 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) void evlist__toggle_enable(struct evlist *evlist) { - (evlist->enabled ? evlist__disable : evlist__enable)(evlist); + (evlist__enabled(evlist) ? evlist__disable : evlist__enable)(evlist); } int evlist__add_pollfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, fdarray_flag__default); + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__default); } int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) { - return perf_evlist__filter_pollfd(&evlist->core, revents_and_mask); + return perf_evlist__filter_pollfd(evlist__core(evlist), revents_and_mask); } #ifdef HAVE_EVENTFD_SUPPORT int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) { - return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, fdarray_flag__nonfilterable | fdarray_flag__non_perf_event); } @@ -662,7 +717,7 @@ int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) int evlist__poll(struct evlist *evlist, int timeout) { - return perf_evlist__poll(&evlist->core, timeout); + return perf_evlist__poll(evlist__core(evlist), timeout); } struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) @@ -672,7 +727,7 @@ struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) int hash; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) if (sid->id == id) @@ -685,7 +740,7 @@ struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id) { struct perf_sample_id *sid; - if (evlist->core.nr_entries == 1 || !id) + if (evlist__nr_entries(evlist) == 1 || !id) return evlist__first(evlist); sid = evlist__id2sid(evlist, id); @@ -720,13 +775,13 @@ static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64 n = (event->header.size - sizeof(event->header)) >> 3; if (event->header.type == PERF_RECORD_SAMPLE) { - if (evlist->id_pos >= n) + if (evlist__id_pos(evlist) >= n) return -1; - *id = array[evlist->id_pos]; + *id = array[evlist__id_pos(evlist)]; } else { - if (evlist->is_pos > n) + if (evlist__is_pos(evlist) > n) return -1; - n -= evlist->is_pos; + n -= evlist__is_pos(evlist); *id = array[n]; } return 0; @@ -740,7 +795,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event int hash; u64 id; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return first; if (!first->core.attr.sample_id_all && @@ -755,7 +810,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event return first; hash = hash_64(id, PERF_EVLIST__HLIST_BITS); - head = &evlist->core.heads[hash]; + head = &evlist__core(evlist)->heads[hash]; hlist_for_each_entry(sid, head, node) { if (sid->id == id) @@ -768,11 +823,11 @@ static int evlist__set_paused(struct evlist *evlist, bool value) { int i; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return 0; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - int fd = evlist->overwrite_mmap[i].core.fd; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + int fd = evlist__overwrite_mmap(evlist)[i].core.fd; int err; if (fd < 0) @@ -798,20 +853,20 @@ static void evlist__munmap_nofree(struct evlist *evlist) { int i; - if (evlist->mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->mmap[i].core); + if (evlist__mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__mmap(evlist)[i].core); - if (evlist->overwrite_mmap) - for (i = 0; i < evlist->core.nr_mmaps; i++) - perf_mmap__munmap(&evlist->overwrite_mmap[i].core); + if (evlist__overwrite_mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__overwrite_mmap(evlist)[i].core); } -void evlist__munmap(struct evlist *evlist) +void evlist__do_munmap(struct evlist *evlist) { evlist__munmap_nofree(evlist); - zfree(&evlist->mmap); - zfree(&evlist->overwrite_mmap); + zfree(&RC_CHK_ACCESS(evlist)->mmap); + zfree(&RC_CHK_ACCESS(evlist)->overwrite_mmap); } static void perf_mmap__unmap_cb(struct perf_mmap *map) @@ -825,12 +880,12 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, bool overwrite) { int i; - struct mmap *map = calloc(evlist->core.nr_mmaps, sizeof(struct mmap)); + struct mmap *map = calloc(evlist__core(evlist)->nr_mmaps, sizeof(struct mmap)); if (!map) return NULL; - for (i = 0; i < evlist->core.nr_mmaps; i++) { + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { struct perf_mmap *prev = i ? &map[i - 1].core : NULL; /* @@ -848,41 +903,73 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist, return map; } +static struct evlist *from_list_start(struct perf_evlist *core) +{ +#ifdef REFCNT_CHECKING + RC_STRUCT(evlist) *core_evlist = container_of(core, RC_STRUCT(evlist), core); + struct evlist *evlist; + + if (ADD_RC_CHK(evlist, core_evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return evlist; +#else + return container_of(core, struct evlist, core); +#endif +} + +static void from_list_end(struct evlist *evlist __maybe_unused) +{ +#ifdef REFCNT_CHECKING + evlist__put(evlist); +#endif +} + static void perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist, struct perf_evsel *_evsel, struct perf_mmap_param *_mp, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap_params *mp = container_of(_mp, struct mmap_params, core); struct evsel *evsel = container_of(_evsel, struct evsel, core); + if (!evlist) + return; + auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx); + + from_list_end(evlist); } static struct perf_mmap* perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx) { - struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct evlist *evlist = from_list_start(_evlist); struct mmap *maps; - maps = overwrite ? evlist->overwrite_mmap : evlist->mmap; + if (!evlist) + return NULL; + + maps = overwrite ? evlist__overwrite_mmap(evlist) : evlist__mmap(evlist); if (!maps) { maps = evlist__alloc_mmap(evlist, overwrite); - if (!maps) + if (!maps) { + from_list_end(evlist); return NULL; + } if (overwrite) { - evlist->overwrite_mmap = maps; - if (evlist->bkw_mmap_state == BKW_MMAP_NOTREADY) + RC_CHK_ACCESS(evlist)->overwrite_mmap = maps; + if (evlist__bkw_mmap_state(evlist) == BKW_MMAP_NOTREADY) evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING); } else { - evlist->mmap = maps; + RC_CHK_ACCESS(evlist)->mmap = maps; } } - + from_list_end(evlist); return &maps[idx].core; } @@ -1039,16 +1126,16 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, .mmap = perf_evlist__mmap_cb_mmap, }; - evlist->core.mmap_len = evlist__mmap_size(pages); - pr_debug("mmap size %zuB\n", evlist->core.mmap_len); + evlist__core(evlist)->mmap_len = evlist__mmap_size(pages); + pr_debug("mmap size %zuB\n", evlist__core(evlist)->mmap_len); - auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->core.mmap_len, + auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist__core(evlist)->mmap_len, auxtrace_pages, auxtrace_overwrite); - return perf_evlist__mmap_ops(&evlist->core, &ops, &mp.core); + return perf_evlist__mmap_ops(evlist__core(evlist), &ops, &mp.core); } -int evlist__mmap(struct evlist *evlist, unsigned int pages) +int evlist__do_mmap(struct evlist *evlist, unsigned int pages) { return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0); } @@ -1090,9 +1177,9 @@ int evlist__create_maps(struct evlist *evlist, struct target *target) if (!cpus) goto out_delete_threads; - evlist->core.has_user_cpus = !!target->cpu_list; + evlist__core(evlist)->has_user_cpus = !!target->cpu_list; - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); /* as evlist now has references, put count here */ perf_cpu_map__put(cpus); @@ -1232,15 +1319,15 @@ bool evlist__valid_sample_type(struct evlist *evlist) { struct evsel *pos; - if (evlist->core.nr_entries == 1) + if (evlist__nr_entries(evlist) == 1) return true; - if (evlist->id_pos < 0 || evlist->is_pos < 0) + if (evlist__id_pos(evlist) < 0 || evlist__is_pos(evlist) < 0) return false; evlist__for_each_entry(evlist, pos) { - if (pos->id_pos != evlist->id_pos || - pos->is_pos != evlist->is_pos) + if (pos->id_pos != evlist__id_pos(evlist) || + pos->is_pos != evlist__is_pos(evlist)) return false; } @@ -1251,18 +1338,18 @@ u64 __evlist__combined_sample_type(struct evlist *evlist) { struct evsel *evsel; - if (evlist->combined_sample_type) - return evlist->combined_sample_type; + if (RC_CHK_ACCESS(evlist)->combined_sample_type) + return RC_CHK_ACCESS(evlist)->combined_sample_type; evlist__for_each_entry(evlist, evsel) - evlist->combined_sample_type |= evsel->core.attr.sample_type; + RC_CHK_ACCESS(evlist)->combined_sample_type |= evsel->core.attr.sample_type; - return evlist->combined_sample_type; + return RC_CHK_ACCESS(evlist)->combined_sample_type; } u64 evlist__combined_sample_type(struct evlist *evlist) { - evlist->combined_sample_type = 0; + RC_CHK_ACCESS(evlist)->combined_sample_type = 0; return __evlist__combined_sample_type(evlist); } @@ -1339,7 +1426,7 @@ void evlist__update_br_cntr(struct evlist *evlist) evlist__new_abbr_name(evsel->abbr_name); } } - evlist->nr_br_cntr = i; + evlist__set_nr_br_cntr(evlist, i); } bool evlist__valid_read_format(struct evlist *evlist) @@ -1389,11 +1476,6 @@ bool evlist__sample_id_all(struct evlist *evlist) return first->core.attr.sample_id_all; } -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) -{ - evlist->selected = evsel; -} - void evlist__close(struct evlist *evlist) { struct evsel *evsel; @@ -1410,7 +1492,7 @@ void evlist__close(struct evlist *evlist) perf_evsel__free_fd(&evsel->core); perf_evsel__free_id(&evsel->core); } - perf_evlist__reset_id_hash(&evlist->core); + perf_evlist__reset_id_hash(evlist__core(evlist)); } static int evlist__create_syswide_maps(struct evlist *evlist) @@ -1437,7 +1519,7 @@ static int evlist__create_syswide_maps(struct evlist *evlist) return -ENOMEM; } - perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); perf_thread_map__put(threads); perf_cpu_map__put(cpus); return 0; @@ -1452,7 +1534,8 @@ int evlist__open(struct evlist *evlist) * Default: one fd per CPU, all threads, aka systemwide * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL */ - if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) { + if (evlist__core(evlist)->threads == NULL && + evlist__core(evlist)->user_requested_cpus == NULL) { err = evlist__create_syswide_maps(evlist); if (err < 0) goto out_err; @@ -1479,7 +1562,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int child_ready_pipe[2], go_pipe[2]; char bf; - evlist->workload.cork_fd = -1; + evlist__set_workload_cork_fd(evlist, -1); if (pipe(child_ready_pipe) < 0) { perror("failed to create 'ready' pipe"); @@ -1491,13 +1574,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const goto out_close_ready_pipe; } - evlist->workload.pid = fork(); - if (evlist->workload.pid < 0) { + evlist__set_workload_pid(evlist, fork()); + if (evlist__workload_pid(evlist) < 0) { perror("failed to fork"); goto out_close_pipes; } - if (!evlist->workload.pid) { + if (!evlist__workload_pid(evlist)) { int ret; if (pipe_output) @@ -1563,12 +1646,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } if (target__none(target)) { - if (evlist->core.threads == NULL) { + if (evlist__core(evlist)->threads == NULL) { fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n", __func__, __LINE__); goto out_close_pipes; } - perf_thread_map__set_pid(evlist->core.threads, 0, evlist->workload.pid); + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, + evlist__workload_pid(evlist)); } close(child_ready_pipe[1]); @@ -1582,7 +1666,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const } fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC); - evlist->workload.cork_fd = go_pipe[1]; + evlist__set_workload_cork_fd(evlist, go_pipe[1]); close(child_ready_pipe[0]); return 0; @@ -1597,18 +1681,18 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const int evlist__start_workload(struct evlist *evlist) { - if (evlist->workload.cork_fd >= 0) { + if (evlist__workload_cork_fd(evlist) >= 0) { char bf = 0; int ret; /* * Remove the cork, let it rip! */ - ret = write(evlist->workload.cork_fd, &bf, 1); + ret = write(evlist__workload_cork_fd(evlist), &bf, 1); if (ret < 0) perror("unable to write to pipe"); - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); return ret; } @@ -1619,10 +1703,10 @@ void evlist__cancel_workload(struct evlist *evlist) { int status; - if (evlist->workload.cork_fd >= 0) { - close(evlist->workload.cork_fd); - evlist->workload.cork_fd = -1; - waitpid(evlist->workload.pid, &status, WNOHANG); + if (evlist__workload_cork_fd(evlist) >= 0) { + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); + waitpid(evlist__workload_pid(evlist), &status, WNOHANG); } } @@ -1716,7 +1800,8 @@ int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size) { - int pages_attempted = evlist->core.mmap_len / 1024, pages_max_per_user, printed = 0; + int pages_attempted = evlist__core(evlist)->mmap_len / 1024; + int pages_max_per_user, printed = 0; switch (err) { case EPERM: @@ -1759,7 +1844,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) list_move_tail(&evsel->core.node, &move); } - list_splice(&move, &evlist->core.entries); + list_splice(&move, &evlist__core(evlist)->entries); } struct evsel *evlist__get_tracking_event(struct evlist *evlist) @@ -1801,7 +1886,7 @@ struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_ evlist__set_tracking_event(evlist, evsel); } else if (system_wide) { - perf_evlist__go_system_wide(&evlist->core, &evsel->core); + perf_evlist__go_system_wide(evlist__core(evlist), &evsel->core); } return evsel; @@ -1823,14 +1908,14 @@ struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) { - enum bkw_mmap_state old_state = evlist->bkw_mmap_state; + enum bkw_mmap_state old_state = evlist__bkw_mmap_state(evlist); enum action { NONE, PAUSE, RESUME, } action = NONE; - if (!evlist->overwrite_mmap) + if (!evlist__overwrite_mmap(evlist)) return; switch (old_state) { @@ -1860,7 +1945,7 @@ void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) WARN_ONCE(1, "Shouldn't get there\n"); } - evlist->bkw_mmap_state = state; + evlist__set_bkw_mmap_state(evlist, state); switch (action) { case PAUSE: @@ -2038,40 +2123,41 @@ int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) return 0; } - evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, - fdarray_flag__nonfilterable | - fdarray_flag__non_perf_event); - if (evlist->ctl_fd.pos < 0) { - evlist->ctl_fd.pos = -1; + evlist__set_ctl_fd_pos(evlist, + perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event)); + if (evlist__ctl_fd_pos(evlist) < 0) { + evlist__set_ctl_fd_pos(evlist, -1); pr_err("Failed to add ctl fd entry: %m\n"); return -1; } - evlist->ctl_fd.fd = fd; - evlist->ctl_fd.ack = ack; + evlist__set_ctl_fd_fd(evlist, fd); + evlist__set_ctl_fd_ack(evlist, ack); return 0; } bool evlist__ctlfd_initialized(struct evlist *evlist) { - return evlist->ctl_fd.pos >= 0; + return evlist__ctl_fd_pos(evlist) >= 0; } int evlist__finalize_ctlfd(struct evlist *evlist) { - struct pollfd *entries = evlist->core.pollfd.entries; + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist)) return 0; - entries[evlist->ctl_fd.pos].fd = -1; - entries[evlist->ctl_fd.pos].events = 0; - entries[evlist->ctl_fd.pos].revents = 0; + entries[evlist__ctl_fd_pos(evlist)].fd = -1; + entries[evlist__ctl_fd_pos(evlist)].events = 0; + entries[evlist__ctl_fd_pos(evlist)].revents = 0; - evlist->ctl_fd.pos = -1; - evlist->ctl_fd.ack = -1; - evlist->ctl_fd.fd = -1; + evlist__set_ctl_fd_pos(evlist, -1); + evlist__set_ctl_fd_ack(evlist, -1); + evlist__set_ctl_fd_fd(evlist, -1); return 0; } @@ -2088,7 +2174,7 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, data_size--; do { - err = read(evlist->ctl_fd.fd, &c, 1); + err = read(evlist__ctl_fd_fd(evlist), &c, 1); if (err > 0) { if (c == '\n' || c == '\0') break; @@ -2102,7 +2188,8 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, if (errno == EAGAIN || errno == EWOULDBLOCK) err = 0; else - pr_err("Failed to read from ctlfd %d: %m\n", evlist->ctl_fd.fd); + pr_err("Failed to read from ctlfd %d: %m\n", + evlist__ctl_fd_fd(evlist)); } break; } while (1); @@ -2140,13 +2227,13 @@ int evlist__ctlfd_ack(struct evlist *evlist) { int err; - if (evlist->ctl_fd.ack == -1) + if (evlist__ctl_fd_ack(evlist) == -1) return 0; - err = write(evlist->ctl_fd.ack, EVLIST_CTL_CMD_ACK_TAG, + err = write(evlist__ctl_fd_ack(evlist), EVLIST_CTL_CMD_ACK_TAG, sizeof(EVLIST_CTL_CMD_ACK_TAG)); if (err == -1) - pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist->ctl_fd.ack); + pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist__ctl_fd_ack(evlist)); return err; } @@ -2247,8 +2334,8 @@ int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) { int err = 0; char cmd_data[EVLIST_CTL_CMD_MAX_LEN]; - int ctlfd_pos = evlist->ctl_fd.pos; - struct pollfd *entries = evlist->core.pollfd.entries; + int ctlfd_pos = evlist__ctl_fd_pos(evlist); + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents) return 0; @@ -2419,14 +2506,15 @@ int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *o goto free_eet_times; } - eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags); + eet->pollfd_pos = perf_evlist__add_pollfd(evlist__core(evlist), eet->timerfd, + NULL, POLLIN, flags); if (eet->pollfd_pos < 0) { err = eet->pollfd_pos; goto close_timerfd; } eet->evlist = evlist; - evlist->eet = eet; + RC_CHK_ACCESS(evlist)->eet = eet; opts->target.initial_delay = eet->times[0].start; return 0; @@ -2476,7 +2564,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) if (!eet) return 0; - entries = eet->evlist->core.pollfd.entries; + entries = evlist__core(eet->evlist)->pollfd.entries; revents = entries[eet->pollfd_pos].revents; entries[eet->pollfd_pos].revents = 0; @@ -2512,7 +2600,7 @@ int event_enable_timer__process(struct event_enable_timer *eet) return 0; } -void event_enable_timer__exit(struct event_enable_timer **ep) +static void event_enable_timer__exit(struct event_enable_timer **ep) { if (!ep || !*ep) return; @@ -2616,7 +2704,7 @@ void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_lis } /* Should uniquify be disabled for the evlist? */ -static bool evlist__disable_uniquify(const struct evlist *evlist) +static bool evlist__disable_uniquify(struct evlist *evlist) { struct evsel *counter; struct perf_pmu *last_pmu = NULL; diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index e507f5f20ef6..838e263b76f3 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -2,29 +2,33 @@ #ifndef __PERF_EVLIST_H #define __PERF_EVLIST_H 1 +#include + #include #include -#include #include +#include +#include +#include + #include #include #include +#include #include + #include "affinity.h" #include "events_stats.h" #include "evsel.h" #include "rblist.h" -#include -#include -#include -struct pollfd; -struct thread_map; struct perf_cpu_map; struct perf_stat_config; +struct pollfd; struct record_opts; struct strbuf; struct target; +struct thread_map; /* * State machine of bkw_mmap_state: @@ -56,8 +60,9 @@ enum bkw_mmap_state { struct event_enable_timer; -struct evlist { +DECLARE_RC_STRUCT(evlist) { struct perf_evlist core; + refcount_t refcnt; bool enabled; bool no_affinity; int id_pos; @@ -82,7 +87,7 @@ struct evlist { struct { pthread_t th; volatile int done; - } thread; + } sb_thread; struct { int fd; /* control file descriptor */ int ack; /* ack file descriptor for control commands */ @@ -103,13 +108,232 @@ struct evsel_str_handler { void *handler; }; +static inline struct perf_evlist *evlist__core(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline const struct perf_evlist *evlist__const_core(const struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline int evlist__nr_entries(const struct evlist *evlist) +{ + return evlist__const_core(evlist)->nr_entries; +} + +static inline bool evlist__enabled(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->enabled; +} + +static inline void evlist__set_enabled(struct evlist *evlist, bool enabled) +{ + RC_CHK_ACCESS(evlist)->enabled = enabled; +} + +static inline bool evlist__no_affinity(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->no_affinity; +} + +static inline void evlist__set_no_affinity(struct evlist *evlist, bool no_affinity) +{ + RC_CHK_ACCESS(evlist)->no_affinity = no_affinity; +} + +static inline int evlist__sb_thread_done(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->sb_thread.done; +} + +static inline void evlist__set_sb_thread_done(struct evlist *evlist, int done) +{ + RC_CHK_ACCESS(evlist)->sb_thread.done = done; +} + +static inline pthread_t *evlist__sb_thread_th(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->sb_thread.th; +} + +static inline int evlist__id_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->id_pos; +} + +static inline int evlist__is_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->is_pos; +} + +static inline struct event_enable_timer *evlist__event_enable_timer(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->eet; +} + +static inline enum bkw_mmap_state evlist__bkw_mmap_state(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->bkw_mmap_state; +} + +static inline void evlist__set_bkw_mmap_state(struct evlist *evlist, enum bkw_mmap_state state) +{ + RC_CHK_ACCESS(evlist)->bkw_mmap_state = state; +} + +static inline struct mmap *evlist__mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->mmap; +} + +static inline struct mmap *evlist__overwrite_mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->overwrite_mmap; +} + +static inline struct events_stats *evlist__stats(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->stats; +} + +static inline u64 evlist__first_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->first_sample_time; +} + +static inline void evlist__set_first_sample_time(struct evlist *evlist, u64 first) +{ + RC_CHK_ACCESS(evlist)->first_sample_time = first; +} + +static inline u64 evlist__last_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->last_sample_time; +} + +static inline void evlist__set_last_sample_time(struct evlist *evlist, u64 last) +{ + RC_CHK_ACCESS(evlist)->last_sample_time = last; +} + +static inline int evlist__nr_br_cntr(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->nr_br_cntr; +} + +static inline void evlist__set_nr_br_cntr(struct evlist *evlist, int nr) +{ + RC_CHK_ACCESS(evlist)->nr_br_cntr = nr; +} + +static inline struct perf_session *evlist__session(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->session; +} + +static inline void evlist__set_session(struct evlist *evlist, struct perf_session *session) +{ + RC_CHK_ACCESS(evlist)->session = session; +} + +static inline void (*evlist__trace_event_sample_raw(struct evlist *evlist)) + (struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample) +{ + return RC_CHK_ACCESS(evlist)->trace_event_sample_raw; +} + +static inline void evlist__set_trace_event_sample_raw(struct evlist *evlist, + void (*fun)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample)) +{ + RC_CHK_ACCESS(evlist)->trace_event_sample_raw = fun; +} + +static inline pid_t evlist__workload_pid(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.pid; +} + +static inline void evlist__set_workload_pid(struct evlist *evlist, pid_t pid) +{ + RC_CHK_ACCESS(evlist)->workload.pid = pid; +} + +static inline int evlist__workload_cork_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.cork_fd; +} + +static inline void evlist__set_workload_cork_fd(struct evlist *evlist, int cork_fd) +{ + RC_CHK_ACCESS(evlist)->workload.cork_fd = cork_fd; +} + +static inline int evlist__ctl_fd_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.fd; +} + +static inline void evlist__set_ctl_fd_fd(struct evlist *evlist, int fd) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.fd = fd; +} + +static inline int evlist__ctl_fd_ack(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.ack; +} + +static inline void evlist__set_ctl_fd_ack(struct evlist *evlist, int ack) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.ack = ack; +} + +static inline int evlist__ctl_fd_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.pos; +} + +static inline void evlist__set_ctl_fd_pos(struct evlist *evlist, int pos) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.pos = pos; +} + +static inline refcount_t *evlist__refcnt(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->refcnt; +} + +static inline struct rblist *evlist__metric_events(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->metric_events; +} + +static inline struct list_head *evlist__deferred_samples(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->deferred_samples; +} + +static inline struct evsel *evlist__selected(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->selected; +} + +static inline void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) +{ + RC_CHK_ACCESS(evlist)->selected = evsel; +} + struct evlist *evlist__new(void); struct evlist *evlist__new_default(const struct target *target, bool sample_callchains); struct evlist *evlist__new_dummy(void); -void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, - struct perf_thread_map *threads); -void evlist__exit(struct evlist *evlist); -void evlist__delete(struct evlist *evlist); +struct evlist *evlist__get(struct evlist *evlist); +void evlist__put(struct evlist *evlist); void evlist__add(struct evlist *evlist, struct evsel *entry); void evlist__remove(struct evlist *evlist, struct evsel *evsel); @@ -198,8 +422,8 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, unsigned int auxtrace_pages, bool auxtrace_overwrite, int nr_cblocks, int affinity, int flush, int comp_level); -int evlist__mmap(struct evlist *evlist, unsigned int pages); -void evlist__munmap(struct evlist *evlist); +int evlist__do_mmap(struct evlist *evlist, unsigned int pages); +void evlist__do_munmap(struct evlist *evlist); size_t evlist__mmap_size(unsigned long pages); @@ -211,8 +435,6 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); void evlist__disable_non_dummy(struct evlist *evlist); void evlist__enable_non_dummy(struct evlist *evlist); -void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); - int evlist__create_maps(struct evlist *evlist, struct target *target); int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, struct target *target); @@ -235,26 +457,26 @@ void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list); static inline bool evlist__empty(struct evlist *evlist) { - return list_empty(&evlist->core.entries); + return list_empty(&evlist__core(evlist)->entries); } static inline struct evsel *evlist__first(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__first(&evlist->core); + struct perf_evsel *evsel = perf_evlist__first(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline struct evsel *evlist__last(struct evlist *evlist) { - struct perf_evsel *evsel = perf_evlist__last(&evlist->core); + struct perf_evsel *evsel = perf_evlist__last(evlist__core(evlist)); return container_of(evsel, struct evsel, core); } static inline int evlist__nr_groups(struct evlist *evlist) { - return perf_evlist__nr_groups(&evlist->core); + return perf_evlist__nr_groups(evlist__core(evlist)); } int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size); @@ -277,7 +499,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry(evlist, evsel) \ - __evlist__for_each_entry(&(evlist)->core.entries, evsel) + __evlist__for_each_entry(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_continue - continue iteration thru all the evsels @@ -293,7 +515,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_continue(evlist, evsel) \ - __evlist__for_each_entry_continue(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_continue(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_from - continue iteration from @evsel (included) @@ -309,7 +531,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_from(evlist, evsel) \ - __evlist__for_each_entry_from(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_from(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order @@ -325,7 +547,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @evsel: struct evsel iterator */ #define evlist__for_each_entry_reverse(evlist, evsel) \ - __evlist__for_each_entry_reverse(&(evlist)->core.entries, evsel) + __evlist__for_each_entry_reverse(&evlist__core(evlist)->entries, evsel) /** * __evlist__for_each_entry_safe - safely iterate thru all the evsels @@ -343,7 +565,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); * @tmp: struct evsel temp iterator */ #define evlist__for_each_entry_safe(evlist, tmp, evsel) \ - __evlist__for_each_entry_safe(&(evlist)->core.entries, tmp, evsel) + __evlist__for_each_entry_safe(&evlist__core(evlist)->entries, tmp, evsel) /** Iterator state for evlist__for_each_cpu */ struct evlist_cpu_iterator { @@ -449,7 +671,6 @@ int evlist__ctlfd_ack(struct evlist *evlist); int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, const char *str, int unset); int event_enable_timer__start(struct event_enable_timer *eet); -void event_enable_timer__exit(struct event_enable_timer **ep); int event_enable_timer__process(struct event_enable_timer *eet); struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 2ee87fd84d3e..d0484e4ee679 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -11,68 +11,71 @@ */ #define __SANE_USERSPACE_TYPES__ -#include +#include "evsel.h" + #include #include +#include + +#include #include -#include -#include -#include -#include #include +#include #include +#include +#include #include #include #include #include #include -#include -#include + +#include +#include +#include +#include +#include +#include +#include #include + +#include "../perf-sys.h" #include "asm/bug.h" +#include "bpf-filter.h" #include "bpf_counter.h" #include "callchain.h" #include "cgroup.h" #include "counts.h" +#include "debug.h" +#include "drm_pmu.h" #include "dwarf-regs.h" +#include "env.h" #include "event.h" -#include "evsel.h" -#include "time-utils.h" -#include "util/env.h" -#include "util/evsel_config.h" -#include "util/evsel_fprintf.h" #include "evlist.h" -#include -#include "thread_map.h" -#include "target.h" +#include "evsel_config.h" +#include "evsel_fprintf.h" +#include "hashmap.h" +#include "hist.h" +#include "hwmon_pmu.h" +#include "intel-tpebs.h" +#include "memswap.h" +#include "off_cpu.h" +#include "parse-branch-options.h" #include "perf_regs.h" +#include "pmu.h" +#include "pmus.h" #include "record.h" -#include "debug.h" -#include "trace-event.h" +#include "rlimit.h" #include "session.h" #include "stat.h" #include "string2.h" -#include "memswap.h" -#include "util.h" -#include "util/hashmap.h" -#include "off_cpu.h" -#include "pmu.h" -#include "pmus.h" -#include "drm_pmu.h" -#include "hwmon_pmu.h" +#include "target.h" +#include "thread_map.h" +#include "time-utils.h" #include "tool_pmu.h" #include "tp_pmu.h" -#include "rlimit.h" -#include "../perf-sys.h" -#include "util/parse-branch-options.h" -#include "util/bpf-filter.h" -#include "util/hist.h" -#include -#include -#include -#include "util/intel-tpebs.h" - -#include +#include "trace-event.h" +#include "util.h" #ifdef HAVE_LIBTRACEEVENT #include @@ -251,6 +254,58 @@ const char *evsel__pmu_name(const struct evsel *evsel) return event_type(evsel->core.attr.type); } +enum evsel_probe_type { + PROBE__UNKNOWN = 0, + PROBE__NOPE = 1, + PROBE__KPROBE = 2, + PROBE__UPROBE = 3, + /* + * Ftrace-based dynamic probes (kprobes/uprobes/fprobes created via + * tracefs) report PMU "tracepoint", not "kprobe"/"uprobe". Detect + * them by the __probe_ip field that the kernel adds to all dynamic + * probe formats. + */ + PROBE__FTRACE = 4, +}; + +static void evsel__resolve_probe_type(struct evsel *evsel) +{ + const char *name = evsel__pmu_name(evsel); + + if (!strcmp(name, "kprobe")) + evsel->probe_type = PROBE__KPROBE; + else if (!strcmp(name, "uprobe")) + evsel->probe_type = PROBE__UPROBE; + else if (!strcmp(name, "tracepoint") && evsel__field(evsel, "__probe_ip")) + evsel->probe_type = PROBE__FTRACE; + else + evsel->probe_type = PROBE__NOPE; +} + +bool evsel__is_probe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type > PROBE__NOPE; +} + +bool evsel__is_kprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__KPROBE; +} + +bool evsel__is_uprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__UPROBE; +} + #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) int __evsel__sample_size(u64 sample_type) @@ -367,6 +422,11 @@ void evsel__set_sample_id(struct evsel *evsel, evsel->core.attr.read_format |= PERF_FORMAT_ID; } +bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel) +{ + return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END; +} + /** * evsel__is_function_event - Return whether given evsel is a function * trace event @@ -385,10 +445,11 @@ bool evsel__is_function_event(struct evsel *evsel) #undef FUNCTION_EVENT } -void evsel__init(struct evsel *evsel, +static void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx) { perf_evsel__init(&evsel->core, attr, idx); + refcount_set(&evsel->refcnt, 1); evsel->tracking = !idx; evsel->unit = strdup(""); evsel->scale = 1.0; @@ -410,6 +471,7 @@ void evsel__init(struct evsel *evsel, evsel->supported = true; evsel->alternate_hw_config = PERF_COUNT_HW_MAX; evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c + evsel->probe_type = PROBE__UNKNOWN; } struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) @@ -469,7 +531,7 @@ static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) * The assumption is that @orig is not configured nor opened yet. * So we only care about the attributes that can be set while it's parsed. */ -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) +struct evsel *evsel__clone(struct evsel *orig) { struct evsel *evsel; @@ -482,11 +544,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) if (orig->bpf_obj) return NULL; - if (dest) - evsel = dest; - else - evsel = evsel__new(&orig->core.attr); - + evsel = evsel__new(&orig->core.attr); if (evsel == NULL) return NULL; @@ -571,7 +629,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) return evsel; out_err: - evsel__delete(evsel); + evsel__put(evsel); return NULL; } @@ -630,6 +688,12 @@ struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool return ERR_PTR(err); } +struct evsel *evsel__get(struct evsel *evsel) +{ + refcount_inc(&evsel->refcnt); + return evsel; +} + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel) { @@ -1396,6 +1460,12 @@ void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, perf_pmu__format_pack(format->bits, val, vp, /*zero=*/true); } +bool evsel__config_exists(const struct evsel *evsel, const char *config_name) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + return format && !bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS); +} int evsel__get_config_val(const struct evsel *evsel, const char *config_name, u64 *val) @@ -1795,27 +1865,114 @@ int evsel__append_addr_filter(struct evsel *evsel, const char *filter) /* Caller has to clear disabled after going through all CPUs. */ int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx) { - return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; } int evsel__enable(struct evsel *evsel) { - int err = perf_evsel__enable(&evsel->core); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable(evsel); + else + err = perf_evsel__enable(&evsel->core); if (!err) evsel->disabled = false; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable(member); + + if (mem_err) + return mem_err; + } + member->disabled = false; + } + } + return err; } /* Caller has to set disabled after going through all CPUs. */ int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx) { - return perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; } int evsel__disable(struct evsel *evsel) { - int err = perf_evsel__disable(&evsel->core); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable(evsel); + else + err = perf_evsel__disable(&evsel->core); + /* * We mark it disabled here so that tools that disable a event can * ignore events after they disable it. I.e. the ring buffer may have @@ -1825,6 +1982,27 @@ int evsel__disable(struct evsel *evsel) if (!err) evsel->disabled = true; + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable(member); + + if (mem_err) + return mem_err; + } + member->disabled = true; + } + } + return err; } @@ -1854,7 +2032,7 @@ void evsel__set_priv_destructor(void (*destructor)(void *priv)) evsel__priv_destructor = destructor; } -void evsel__exit(struct evsel *evsel) +static void evsel__exit(struct evsel *evsel) { assert(list_empty(&evsel->core.node)); assert(evsel->evlist == NULL); @@ -1885,15 +2063,20 @@ void evsel__exit(struct evsel *evsel) evsel__priv_destructor(evsel->priv); perf_evsel__object.fini(evsel); if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || - evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) - xyarray__delete(evsel->start_times); + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) { + xyarray__delete(evsel->process_time.start_times); + xyarray__delete(evsel->process_time.accumulated_times); + } } -void evsel__delete(struct evsel *evsel) +void evsel__put(struct evsel *evsel) { if (!evsel) return; + if (!refcount_dec_and_test(&evsel->refcnt)) + return; + evsel__exit(evsel); free(evsel); } @@ -3003,52 +3186,62 @@ int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads) return ret; } -static int perf_evsel__parse_id_sample(const struct evsel *evsel, - const union perf_event *event, +static int perf_evsel__parse_id_sample(const union perf_event *event, struct perf_sample *sample) { + const struct evsel *evsel = sample->evsel; u64 type = evsel->core.attr.sample_type; const __u64 *array = event->sample.array; bool swapped = evsel->needs_swap; union u64_swap u; - - array += ((event->header.size - - sizeof(event->header)) / sizeof(u64)) - 1; + int i = ((event->header.size - sizeof(event->header)) / sizeof(u64)) - 1; if (type & PERF_SAMPLE_IDENTIFIER) { - sample->id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; } if (type & PERF_SAMPLE_CPU) { - u.val64 = *array; + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; if (swapped) { /* undo swap of u64, then swap on individual u32s */ u.val64 = bswap_64(u.val64); u.val32[0] = bswap_32(u.val32[0]); } - sample->cpu = u.val32[0]; - array--; } if (type & PERF_SAMPLE_STREAM_ID) { - sample->stream_id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->stream_id = array[i--]; } if (type & PERF_SAMPLE_ID) { - sample->id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; } if (type & PERF_SAMPLE_TIME) { - sample->time = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->time = array[i--]; } if (type & PERF_SAMPLE_TID) { - u.val64 = *array; + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; if (swapped) { /* undo swap of u64, then swap on individual u32s */ u.val64 = bswap_64(u.val64); @@ -3058,7 +3251,6 @@ static int perf_evsel__parse_id_sample(const struct evsel *evsel, sample->pid = u.val32[0]; sample->tid = u.val32[1]; - array--; } return 0; @@ -3169,7 +3361,7 @@ static inline bool evsel__has_branch_counters(const struct evsel *evsel) if (!leader || !evsel->evlist) return false; - if (evsel->evlist->nr_br_cntr < 0) + if (evlist__nr_br_cntr(evsel->evlist) < 0) evlist__update_br_cntr(evsel->evlist); if (leader->br_cntr_nr > 0) @@ -3209,11 +3401,11 @@ static int __set_offcpu_sample(struct perf_sample *data) return -EFAULT; } -int evsel__parse_sample(struct evsel *evsel, union perf_event *event, - struct perf_sample *data) +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap) { u64 type = evsel->core.attr.sample_type; - bool swapped = evsel->needs_swap; + bool swapped = needs_swap; const __u64 *array; u16 max_size = event->header.size; const void *endp = (void *)event + max_size; @@ -3226,7 +3418,7 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event, union u64_swap u; perf_sample__init(data, /*all=*/true); - data->evsel = evsel; + data->evsel = evsel__get(evsel); data->cpu = data->pid = data->tid = -1; data->stream_id = data->id = data->time = -1ULL; data->period = evsel->core.attr.sample_period; @@ -3244,15 +3436,18 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event, data->deferred_cookie = event->callchain_deferred.cookie; - if (evsel->core.attr.sample_id_all) - perf_evsel__parse_id_sample(evsel, event, data); - + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } return 0; } if (event->header.type != PERF_RECORD_SAMPLE) { - if (evsel->core.attr.sample_id_all) - perf_evsel__parse_id_sample(evsel, event, data); + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } return 0; } @@ -3614,12 +3809,13 @@ int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, if (event->header.type != PERF_RECORD_SAMPLE) { struct perf_sample data = { + .evsel = evsel, .time = -1ULL, }; if (!evsel->core.attr.sample_id_all) return -1; - if (perf_evsel__parse_id_sample(evsel, event, &data)) + if (perf_evsel__parse_id_sample(event, &data)) return -1; *timestamp = data.time; @@ -3687,22 +3883,63 @@ struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *na return tp_format ? tep_find_common_field(tp_format, name) : NULL; } -void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name) +static bool out_of_bounds(const struct tep_format_field *field, int offset, int size, u32 raw_size) +{ + if (offset < 0) { + pr_warning("Negative trace point field offset %d in %s\n", + offset, field->name); + return true; + } + if (size < 0) { + pr_warning("Negative trace point field size %d in %s\n", + size, field->name); + return true; + } + if ((u32)offset + (u32)size > raw_size) { + pr_warning("Out of bound tracepoint field (%s) offset %d size %d in %u\n", + field->name, offset, size, raw_size); + return true; + } + return false; +} + +void *perf_sample__rawptr(struct perf_sample *sample, const char *name) { - struct tep_format_field *field = evsel__field(evsel, name); - int offset; + struct tep_format_field *field = evsel__field(sample->evsel, name); + int offset, size; if (!field) return NULL; offset = field->offset; - + size = field->size; if (field->flags & TEP_FIELD_IS_DYNAMIC) { - offset = *(int *)(sample->raw_data + field->offset); - offset &= 0xffff; - if (tep_field_is_relative(field->flags)) + int dynamic_data; + + if (out_of_bounds(field, offset, 4, sample->raw_size)) + return NULL; + + dynamic_data = *(int *)(sample->raw_data + field->offset); + + if (sample->evsel->needs_swap) + dynamic_data = bswap_32(dynamic_data); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) { + /* + * Newer kernel feature: Relative offsets (__rel_loc). + * If the relative flag is set, the parsed offset is not + * absolute from the start of the record. Instead, it is + * relative to the *end* of the dynamic field descriptor + * itself. + */ offset += field->offset + field->size; + } } + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; return sample->raw_data + offset; } @@ -3713,6 +3950,9 @@ u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sam u64 value; void *ptr = sample->raw_data + field->offset; + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return 0; + switch (field->size) { case 1: return *(u8 *)ptr; @@ -3746,21 +3986,21 @@ u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sam return 0; } -u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name) +u64 perf_sample__intval(struct perf_sample *sample, const char *name) { - struct tep_format_field *field = evsel__field(evsel, name); + struct tep_format_field *field = evsel__field(sample->evsel, name); - return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; } -u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name) +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name) { - struct tep_format_field *field = evsel__common_field(evsel, name); + struct tep_format_field *field = evsel__common_field(sample->evsel, name); - return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; } -char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const char *name) +char perf_sample__taskstate(struct perf_sample *sample, const char *name) { static struct tep_format_field *prev_state_field; static const char *states; @@ -3769,7 +4009,7 @@ char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const cha unsigned int bit; char state = '?'; /* '?' denotes unknown task state */ - field = evsel__field(evsel, name); + field = evsel__field(sample->evsel, name); if (!field) return state; @@ -3788,7 +4028,7 @@ char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const cha * * We can change this if we have a good reason in the future. */ - val = evsel__intval(evsel, sample, name); + val = perf_sample__intval(sample, name); bit = val ? ffs(val) : 0; state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1]; return state; @@ -4153,7 +4393,7 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target, struct perf_session *evsel__session(struct evsel *evsel) { - return evsel && evsel->evlist ? evsel->evlist->session : NULL; + return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL; } struct perf_env *evsel__env(struct evsel *evsel) @@ -4178,7 +4418,7 @@ static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) thread++) { int fd = FD(evsel, cpu_map_idx, thread); - if (perf_evlist__id_add_fd(&evlist->core, &evsel->core, + if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core, cpu_map_idx, thread, fd) < 0) return -1; } diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 339b5c08a33d..6b0c08958616 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -2,28 +2,31 @@ #ifndef __PERF_EVSEL_H #define __PERF_EVSEL_H 1 -#include #include -#include + +#include #include +#include #include +#include + #include #include + #include "symbol_conf.h" -#include "pmus.h" -#include "pmu.h" +struct bperf_follower_bpf; +struct bperf_leader_bpf; +struct bpf_counter_ops; struct bpf_object; struct cgroup; +struct hashmap; struct perf_counts; +struct perf_pmu; struct perf_stat_config; struct perf_stat_evsel; -union perf_event; -struct bpf_counter_ops; struct target; -struct hashmap; -struct bperf_leader_bpf; -struct bperf_follower_bpf; +union perf_event; typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); @@ -45,6 +48,7 @@ typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); struct evsel { struct perf_evsel core; struct evlist *evlist; + refcount_t refcnt; off_t id_offset; int id_pos; int is_pos; @@ -126,6 +130,7 @@ struct evsel { bool needs_uniquify; bool fallenback_eacces; bool fallenback_eopnotsupp; + u8 probe_type:3; struct hashmap *per_pkg_mask; int err; int script_output_type; @@ -190,12 +195,18 @@ struct evsel { double max; } retirement_latency; /* duration_time is a single global time. */ - __u64 start_time; + struct { + __u64 start_time; + __u64 accumulated_time; + } duration_time; /* * user_time and system_time read an initial value potentially * per-CPU or per-pid. */ - struct xyarray *start_times; + struct { + struct xyarray *start_times; + struct xyarray *accumulated_times; + } process_time; }; /* Is the tool's fd for /proc/pid/stat or /proc/stat. */ bool pid_stat; @@ -253,6 +264,10 @@ struct perf_pmu *evsel__find_pmu(const struct evsel *evsel); const char *evsel__pmu_name(const struct evsel *evsel); bool evsel__is_aux_event(const struct evsel *evsel); +bool evsel__is_probe(struct evsel *evsel); +bool evsel__is_kprobe(struct evsel *evsel); +bool evsel__is_uprobe(struct evsel *evsel); + struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx); static inline struct evsel *evsel__new(struct perf_event_attr *attr) @@ -260,7 +275,7 @@ static inline struct evsel *evsel__new(struct perf_event_attr *attr) return evsel__new_idx(attr, 0); } -struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig); +struct evsel *evsel__clone(struct evsel *orig); int copy_config_terms(struct list_head *dst, struct list_head *src); void free_config_terms(struct list_head *config_terms); @@ -275,14 +290,13 @@ static inline struct evsel *evsel__newtp(const char *sys, const char *name) return evsel__newtp_idx(sys, name, 0, true); } +struct evsel *evsel__get(struct evsel *evsel); +void evsel__put(struct evsel *evsel); + #ifdef HAVE_LIBTRACEEVENT struct tep_event *evsel__tp_format(struct evsel *evsel); #endif -void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx); -void evsel__exit(struct evsel *evsel); -void evsel__delete(struct evsel *evsel); - void evsel__set_priv_destructor(void (*destructor)(void *priv)); struct callchain_param; @@ -350,6 +364,7 @@ void arch_evsel__apply_ratio_to_prev(struct evsel *evsel, struct perf_event_attr int evsel__set_filter(struct evsel *evsel, const char *filter); int evsel__append_tp_filter(struct evsel *evsel, const char *filter); int evsel__append_addr_filter(struct evsel *evsel, const char *filter); + int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx); int evsel__enable(struct evsel *evsel); int evsel__disable(struct evsel *evsel); @@ -371,14 +386,14 @@ bool evsel__precise_ip_fallback(struct evsel *evsel); struct perf_sample; #ifdef HAVE_LIBTRACEEVENT -void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name); -u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name); -u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name); -char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const char *name); +void *perf_sample__rawptr(struct perf_sample *sample, const char *name); +u64 perf_sample__intval(struct perf_sample *sample, const char *name); +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name); +char perf_sample__taskstate(struct perf_sample *sample, const char *name); -static inline char *evsel__strval(struct evsel *evsel, struct perf_sample *sample, const char *name) +static inline char *perf_sample__strval(struct perf_sample *sample, const char *name) { - return evsel__rawptr(evsel, sample, name); + return perf_sample__rawptr(sample, name); } #endif @@ -386,8 +401,22 @@ struct tep_format_field; u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); +#ifdef HAVE_LIBTRACEEVENT struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); +#else +static inline struct tep_format_field * +evsel__field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} + +static inline struct tep_format_field * +evsel__common_field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} +#endif bool __evsel__match(const struct evsel *evsel, u32 type, u64 config); @@ -421,8 +450,14 @@ static inline int evsel__read_on_cpu_scaled(struct evsel *evsel, int cpu_map_idx return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, true); } -int evsel__parse_sample(struct evsel *evsel, union perf_event *event, - struct perf_sample *sample); +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap); + +static inline int evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data) +{ + return __evsel__parse_sample(evsel, event, data, evsel->needs_swap); +} int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, u64 *timestamp); @@ -467,6 +502,7 @@ static inline bool evsel__is_group_event(struct evsel *evsel) return evsel__is_group_leader(evsel) && evsel->core.nr_members > 1; } +bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel); bool evsel__is_function_event(struct evsel *evsel); static inline bool evsel__is_bpf_output(struct evsel *evsel) @@ -498,7 +534,7 @@ for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_member(_evsel, _leader) \ - for_each_group_member_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_member_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) /* Iterates group WITH the leader. */ #define for_each_group_evsel_head(_evsel, _leader, _head) \ @@ -508,7 +544,7 @@ for ((_evsel) = _leader; \ (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) #define for_each_group_evsel(_evsel, _leader) \ - for_each_group_evsel_head(_evsel, _leader, &(_leader)->evlist->core.entries) + for_each_group_evsel_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) static inline bool evsel__has_branch_callstack(const struct evsel *evsel) { @@ -578,6 +614,7 @@ void evsel__uniquify_counter(struct evsel *counter); ((((src) >> (pos)) & ((1ull << (size)) - 1)) << (63 - ((pos) + (size) - 1))) u64 evsel__bitfield_swap_branch_flags(u64 value); +bool evsel__config_exists(const struct evsel *evsel, const char *config_name); int evsel__get_config_val(const struct evsel *evsel, const char *config_name, u64 *val); void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index 5521d00bff2c..0f7a25500a44 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -146,7 +146,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, sym = node->ms.sym; map = node->ms.map; - if (sym && sym->ignore && print_skip_ignored) + if (sym && symbol__ignore(sym) && print_skip_ignored) goto next; printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); @@ -182,7 +182,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, addr_location__exit(&node_al); } - if (print_dso && (!sym || !sym->inlined)) + if (print_dso && (!sym || !symbol__inlined(sym))) printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); if (print_srcline) { @@ -192,7 +192,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, printed += map__fprintf_srcline(map, addr, "\n ", fp); } - if (sym && sym->inlined) + if (sym && symbol__inlined(sym)) printed += fprintf(fp, " (inlined)"); if (!print_oneline) diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 644769e92708..8aef3c7418f7 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -27,6 +27,7 @@ struct expr_id_data { struct { double val; int source_count; + int aggr_nr; } val; struct { double val; @@ -151,8 +152,8 @@ int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val) } /* Caller must make sure id is allocated */ -int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, - double val, int source_count) +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr) { struct expr_id_data *data_ptr = NULL, *old_data = NULL; char *old_key = NULL; @@ -163,6 +164,7 @@ int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, return -ENOMEM; data_ptr->val.val = val; data_ptr->val.source_count = source_count; + data_ptr->val.aggr_nr = aggr_nr; data_ptr->kind = EXPR_ID_DATA__VALUE; ret = hashmap__set(ctx->ids, id, data_ptr, &old_key, &old_data); @@ -171,12 +173,20 @@ int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, } else if (old_data) { data_ptr->val.val += old_data->val.val; data_ptr->val.source_count += old_data->val.source_count; + data_ptr->val.aggr_nr += old_data->val.aggr_nr; } free(old_key); free(old_data); return ret; } +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count) +{ + return expr__add_id_val_source_count_aggr_nr(ctx, id, val, source_count, 1); +} + int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) { struct expr_id_data *data_ptr = NULL, *old_data = NULL; @@ -390,8 +400,16 @@ double expr_id_data__value(const struct expr_id_data *data) double expr_id_data__source_count(const struct expr_id_data *data) { - assert(data->kind == EXPR_ID_DATA__VALUE); - return data->val.source_count; + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.source_count; + return 1.0; +} + +double expr_id_data__aggr_nr(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.aggr_nr; + return 1.0; } double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) @@ -450,7 +468,7 @@ double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const ret = parse_event(tmp, id) ? 0 : 1; } out: - evlist__delete(tmp); + evlist__put(tmp); return ret; } diff --git a/tools/perf/util/expr.h b/tools/perf/util/expr.h index c0cec29ddc29..ed12e4007d2d 100644 --- a/tools/perf/util/expr.h +++ b/tools/perf/util/expr.h @@ -36,7 +36,9 @@ void expr__del_id(struct expr_parse_ctx *ctx, const char *id); int expr__add_id(struct expr_parse_ctx *ctx, const char *id); int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val); int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, - double val, int source_count); + double val, int source_count); +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr); int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref); int expr__get_id(struct expr_parse_ctx *ctx, const char *id, struct expr_id_data **data); @@ -53,6 +55,8 @@ int expr__find_ids(const char *expr, const char *one, double expr_id_data__value(const struct expr_id_data *data); double expr_id_data__source_count(const struct expr_id_data *data); +double expr_id_data__aggr_nr(const struct expr_id_data *data); + double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx); double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); diff --git a/tools/perf/util/expr.l b/tools/perf/util/expr.l index a2fc43159ee9..f16ccff278d0 100644 --- a/tools/perf/util/expr.l +++ b/tools/perf/util/expr.l @@ -121,6 +121,7 @@ min { return MIN; } if { return IF; } else { return ELSE; } source_count { return SOURCE_COUNT; } +aggr_nr { return AGGR_NR; } has_event { return HAS_EVENT; } strcmp_cpuid_str { return STRCMP_CPUID_STR; } NaN { return nan_value(yyscanner); } diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y index e364790babb5..e20f649354cf 100644 --- a/tools/perf/util/expr.y +++ b/tools/perf/util/expr.y @@ -41,7 +41,7 @@ int expr_lex(YYSTYPE * yylval_param , void *yyscanner); } ids; } -%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR +%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT AGGR_NR HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR %left MIN MAX IF %left '|' %left '^' @@ -87,8 +87,14 @@ static struct ids union_expr(struct ids ids1, struct ids ids2) return result; } +enum expr_id_kind { + EXPR_ID_KIND__VALUE, + EXPR_ID_KIND__SOURCE_COUNT, + EXPR_ID_KIND__AGGR_NR, +}; + static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, - bool compute_ids, bool source_count) + bool compute_ids, enum expr_id_kind kind) { struct ids result; @@ -101,9 +107,12 @@ static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, result.val = NAN; if (expr__resolve_id(ctx, id, &data) == 0) { - result.val = source_count - ? expr_id_data__source_count(data) - : expr_id_data__value(data); + if (kind == EXPR_ID_KIND__SOURCE_COUNT) + result.val = expr_id_data__source_count(data); + else if (kind == EXPR_ID_KIND__AGGR_NR) + result.val = expr_id_data__aggr_nr(data); + else + result.val = expr_id_data__value(data); } result.ids = NULL; free(id); @@ -201,8 +210,9 @@ expr: NUMBER $$.val = $1; $$.ids = NULL; } -| ID { $$ = handle_id(ctx, $1, compute_ids, /*source_count=*/false); } -| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, /*source_count=*/true); } +| ID { $$ = handle_id(ctx, $1, compute_ids, EXPR_ID_KIND__VALUE); } +| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__SOURCE_COUNT); } +| AGGR_NR '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__AGGR_NR); } | HAS_EVENT '(' ID ')' { $$.val = expr__has_event(ctx, compute_ids, $3); diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index f30e48eb3fc3..e90e541f546b 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include "string2.h" #include #include @@ -47,6 +48,7 @@ #include #include "asm/bug.h" #include "tool.h" +#include "../perf.h" #include "time-utils.h" #include "units.h" #include "util/util.h" // perf_exe() @@ -63,6 +65,25 @@ #include #endif +/* + * nr_ids * sizeof(struct perf_sample_id) must not overflow + * size_t on 32-bit; the struct is ~104 bytes (32-bit) or + * ~184 bytes (64-bit), so 1<<24 (16M) keeps the product + * under 2 GB on 32-bit. + * + * This is a per-attribute cap only — the total across all + * attributes is not capped because legitimate high-core-count + * workloads (e.g. 5000 tracepoints × 4096 CPUs) can exceed + * a single-attribute limit. + */ +#define MAX_IDS_PER_ATTR (1 << 24) +/* + * Cap nr_attrs to prevent resource exhaustion from crafted + * files. 65536 is well beyond any real workload (perf stat + * typically uses < 100 events) but prevents u64-to-int + * truncation on the attr count. + */ +#define MAX_NR_ATTRS (1 << 16) #define MAX_BPF_DATA_LEN (256 * 1024 * 1024) #define MAX_BPF_PROGS 131072 #define MAX_CACHE_ENTRIES 32768 @@ -158,15 +179,25 @@ int do_write(struct feat_fd *ff, const void *buf, size_t size) /* Return: 0 if succeeded, -ERR if failed. */ static int do_write_bitmap(struct feat_fd *ff, unsigned long *set, u64 size) { - u64 *p = (u64 *) set; + size_t byte_size = BITS_TO_LONGS(size) * sizeof(unsigned long); int i, ret; ret = do_write(ff, &size, sizeof(size)); if (ret < 0) return ret; + /* + * The on-disk format uses u64 elements, but the in-memory bitmap + * uses unsigned long, which is only 4 bytes on 32-bit architectures. + * Copy with bounded size so the last element doesn't read past the + * bitmap allocation when BITS_TO_LONGS(size) is odd. + */ for (i = 0; (u64) i < BITS_TO_U64(size); i++) { - ret = do_write(ff, p + i, sizeof(*p)); + u64 val = 0; + size_t off = i * sizeof(val); + + memcpy(&val, (char *)set + off, min(sizeof(val), byte_size - off)); + ret = do_write(ff, &val, sizeof(val)); if (ret < 0) return ret; } @@ -213,23 +244,32 @@ static int __do_read_fd(struct feat_fd *ff, void *addr, ssize_t size) if (ret != size) return ret < 0 ? (int)ret : -1; + ff->offset += size; return 0; } static int __do_read_buf(struct feat_fd *ff, void *addr, ssize_t size) { - if (size > (ssize_t)ff->size - ff->offset) - return -1; - memcpy(addr, ff->buf + ff->offset, size); ff->offset += size; return 0; - } static int __do_read(struct feat_fd *ff, void *addr, ssize_t size) { + /* + * Reject negative sizes, which on 32-bit can occur when a + * u32 >= 0x80000000 is passed as ssize_t. The cast to + * ssize_t is safe because perf_header__process_sections() + * validates that each section fits within the file size + * before any feature callback reaches here, and only + * feature sections (metadata like build IDs, topology, etc.) + * use this path — these cannot legitimately approach 2GB. + */ + if (size < 0 || size > (ssize_t)ff->size - ff->offset) + return -1; + if (!ff->buf) return __do_read_fd(ff, addr, size); return __do_read_buf(ff, addr, size); @@ -269,16 +309,25 @@ static char *do_read_string(struct feat_fd *ff) if (do_read_u32(ff, &len)) return NULL; + /* At least the null terminator. */ + if (len < 1 || len > ff->size - ff->offset) { + pr_debug("do_read_string: invalid length %u (remaining %zu)\n", + len, (size_t)(ff->size - ff->offset)); + return NULL; + } + buf = malloc(len); if (!buf) return NULL; if (!__do_read(ff, buf, len)) { /* - * strings are padded by zeroes - * thus the actual strlen of buf - * may be less than len + * do_write_string() writes len including the null + * terminator, padded to NAME_ALIGN. Ensure the + * string is always null-terminated even if the file + * data has been tampered with. */ + buf[len - 1] = '\0'; return buf; } @@ -297,7 +346,20 @@ static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) if (ret) return ret; - set = bitmap_zalloc(size); + /* Bitmap APIs use int for nbits; reject u64 values that truncate. */ + if (size > INT_MAX || + BITS_TO_U64(size) > (ff->size - ff->offset) / sizeof(u64)) { + pr_debug("do_read_bitmap: size %" PRIu64 " exceeds section bounds\n", size); + return -1; + } + + /* + * bitmap_zalloc() allocates in unsigned long units, which are only + * 4 bytes on 32-bit architectures. The read loop below casts the + * buffer to u64 * and writes 8-byte elements, so allocate in u64 + * units to ensure the buffer is large enough. + */ + set = calloc(BITS_TO_U64(size), sizeof(u64)); if (!set) return -ENOMEM; @@ -323,7 +385,7 @@ static int write_tracing_data(struct feat_fd *ff, return -1; #ifdef HAVE_LIBTRACEEVENT - return read_tracing_data(ff->fd, &evlist->core.entries); + return read_tracing_data(ff->fd, &evlist__core(evlist)->entries); #else pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n"); return -1; @@ -370,34 +432,44 @@ static int write_osrelease(struct feat_fd *ff, struct evlist *evlist __maybe_unused) { struct utsname uts; - int ret; + const char *release = NULL; - ret = uname(&uts); - if (ret < 0) - return -1; + if (evlist__session(evlist)) + release = perf_env__os_release(perf_session__env(evlist__session(evlist))); - return do_write_string(ff, uts.release); + if (!release) { + int ret = uname(&uts); + + if (ret < 0) + return -1; + release = uts.release; + } + return do_write_string(ff, release); } -static int write_arch(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_arch(struct feat_fd *ff, struct evlist *evlist) { struct utsname uts; - int ret; + const char *arch = NULL; - ret = uname(&uts); - if (ret < 0) - return -1; + if (evlist__session(evlist)) + arch = perf_env__arch(perf_session__env(evlist__session(evlist))); + + if (!arch) { + int ret = uname(&uts); - return do_write_string(ff, uts.machine); + if (ret < 0) + return -1; + arch = uts.machine; + } + return do_write_string(ff, arch); } -static int write_e_machine(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) { /* e_machine expanded from 16 to 32-bits for alignment. */ uint32_t e_flags; - uint32_t e_machine = perf_session__e_machine(evlist->session, &e_flags); + uint32_t e_machine = perf_session__e_machine(evlist__session(evlist), &e_flags); int ret; ret = do_write(ff, &e_machine, sizeof(e_machine)); @@ -533,7 +605,7 @@ static int write_event_desc(struct feat_fd *ff, u32 nre, nri, sz; int ret; - nre = evlist->core.nr_entries; + nre = evlist__nr_entries(evlist); /* * write number of events @@ -915,7 +987,7 @@ int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) { - struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); char buffer[64]; int ret; @@ -1348,14 +1420,14 @@ static int write_sample_time(struct feat_fd *ff, struct evlist *evlist) { int ret; + u64 data = evlist__first_sample_time(evlist); - ret = do_write(ff, &evlist->first_sample_time, - sizeof(evlist->first_sample_time)); + ret = do_write(ff, &data, sizeof(data)); if (ret < 0) return ret; - return do_write(ff, &evlist->last_sample_time, - sizeof(evlist->last_sample_time)); + data = evlist__last_sample_time(evlist); + return do_write(ff, &data, sizeof(data)); } @@ -2035,8 +2107,7 @@ static void print_bpf_prog_info(struct feat_fd *ff __maybe_unused, FILE *fp) node = rb_entry(next, struct bpf_prog_info_node, rb_node); next = rb_next(&node->rb_node); - __bpf_event__print_bpf_prog_info(&node->info_linear->info, - env, fp); + __bpf_event__print_bpf_prog_info(node->info_linear, env, fp); } up_read(&env->bpf_progs.lock); @@ -2135,9 +2206,28 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (do_read_u32(ff, &nre)) goto error; + /* Size of each of the nre attributes. */ if (do_read_u32(ff, &sz)) goto error; + /* + * Require at least one event with an attr no smaller than the + * first published struct, and reject sz values where + * sz + sizeof(u32) would overflow size_t (possible on 32-bit) + * or nre == UINT32_MAX where nre + 1 wraps to 0 in the calloc. + * + * The minimum section footprint per event is sz bytes for the + * attr plus a u32 for the id count, check that nre events fit. + */ + if (!nre || sz < PERF_ATTR_SIZE_VER0 || + sz > ff->size || (size_t)sz > SIZE_MAX - sizeof(u32) || + nre == UINT32_MAX || + nre > (ff->size - ff->offset) / (sz + sizeof(u32))) { + pr_err("Invalid HEADER_EVENT_DESC: nre=%u sz=%u (min %d)\n", + nre, sz, PERF_ATTR_SIZE_VER0); + goto error; + } + /* buffer to hold on file attr struct */ buf = malloc(sz); if (!buf) @@ -2153,6 +2243,9 @@ static struct evsel *read_event_desc(struct feat_fd *ff) msz = sz; for (i = 0, evsel = events; i < nre; evsel++, i++) { + struct perf_event_attr *attr = buf; + u32 attr_size; + evsel->core.idx = i; /* @@ -2162,6 +2255,32 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (__do_read(ff, buf, sz)) goto error; + /* Reject before attr_swap to prevent OOB via bswap_safe() */ + attr_size = ff->ph->needs_swap ? bswap_32(attr->size) : attr->size; + /* ABI0: size == 0 means the producer didn't set it */ + if (!attr_size) { + attr_size = PERF_ATTR_SIZE_VER0; + /* + * Write back so free_event_desc() doesn't + * treat this event as the end-of-array sentinel + * (it iterates while attr.size != 0). + * + * Only for native — the swap path must NOT + * write native-endian VER0 here because + * perf_event__attr_swap() would re-swap it + * to 0x40000000, defeating bswap_safe() bounds. + * perf_event__attr_swap() has its own ABI0 + * fallback that sets VER0 after swapping. + */ + if (!ff->ph->needs_swap) + attr->size = attr_size; + } + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size > sz) { + pr_err("Event %d attr.size (%u) invalid (min: %d, max: %u)\n", + i, attr_size, PERF_ATTR_SIZE_VER0, sz); + goto error; + } + if (ff->ph->needs_swap) perf_event__attr_swap(buf); @@ -2183,6 +2302,12 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (!nr) continue; + /* Prevent oversized allocation from crafted nr */ + if (nr > (ff->size - ff->offset) / sizeof(*id)) { + pr_err("Event %d: id count %u exceeds remaining section\n", i, nr); + goto error; + } + id = calloc(nr, sizeof(*id)); if (!id) goto error; @@ -2341,15 +2466,16 @@ static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) static void print_pmu_caps(struct feat_fd *ff, FILE *fp) { struct perf_env *env = &ff->ph->env; - struct pmu_caps *pmu_caps; + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); for (int i = 0; i < env->nr_pmus_with_caps; i++) { - pmu_caps = &env->pmu_caps[i]; + struct pmu_caps *pmu_caps = &env->pmu_caps[i]; + __print_pmu_caps(fp, pmu_caps->nr_caps, pmu_caps->caps, pmu_caps->pmu_name); } - if (strcmp(perf_env__arch(env), "x86") == 0 && + if ((e_machine == EM_X86_64 || e_machine == EM_386) && perf_env__has_pmu_mapping(env, "ibs_op")) { char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise"); @@ -2425,16 +2551,16 @@ static void print_sample_time(struct feat_fd *ff, FILE *fp) session = container_of(ff->ph, struct perf_session, header); - timestamp__scnprintf_usec(session->evlist->first_sample_time, + timestamp__scnprintf_usec(evlist__first_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of first sample : %s\n", time_buf); - timestamp__scnprintf_usec(session->evlist->last_sample_time, + timestamp__scnprintf_usec(evlist__last_sample_time(session->evlist), time_buf, sizeof(time_buf)); fprintf(fp, "# time of last sample : %s\n", time_buf); - d = (double)(session->evlist->last_sample_time - - session->evlist->first_sample_time) / NSEC_PER_MSEC; + d = (double)(evlist__last_sample_time(session->evlist) - + evlist__first_sample_time(session->evlist)) / NSEC_PER_MSEC; fprintf(fp, "# sample duration : %10.3f ms\n", d); } @@ -2578,7 +2704,13 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, } old_bev; struct perf_record_header_build_id bev; char filename[PATH_MAX]; - u64 limit = offset + size; + u64 limit; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; while (offset < limit) { ssize_t len; @@ -2589,6 +2721,10 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, if (header->needs_swap) perf_event_header__bswap(&old_bev.header); + /* size == 0 loops forever; size > remaining reads past section */ + if (old_bev.header.size == 0 || old_bev.header.size > limit - offset) + return -1; + len = old_bev.header.size - sizeof(old_bev); if (len < 0 || len >= PATH_MAX) { pr_warning("invalid build_id filename length %zd\n", len); @@ -2597,6 +2733,13 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, if (readn(input, filename, len) != len) return -1; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; bev.header = old_bev.header; @@ -2624,17 +2767,32 @@ static int perf_header__read_build_ids(struct perf_header *header, struct perf_session *session = container_of(header, struct perf_session, header); struct perf_record_header_build_id bev; char filename[PATH_MAX]; - u64 limit = offset + size, orig_offset = offset; + u64 limit, orig_offset = offset; int err = -1; + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + while (offset < limit) { ssize_t len; if (readn(input, &bev, sizeof(bev)) != sizeof(bev)) goto out; - if (header->needs_swap) + if (header->needs_swap) { perf_event_header__bswap(&bev.header); + bev.pid = bswap_32(bev.pid); + } + + /* + * size == 0 would loop forever (offset never advances); + * size > remaining would read past the section boundary. + */ + if (bev.header.size == 0 || bev.header.size > limit - offset) + goto out; len = bev.header.size - sizeof(bev); if (len < 0 || len >= PATH_MAX) { @@ -2644,6 +2802,13 @@ static int perf_header__read_build_ids(struct perf_header *header, if (readn(input, filename, len) != len) goto out; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; /* * The a1645ce1 changeset: * @@ -2657,7 +2822,9 @@ static int perf_header__read_build_ids(struct perf_header *header, * '[kernel.kallsyms]' string for the kernel build-id has the * first 4 characters chopped off (where the pid_t sits). */ - if (memcmp(filename, "nel.kallsyms]", 13) == 0) { + /* Guard short filenames against memcmp reading past the buffer */ + if (len >= (ssize_t)sizeof("nel.kallsyms]") - 1 && + memcmp(filename, "nel.kallsyms]", sizeof("nel.kallsyms]") - 1) == 0) { if (lseek(input, orig_offset, SEEK_SET) == (off_t)-1) return -1; return perf_header__read_build_ids_abi_quirk(header, input, offset, size); @@ -2684,10 +2851,18 @@ static int process_##__feat(struct feat_fd *ff, void *data __maybe_unused) \ FEAT_PROCESS_STR_FUN(hostname, hostname); FEAT_PROCESS_STR_FUN(osrelease, os_release); FEAT_PROCESS_STR_FUN(version, version); -FEAT_PROCESS_STR_FUN(arch, arch); FEAT_PROCESS_STR_FUN(cpudesc, cpu_desc); FEAT_PROCESS_STR_FUN(cpuid, cpuid); +static int process_arch(struct feat_fd *ff, void *data __maybe_unused) +{ + free(ff->ph->env.arch); + ff->ph->env.arch = do_read_string(ff); + if (!ff->ph->env.arch) + return -ENOMEM; + return 0; +} + static int process_e_machine(struct feat_fd *ff, void *data __maybe_unused) { int ret; @@ -2706,14 +2881,21 @@ static int process_tracing_data(struct feat_fd *ff __maybe_unused, void *data __ return ret < 0 ? -1 : 0; #else - pr_err("ERROR: Trying to read tracing data without libtraceevent support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("Tracing data present but libtraceevent not available, skipping.\n"); + return 0; #endif } static int process_build_id(struct feat_fd *ff, void *data __maybe_unused) { - if (perf_header__read_build_ids(ff->ph, ff->fd, ff->offset, ff->size)) + /* lseek fails in pipe mode — fall back to ff->offset */ + off_t offset = lseek(ff->fd, 0, SEEK_CUR); + + if (offset == (off_t)-1) + offset = ff->offset; + + if (perf_header__read_build_ids(ff->ph, ff->fd, offset, ff->size)) pr_debug("Failed to read buildids, continuing...\n"); return 0; } @@ -2732,6 +2914,17 @@ static int process_nrcpus(struct feat_fd *ff, void *data __maybe_unused) if (ret) return ret; + /* + * Cap at 1M CPUs — generous for any real system but prevents + * stack overflow from VLA allocations sized by nr_cpus_avail + * (e.g. DECLARE_BITMAP in builtin-c2c.c node_entry()). + */ + if (nr_cpus_avail > (1U << 20)) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_avail (%u) exceeds maximum (%u)\n", + nr_cpus_avail, 1U << 20); + return -1; + } + if (nr_cpus_online > nr_cpus_avail) { pr_err("Invalid HEADER_NRCPUS: nr_cpus_online (%u) > nr_cpus_avail (%u)\n", nr_cpus_online, nr_cpus_avail); @@ -3326,8 +3519,8 @@ static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused) if (ret) return -1; - session->evlist->first_sample_time = first_sample_time; - session->evlist->last_sample_time = last_sample_time; + evlist__set_first_sample_time(session->evlist, first_sample_time); + evlist__set_last_sample_time(session->evlist, last_sample_time); return 0; } @@ -3357,7 +3550,8 @@ static int process_mem_topology(struct feat_fd *ff, return -1; } - if (ff->size < 3 * sizeof(u64) + nr * 2 * sizeof(u64)) { + /* Per node: node_id(u64) + mem_size(u64) + bitmap_nr_bits(u64) */ + if (ff->size < 3 * sizeof(u64) + nr * 3 * sizeof(u64)) { pr_err("Invalid HEADER_MEM_TOPOLOGY: section too small (%zu) for %llu nodes\n", ff->size, (unsigned long long)nr); return -1; @@ -3392,7 +3586,7 @@ static int process_mem_topology(struct feat_fd *ff, out: if (ret) - free(nodes); + memory_node__delete_nodes(nodes, nr); return ret; } @@ -3601,8 +3795,9 @@ static int process_bpf_prog_info(struct feat_fd *ff __maybe_unused, void *data _ up_write(&env->bpf_progs.lock); return err; #else - pr_err("ERROR: Trying to read bpf_prog_info without libbpf support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BPF prog info present but libbpf not available, skipping.\n"); + return 0; #endif // HAVE_LIBBPF_SUPPORT } @@ -3670,8 +3865,9 @@ static int process_bpf_btf(struct feat_fd *ff __maybe_unused, void *data __mayb free(node); return err; #else - pr_err("ERROR: Trying to read btf data without libbpf support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BTF data present but libbpf not available, skipping.\n"); + return 0; #endif // HAVE_LIBBPF_SUPPORT } @@ -3695,6 +3891,23 @@ static int process_compressed(struct feat_fd *ff, if (do_read_u32(ff, &(env->comp_mmap_len))) return -1; + /* + * FIXME: perf.data should record the recording system's page + * size — it affects mmap buffer alignment, sample addresses, + * and data_page_size/code_page_size interpretation. Without + * it we assume 4K (the smallest Linux page size) as a safe + * minimum alignment for comp_mmap_len validation. + * + * No upper-bound cap: perf_session__process_compressed_event() + * checks decomp_len + sizeof(struct decomp) against SIZE_MAX + * before allocating, which handles 32-bit safety. + */ + if (env->comp_mmap_len < 4096 || env->comp_mmap_len % 4096) { + pr_err("Invalid HEADER_COMPRESSED: comp_mmap_len (%u) must be a 4K-aligned value >= 4096\n", + env->comp_mmap_len); + return -1; + } + return 0; } @@ -4088,6 +4301,7 @@ static int perf_file_section__fprintf_info(struct perf_file_section *section, ff = (struct feat_fd) { .fd = fd, .ph = ph, + .size = section->size, }; if (!feat_ops[feat].full_only || hd->full) @@ -4396,7 +4610,7 @@ int perf_session__write_header(struct perf_session *session, /*write_attrs_after_data=*/false); } -size_t perf_session__data_offset(const struct evlist *evlist) +size_t perf_session__data_offset(struct evlist *evlist) { struct evsel *evsel; size_t data_offset; @@ -4405,7 +4619,7 @@ size_t perf_session__data_offset(const struct evlist *evlist) evlist__for_each_entry(evlist, evsel) { data_offset += evsel->core.ids * sizeof(u64); } - data_offset += evlist->core.nr_entries * sizeof(struct perf_file_attr); + data_offset += evlist__nr_entries(evlist) * sizeof(struct perf_file_attr); return data_offset; } @@ -4423,8 +4637,13 @@ int perf_session__inject_header(struct perf_session *session, static int perf_header__getbuffer64(struct perf_header *header, int fd, void *buf, size_t size) { - if (readn(fd, buf, size) <= 0) + ssize_t n = readn(fd, buf, size); + + if (n <= 0) { + if (n == 0) + errno = EIO; return -1; + } if (header->needs_swap) mem_bswap_64(buf, size); @@ -4443,6 +4662,7 @@ int perf_header__process_sections(struct perf_header *header, int fd, int sec_size; int feat; int err; + struct stat st; nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); if (!nr_sections) @@ -4460,7 +4680,29 @@ int perf_header__process_sections(struct perf_header *header, int fd, if (err < 0) goto out_free; + if (fstat(fd, &st) < 0) { + pr_err("Failed to stat the perf data file\n"); + err = -1; + goto out_free; + } + for_each_set_bit(feat, header->adds_features, header->last_feat) { + /* + * FIXME: block devices have st_size == 0, so we skip + * bounds checking entirely. Historically perf never + * prevented using a block device as input, but it + * probably should — there's no valid use case for it + * and it bypasses all file-size validation. + */ + if (S_ISREG(st.st_mode) && + (sec->offset > (u64)st.st_size || + sec->size > (u64)st.st_size - sec->offset)) { + pr_err("Feature %s (%d) section extends past EOF (offset=%" PRIu64 ", size=%" PRIu64 ", file=%" PRIu64 ")\n", + header_feat__name(feat), feat, + sec->offset, sec->size, (u64)st.st_size); + err = -1; + goto out_free; + } err = process(sec++, header, feat, fd, data); if (err < 0) goto out_free; @@ -4687,7 +4929,7 @@ static int perf_file_section__process(struct perf_file_section *section, .fd = fd, .ph = ph, .size = section->size, - .offset = section->offset, + .offset = 0, }; if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { @@ -4758,6 +5000,8 @@ static int read_attr(int fd, struct perf_header *ph, if (ret <= 0) { pr_debug("cannot read %d bytes of header attr\n", PERF_ATTR_SIZE_VER0); + if (ret == 0) + errno = EIO; return -1; } @@ -4770,9 +5014,15 @@ static int read_attr(int fd, struct perf_header *ph, if (sz == 0) { /* assume ABI0 */ sz = PERF_ATTR_SIZE_VER0; + } else if (sz < PERF_ATTR_SIZE_VER0) { + pr_debug("bad attr size %zu, expected at least %d\n", + sz, PERF_ATTR_SIZE_VER0); + errno = EINVAL; + return -1; } else if (sz > our_sz) { pr_debug("file uses a more recent and unsupported ABI" " (%zu bytes extra)\n", sz - our_sz); + errno = EINVAL; return -1; } /* what we have not yet read and that we know about */ @@ -4782,11 +5032,21 @@ static int read_attr(int fd, struct perf_header *ph, ptr += PERF_ATTR_SIZE_VER0; ret = readn(fd, ptr, left); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } } /* read perf_file_section, ids are read in caller */ ret = readn(fd, &f_attr->ids, sizeof(f_attr->ids)); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } - return ret <= 0 ? -1 : 0; + return 0; } #ifdef HAVE_LIBTRACEEVENT @@ -4842,14 +5102,15 @@ int perf_session__read_header(struct perf_session *session) struct perf_file_header f_header; struct perf_file_attr f_attr; u64 f_id; - int nr_attrs, nr_ids, i, j, err; + struct stat input_stat; + int nr_attrs, nr_ids, i, j, err = -ENOMEM; int fd = perf_data__fd(data); session->evlist = evlist__new(); if (session->evlist == NULL) return -ENOMEM; - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.env = &header->env; /* @@ -4862,6 +5123,7 @@ int perf_session__read_header(struct perf_session *session) return err; } + err = -ENOMEM; if (perf_file_header__read(&f_header, header, fd) < 0) return -EINVAL; @@ -4889,6 +5151,15 @@ int perf_session__read_header(struct perf_session *session) return -EINVAL; } + if (fstat(fd, &input_stat) < 0) + return -errno; + + /* Check before assigning to int to avoid u64-to-int truncation */ + if (f_header.attrs.size / f_header.attr_size > MAX_NR_ATTRS) { + pr_err("Too many attributes: %" PRIu64 " (max %d)\n", + f_header.attrs.size / f_header.attr_size, MAX_NR_ATTRS); + return -EINVAL; + } nr_attrs = f_header.attrs.size / f_header.attr_size; lseek(fd, f_header.attrs.offset, SEEK_SET); @@ -4905,16 +5176,55 @@ int perf_session__read_header(struct perf_session *session) perf_event__attr_swap(&f_attr.attr); } + /* + * Validate ids section: must be aligned to u64, and + * the count must fit in an int to avoid truncation in + * nr_ids and size_t overflow in perf_evsel__alloc_id() + * on 32-bit architectures. + */ + if (f_attr.ids.size % sizeof(u64)) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_put_evlist; + } + + /* + * Cap the ID count to avoid int truncation of nr_ids + * on 64-bit and size_t overflow in the allocation + * paths (nr_ids * sizeof(u64), nr_ids * + * sizeof(struct perf_sample_id)) on 32-bit. + */ + if (f_attr.ids.size / sizeof(u64) > MAX_IDS_PER_ATTR) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_put_evlist; + } + + /* + * FIXME: see perf_header__process_sections() — block + * devices bypass this check because st_size is 0. + */ + if (S_ISREG(input_stat.st_mode) && + (f_attr.ids.offset > (u64)input_stat.st_size || + f_attr.ids.size > (u64)input_stat.st_size - f_attr.ids.offset)) { + pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", + i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); + err = -EINVAL; + goto out_put_evlist; + } + tmp = lseek(fd, 0, SEEK_CUR); evsel = evsel__new(&f_attr.attr); if (evsel == NULL) - goto out_delete_evlist; + goto out_put_evlist; evsel->needs_swap = header->needs_swap; /* * Do it before so that if perf_evsel__alloc_id fails, this - * entry gets purged too at evlist__delete(). + * entry gets purged too at evlist__put(). */ evlist__add(session->evlist, evsel); @@ -4925,7 +5235,7 @@ int perf_session__read_header(struct perf_session *session) * hattr->ids threads. */ if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids)) - goto out_delete_evlist; + goto out_put_evlist; lseek(fd, f_attr.ids.offset, SEEK_SET); @@ -4933,30 +5243,70 @@ int perf_session__read_header(struct perf_session *session) if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id))) goto out_errno; - perf_evlist__id_add(&session->evlist->core, &evsel->core, 0, j, f_id); + perf_evlist__id_add(evlist__core(session->evlist), + &evsel->core, 0, j, f_id); } lseek(fd, tmp, SEEK_SET); } + /* + * Skip feature section processing for truncated files + * (data.size == 0 means recording was interrupted). The + * section table is unreliable in that case, and the event + * data can still be processed without the feature headers. + * Clear the bitmap so has_feat() returns false and tools + * use their "feature not present" fallbacks instead of + * accessing uninitialized env fields. + */ + if (f_header.data.size == 0) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + } else { #ifdef HAVE_LIBTRACEEVENT - perf_header__process_sections(header, fd, &session->tevent, - perf_file_section__process); + err = perf_header__process_sections(header, fd, &session->tevent, + perf_file_section__process); + if (err < 0) + goto out_put_evlist; - if (evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent)) - goto out_delete_evlist; + if (evlist__prepare_tracepoint_events(session->evlist, + session->tevent.pevent)) { + err = -ENOMEM; + goto out_put_evlist; + } #else - perf_header__process_sections(header, fd, NULL, perf_file_section__process); + err = perf_header__process_sections(header, fd, NULL, + perf_file_section__process); + if (err < 0) + goto out_put_evlist; #endif + } + + /* + * Without nr_cpus_avail the sample CPU bounds check in + * perf_session__deliver_event() is bypassed, allowing crafted + * CPU IDs to reach downstream consumers that index fixed-size + * arrays (timechart, kwork, sched — all sized MAX_NR_CPUS). + * + * This can happen with truncated files (interrupted recording + * loses all feature sections), very old files that predate + * HEADER_NRCPUS, or crafted files that omit it. Fall back to + * MAX_NR_CPUS so the bounds check is still effective — any + * CPU ID below that limit is safe for all downstream arrays. + */ + if (header->env.nr_cpus_avail == 0) { + header->env.nr_cpus_avail = MAX_NR_CPUS; + pr_warning("WARNING: perf.data is missing HEADER_NRCPUS, using MAX_NR_CPUS (%d) as CPU bound\n", + MAX_NR_CPUS); + } return 0; out_errno: return -errno; -out_delete_evlist: - evlist__delete(session->evlist); +out_put_evlist: + evlist__put(session->evlist); session->evlist = NULL; - return -ENOMEM; + return err; } int perf_event__process_feature(const struct perf_tool *tool __maybe_unused, @@ -5059,15 +5409,76 @@ size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) switch (ev->type) { case PERF_EVENT_UPDATE__SCALE: + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + ret += fprintf(fp, "... scale: (truncated)\n"); + break; + } ret += fprintf(fp, "... scale: %f\n", ev->scale.scale); break; case PERF_EVENT_UPDATE__UNIT: - ret += fprintf(fp, "... unit: %s\n", ev->unit); - break; - case PERF_EVENT_UPDATE__NAME: - ret += fprintf(fp, "... name: %s\n", ev->name); + case PERF_EVENT_UPDATE__NAME: { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + ret += fprintf(fp, "... %s: (unterminated)\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + break; + } + ret += fprintf(fp, "... %s: %s\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name", + ev->unit); break; - case PERF_EVENT_UPDATE__CPUS: + } + case PERF_EVENT_UPDATE__CPUS: { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + u32 cpus_payload; + + if (event->header.size < cpus_off + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map)) { + ret += fprintf(fp, "... cpus: (truncated)\n"); + break; + } + + /* + * Validate nr against payload — this function may be + * called from the stub handler (dump_trace path) which + * bypasses perf_event__process_event_update() validation. + */ + cpus_payload = event->header.size - cpus_off; + if (ev->cpus.cpus.type == PERF_CPU_MAP__CPUS) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, cpus_data.cpu) || + ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + ret += fprintf(fp, "... cpus: nr %u exceeds payload\n", + ev->cpus.cpus.cpus_data.nr); + break; + } + } else if (ev->cpus.cpus.type == PERF_CPU_MAP__MASK) { + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask32_data.nr); + break; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask64_data.nr); + break; + } + } + } + ret += fprintf(fp, "... "); map = cpu_map__new_data(&ev->cpus.cpus); @@ -5077,6 +5488,7 @@ size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) } else ret += fprintf(fp, "failed to get cpus\n"); break; + } default: ret += fprintf(fp, "... unknown type\n"); break; @@ -5094,11 +5506,42 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct evlist **pevlist) { - u32 i, n_ids; + struct perf_event_attr attr; + u32 i, n_ids, raw_attr_size; u64 *ids; + size_t attr_size, copy_size; struct evsel *evsel; struct evlist *evlist = *pevlist; + /* + * HEADER_ATTR event layout (pipe/inject mode): + * + * [header (8 bytes)] [attr (attr_size bytes)] [id0 id1 ... idN] + * |<------------------ header.size --------------------------->| + * + * attr_size varies across perf versions: VER0 = 64 bytes, + * current sizeof(struct perf_event_attr) = larger. A newer + * producer may emit a larger attr than we understand. + * + * attr.size == 0 (ABI0) means the producer didn't set it + * (e.g., bench/inject-buildid, older perf). Treat as VER0. + * + * Require 8-byte alignment so the u64 ID array is aligned + * and attr.size fits cleanly within the payload. + * + * Read attr.size once — the event may be on a shared mmap + * and re-reading could yield a different value. + */ + raw_attr_size = event->attr.attr.size; + if (event->header.size < sizeof(event->header) + PERF_ATTR_SIZE_VER0 || + (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || + raw_attr_size % sizeof(u64) || + raw_attr_size > event->header.size - sizeof(event->header)))) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (event size %u, min %d)\n", + raw_attr_size, event->header.size, PERF_ATTR_SIZE_VER0); + return -EINVAL; + } + if (dump_trace) perf_event__fprintf_attr(event, stdout); @@ -5108,13 +5551,46 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, return -ENOMEM; } - evsel = evsel__new(&event->attr.attr); + /* + * attr_size = footprint of the attr in the event — determines + * where the ID array starts. For ABI0, assume VER0 (64 bytes). + * + * copy_size = how much we copy into our local struct, capped at + * sizeof(attr) so a newer producer's larger attr doesn't + * overflow. Fields beyond copy_size are zeroed. + * + * Do NOT write attr_size back to the event — native-endian + * files use MAP_SHARED (read-only), writing would SIGSEGV. + * The swap path handles ABI0 in perf_event__attr_swap() + * which writes to the writable MAP_PRIVATE copy instead. + */ + attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; + copy_size = min(attr_size, sizeof(attr)); + memcpy(&attr, &event->attr.attr, copy_size); + if (copy_size < sizeof(attr)) + memset((void *)&attr + copy_size, 0, sizeof(attr) - copy_size); + + /* + * Normalize ABI0: the swap path sets attr.size = VER0 on the + * event, but the native path leaves it as 0. Set it on the + * local copy so perf inject re-synthesizes with consistent + * layout regardless of endianness. + */ + attr.size = attr_size; + + evsel = evsel__new(&attr); if (evsel == NULL) return -ENOMEM; evlist__add(evlist, evsel); - n_ids = event->header.size - sizeof(event->header) - event->attr.attr.size; + /* + * IDs occupy the remainder after header + attr. Use attr_size + * (not copy_size) — even if the producer's attr is larger than + * our struct, the IDs start after attr_size bytes in the event. + * Validation above guarantees attr_size <= payload size. + */ + n_ids = event->header.size - sizeof(event->header) - attr_size; n_ids = n_ids / sizeof(u64); /* * We don't have the cpu and thread maps on the header, so @@ -5124,9 +5600,15 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, if (perf_evsel__alloc_id(&evsel->core, 1, n_ids)) return -ENOMEM; - ids = perf_record_header_attr_id(event); + /* + * Locate IDs at attr_size bytes past the attr start in the + * event. Cannot use perf_record_header_attr_id() — that + * macro reads event->attr.attr.size, which is 0 for ABI0 + * on the native-endian path (no swap handler to fix it up). + */ + ids = (void *)&event->attr.attr + attr_size; for (i = 0; i < n_ids; i++) { - perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]); + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, i, ids[i]); } return 0; @@ -5141,6 +5623,83 @@ int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused struct evsel *evsel; struct perf_cpu_map *map; + /* + * Validate payload before dump_trace or processing — both + * paths access variant-specific fields without further checks. + */ + if (ev->type == PERF_EVENT_UPDATE__UNIT || + ev->type == PERF_EVENT_UPDATE__NAME) { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: %s not null-terminated, skipping\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: SCALE payload too small, skipping\n"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + size_t min_cpus = sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map); + u32 cpus_payload; + + if (event->header.size < cpus_off + min_cpus) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: CPUS payload too small, skipping\n"); + return 0; + } + + /* + * Validate per-variant nr against the remaining + * payload on the native path — the swap path clamps + * nr in perf_event__event_update_swap(), but native + * events are read-only and cannot be clamped in place. + * cpu_map__new_data() trusts nr for allocation and + * iteration, so unchecked values cause OOB reads. + */ + cpus_payload = event->header.size - cpus_off; + switch (ev->cpus.cpus.type) { + case PERF_CPU_MAP__CPUS: + if (ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + pr_warning("WARNING: EVENT_UPDATE CPUS: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.cpus_data.nr); + return 0; + } + break; + case PERF_CPU_MAP__MASK: + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask32_data.nr); + return 0; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask64_data.nr); + return 0; + } + } + break; + default: + break; + } + } + if (dump_trace) perf_event__fprintf_event_update(event, stdout); @@ -5172,6 +5731,7 @@ int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused evsel->core.pmu_cpus = map; } else pr_err("failed to get event_update cpus\n"); + break; default: break; } diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h index 86b1a72026d3..5e03f884b7cc 100644 --- a/tools/perf/util/header.h +++ b/tools/perf/util/header.h @@ -158,7 +158,7 @@ int perf_session__inject_header(struct perf_session *session, struct feat_copier *fc, bool write_attrs_after_data); -size_t perf_session__data_offset(const struct evlist *evlist); +size_t perf_session__data_offset(struct evlist *evlist); void perf_header__set_feat(struct perf_header *header, int feat); void perf_header__clear_feat(struct perf_header *header, int feat); diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 747fdc455c80..c93915625ee7 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -935,8 +935,8 @@ iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al { u64 cost; struct mem_info *mi = iter->mi; - struct hists *hists = evsel__hists(iter->evsel); struct perf_sample *sample = iter->sample; + struct hists *hists = evsel__hists(sample->evsel); struct hist_entry *he; if (mi == NULL) @@ -968,7 +968,7 @@ static int iter_finish_mem_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { - struct evsel *evsel = iter->evsel; + struct evsel *evsel = iter->sample->evsel; struct hists *hists = evsel__hists(evsel); struct hist_entry *he = iter->he; int err = -EINVAL; @@ -1036,9 +1036,9 @@ static int iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) { struct branch_info *bi; - struct evsel *evsel = iter->evsel; - struct hists *hists = evsel__hists(evsel); struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); struct hist_entry *he = NULL; int i = iter->curr; int err = 0; @@ -1078,7 +1078,7 @@ static int iter_finish_branch_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { - struct evsel *evsel = iter->evsel; + struct evsel *evsel = iter->sample->evsel; struct hists *hists = evsel__hists(evsel); for (int i = 0; i < iter->total; i++) @@ -1103,8 +1103,8 @@ iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused, static int iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; struct hist_entry *he; he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, @@ -1121,8 +1121,8 @@ iter_finish_normal_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { struct hist_entry *he = iter->he; - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; if (he == NULL) return 0; @@ -1165,9 +1165,9 @@ static int iter_add_single_cumulative_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; - struct hists *hists = evsel__hists(evsel); struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); struct hist_entry **he_cache = iter->he_cache; struct hist_entry *he; int err = 0; @@ -1224,8 +1224,8 @@ static int iter_add_next_cumulative_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; struct hist_entry **he_cache = iter->he_cache; struct hist_entry *he; struct hist_entry he_tmp = { @@ -1342,7 +1342,7 @@ int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, alm = map__get(al->map); err = sample__resolve_callchain(iter->sample, get_tls_callchain_cursor(), &iter->parent, - iter->evsel, al, max_stack_depth); + al, max_stack_depth); if (err) { map__put(alm); return err; @@ -2826,7 +2826,7 @@ int hists__unlink(struct hists *hists) void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, struct perf_sample *sample, bool nonany_branch_mode, - u64 *total_cycles, struct evsel *evsel) + u64 *total_cycles) { struct branch_info *bi; struct branch_entry *entries = perf_sample__branch_entries(sample); @@ -2850,7 +2850,7 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, for (int i = bs->nr - 1; i >= 0; i--) { addr_map_symbol__account_cycles(&bi[i].from, nonany_branch_mode ? NULL : prev, - bi[i].flags.cycles, evsel, + bi[i].flags.cycles, sample->evsel, bi[i].branch_stack_cntr); prev = &bi[i].to; @@ -2963,9 +2963,10 @@ int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool sh ev_name, sample_freq_str, enable_ref ? ref : " ", nr_events); - if (hists->uid_filter_str) - printed += snprintf(bf + printed, size - printed, - ", UID: %s", hists->uid_filter_str); + if (hists->uid_filter_str) { + printed += scnprintf(bf + printed, size - printed, + ", UID: %s", hists->uid_filter_str); + } if (thread) { if (hists__has(hists, thread)) { printed += scnprintf(bf + printed, size - printed, @@ -3040,7 +3041,7 @@ static void hists__delete_remaining_entries(struct rb_root_cached *root) } } -static void hists__delete_all_entries(struct hists *hists) +void hists__delete_all_entries(struct hists *hists) { hists__delete_entries(hists); hists__delete_remaining_entries(&hists->entries_in_array[0]); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index d97a4efb9250..b830cbe7f95b 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -156,7 +156,6 @@ struct hist_entry_iter { int total; int curr; - struct evsel *evsel; struct perf_sample *sample; struct hist_entry *he; struct symbol *parent; @@ -392,6 +391,7 @@ int hists__collapse_resort(struct hists *hists, struct ui_progress *prog); void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); void hists__delete_entries(struct hists *hists); +void hists__delete_all_entries(struct hists *hists); void hists__output_recalc_col_len(struct hists *hists, int max_rows); struct hist_entry *hists__get_entry(struct hists *hists, int idx); @@ -799,7 +799,7 @@ unsigned int hists__overhead_width(struct hists *hists); void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, struct perf_sample *sample, bool nonany_branch_mode, - u64 *total_cycles, struct evsel *evsel); + u64 *total_cycles); struct option; int parse_filter_percentage(const struct option *opt, const char *arg, int unset); diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c index fb3ffa8d32ad..ed544dca70c3 100644 --- a/tools/perf/util/hwmon_pmu.c +++ b/tools/perf/util/hwmon_pmu.c @@ -202,7 +202,8 @@ bool parse_hwmon_filename(const char *filename, fn_item_len = strlen(fn_item); if (fn_item_len > 6 && !strcmp(&fn_item[fn_item_len - 6], "_alarm")) { assert(strlen(LONGEST_HWMON_ITEM_STR) < sizeof(fn_type)); - strlcpy(fn_type, fn_item, fn_item_len - 5); + /* fn_item_len - 5 strips "_alarm"; clamp to buffer size */ + strlcpy(fn_type, fn_item, min_t(size_t, fn_item_len - 5, sizeof(fn_type))); fn_item = fn_type; *alarm = true; } @@ -289,13 +290,16 @@ static int hwmon_pmu__read_events(struct hwmon_pmu *pmu) if (fd < 0) continue; - read_len = read(fd, buf, sizeof(buf)); + read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; - if (read_len > 0) - buf[read_len] = '\0'; + if (read_len <= 0) { + close(fd); + continue; + } + buf[read_len] = '\0'; if (buf[0] == '\0') { pr_debug("hwmon_pmu: empty label file %s %s\n", @@ -431,8 +435,8 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si hwmon_item_strs[bit], is_alarm ? "_alarm" : ""); fd = openat(dir, buf, O_RDONLY); - if (fd > 0) { - ssize_t read_len = read(fd, buf, sizeof(buf)); + if (fd >= 0) { + ssize_t read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; @@ -442,12 +446,12 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si buf[read_len] = '\0'; val = strtoll(buf, /*endptr=*/NULL, 10); - len += snprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", - len == 0 ? " " : ", ", - hwmon_item_strs[bit], - is_alarm ? "_alarm" : "", - (double)val / 1000.0, - hwmon_units[key.type]); + len += scnprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", + len == 0 ? " " : ", ", + hwmon_item_strs[bit], + is_alarm ? "_alarm" : "", + (double)val / 1000.0, + hwmon_units[key.type]); } close(fd); } @@ -514,14 +518,14 @@ int hwmon_pmu__for_each_event(struct perf_pmu *pmu, void *state, pmu_event_callb int ret; size_t len; - len = snprintf(alias_buf, sizeof(alias_buf), "%s%d", - hwmon_type_strs[key.type], key.num); + scnprintf(alias_buf, sizeof(alias_buf), "%s%d", + hwmon_type_strs[key.type], key.num); if (!info.name) { info.name = info.alias; info.alias = NULL; } - len = snprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", + len = scnprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", hwmon_desc[key.type], pmu->name + 6, value->label ?: info.name); @@ -816,7 +820,7 @@ int evsel__hwmon_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) count = perf_counts(evsel->counts, cpu_map_idx, thread); fd = FD(evsel, cpu_map_idx, thread); - len = pread(fd, buf, sizeof(buf), 0); + len = pread(fd, buf, sizeof(buf) - 1, 0); if (len <= 0) { count->lost++; return -EINVAL; diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 382255393fb3..0b18ebd13f7c 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -303,7 +303,8 @@ static int intel_bts_synth_branch_sample(struct intel_bts_queue *btsq, event.sample.header.size = bts->branches_event_size; ret = perf_event__synthesize_sample(&event, bts->branches_sample_type, - 0, &sample); + /*read_format=*/0, /*branch_sample_type=*/0, + &sample); if (ret) return ret; } diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c index 72c7a4e15d61..f90fcbc4302d 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c @@ -220,7 +220,6 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, { struct insn insn; int n, i, ret; - int left; ret = insn_decode(&insn, inbuf, inlen, x->is64bit ? INSN_MODE_64 : INSN_MODE_32); @@ -229,13 +228,9 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, return ""; if (lenp) *lenp = insn.length; - left = sizeof(x->out); - n = snprintf(x->out, left, "insn: "); - left -= n; - for (i = 0; i < insn.length; i++) { - n += snprintf(x->out + n, left, "%02x ", inbuf[i]); - left -= n; - } + n = scnprintf(x->out, sizeof(x->out), "insn: "); + for (i = 0; i < insn.length; i++) + n += scnprintf(x->out + n, sizeof(x->out) - n, "%02x ", inbuf[i]); return x->out; } diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index fc9eec8b54b8..d83439c8c7b0 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -1307,7 +1307,8 @@ static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, goto out_free; } - if (pt->synth_opts.last_branch || pt->synth_opts.other_events) { + if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch || + pt->synth_opts.other_events) { unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz); ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); @@ -1728,11 +1729,30 @@ static void intel_pt_prep_b_sample(struct intel_pt *pt, event->sample.header.misc = sample->cpumode; } -static int intel_pt_inject_event(union perf_event *event, +static int intel_pt_inject_event(struct intel_pt *pt, union perf_event *event, struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && pt->session && pt->session->evlist) + evsel = evlist__id2evsel(pt->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int intel_pt_opt_inject(struct intel_pt *pt, @@ -1742,7 +1762,7 @@ static inline int intel_pt_opt_inject(struct intel_pt *pt, if (!pt->synth_opts.inject) return 0; - return intel_pt_inject_event(event, sample, type); + return intel_pt_inject_event(pt, event, sample, type); } static int intel_pt_deliver_synth_event(struct intel_pt *pt, @@ -1810,9 +1830,9 @@ static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq) ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt; } - perf_sample__exit(&sample); ret = intel_pt_deliver_synth_event(pt, event, &sample, pt->branches_sample_type); + perf_sample__exit(&sample); return ret; } @@ -1947,6 +1967,7 @@ static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq) union perf_event *event = ptq->event_buf; struct perf_sample sample = { .ip = 0, }; struct perf_synth_intel_ptwrite raw; + int ret; if (intel_pt_skip_event(pt)) return 0; @@ -1963,8 +1984,9 @@ static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq) sample.raw_size = perf_synth__raw_size(raw); sample.raw_data = perf_synth__raw_data(&raw); - return intel_pt_deliver_synth_event(pt, event, &sample, - pt->ptwrites_sample_type); + ret = intel_pt_deliver_synth_event(pt, event, &sample, pt->ptwrites_sample_type); + perf_sample__exit(&sample); + return ret; } static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq) @@ -2486,7 +2508,7 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse intel_pt_add_xmm(intr_regs, pos, items, regs_mask); } - if (sample_type & PERF_SAMPLE_BRANCH_STACK) { + if ((sample_type | evsel->synth_sample_type) & PERF_SAMPLE_BRANCH_STACK) { if (items->mask[INTEL_PT_LBR_0_POS] || items->mask[INTEL_PT_LBR_1_POS] || items->mask[INTEL_PT_LBR_2_POS]) { @@ -2557,7 +2579,8 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse sample.transaction = txn; } - ret = intel_pt_deliver_synth_event(pt, event, &sample, sample_type); + ret = intel_pt_deliver_synth_event(pt, event, &sample, + sample_type | evsel->synth_sample_type); perf_sample__exit(&sample); return ret; } @@ -2979,7 +3002,7 @@ static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip) start = dso__first_symbol(map__dso(map)); for (sym = start; sym; sym = dso__next_symbol(sym)) { - if (sym->binding == STB_GLOBAL && + if (symbol__binding(sym) == STB_GLOBAL && !strcmp(sym->name, "__switch_to")) { ip = map__unmap_ip(map, sym->start); if (ip >= map__start(map) && ip < map__end(map)) { @@ -3426,7 +3449,7 @@ static int intel_pt_process_switch(struct intel_pt *pt, if (evsel != pt->switch_evsel) return 0; - tid = evsel__intval(evsel, sample, "next_pid"); + tid = perf_sample__intval(sample, "next_pid"); cpu = sample->cpu; intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n", diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c index 8b615dc94e9e..b41171b5df77 100644 --- a/tools/perf/util/intel-tpebs.c +++ b/tools/perf/util/intel-tpebs.c @@ -37,6 +37,7 @@ static pthread_t tpebs_reader_thread; static struct child_process tpebs_cmd; static int control_fd[2], ack_fd[2]; static struct mutex tpebs_mtx; +static bool tpebs_stopping; struct tpebs_retire_lat { struct list_head nd; @@ -52,16 +53,18 @@ struct tpebs_retire_lat { bool started; }; -static void tpebs_mtx_init(void) +static void tpebs_init(void) { mutex_init(&tpebs_mtx); + control_fd[0] = control_fd[1] = -1; + ack_fd[0] = ack_fd[1] = -1; } static struct mutex *tpebs_mtx_get(void) { - static pthread_once_t tpebs_mtx_once = PTHREAD_ONCE_INIT; + static pthread_once_t tpebs_once = PTHREAD_ONCE_INIT; - pthread_once(&tpebs_mtx_once, tpebs_mtx_init); + pthread_once(&tpebs_once, tpebs_init); return &tpebs_mtx; } @@ -95,8 +98,9 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel) record_argv[i++] = "-o"; record_argv[i++] = PERF_DATA; - if (!perf_cpu_map__is_any_cpu_or_is_empty(evsel->evlist->core.user_requested_cpus)) { - cpu_map__snprint(evsel->evlist->core.user_requested_cpus, cpumap_buf, + if (!perf_cpu_map__is_any_cpu_or_is_empty( + evlist__core(evsel->evlist)->user_requested_cpus)) { + cpu_map__snprint(evlist__core(evsel->evlist)->user_requested_cpus, cpumap_buf, sizeof(cpumap_buf)); record_argv[i++] = "-C"; record_argv[i++] = cpumap_buf; @@ -111,6 +115,7 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel) /* Note, no workload given so system wide is implied. */ assert(tpebs_cmd.pid == 0); + memset(&tpebs_cmd, 0, sizeof(tpebs_cmd)); tpebs_cmd.argv = record_argv; tpebs_cmd.out = -1; ret = start_command(&tpebs_cmd); @@ -172,7 +177,7 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct if (t->evsel->evlist == NULL) return true; - workload_pid = t->evsel->evlist->workload.pid; + workload_pid = evlist__workload_pid(t->evsel->evlist); if (workload_pid < 0 || workload_pid == sample_pid) return false; @@ -185,7 +190,6 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct static int process_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct tpebs_retire_lat *t; @@ -196,7 +200,7 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, mutex_unlock(tpebs_mtx_get()); return 0; } - t = tpebs_retire_lat__find(evsel); + t = tpebs_retire_lat__find(sample->evsel); if (!t) { mutex_unlock(tpebs_mtx_get()); return -EINVAL; @@ -321,20 +325,43 @@ static int tpebs_stop(void) EXCLUSIVE_LOCKS_REQUIRED(tpebs_mtx_get()) { int ret = 0; + if (tpebs_stopping) + return 0; + /* Like tpebs_start, we should only run tpebs_end once. */ if (tpebs_cmd.pid != 0) { + pid_t actual_pid = tpebs_cmd.pid; + + tpebs_stopping = true; tpebs_send_record_cmd(EVLIST_CTL_CMD_STOP_TAG); tpebs_cmd.pid = 0; mutex_unlock(tpebs_mtx_get()); pthread_join(tpebs_reader_thread, NULL); mutex_lock(tpebs_mtx_get()); - close(control_fd[0]); - close(control_fd[1]); - close(ack_fd[0]); - close(ack_fd[1]); - close(tpebs_cmd.out); + if (control_fd[0] >= 0) { + close(control_fd[0]); + control_fd[0] = -1; + } + if (control_fd[1] >= 0) { + close(control_fd[1]); + control_fd[1] = -1; + } + if (ack_fd[0] >= 0) { + close(ack_fd[0]); + ack_fd[0] = -1; + } + if (ack_fd[1] >= 0) { + close(ack_fd[1]); + ack_fd[1] = -1; + } + if (tpebs_cmd.out >= 0) { + close(tpebs_cmd.out); + tpebs_cmd.out = -1; + } + tpebs_cmd.pid = actual_pid; ret = finish_command(&tpebs_cmd); tpebs_cmd.pid = 0; + tpebs_stopping = false; if (ret == -ERR_RUN_COMMAND_WAITPID_SIGNAL) ret = 0; } @@ -487,30 +514,42 @@ int evsel__tpebs_open(struct evsel *evsel) { int ret; bool tpebs_empty; + bool started_process = false; /* We should only run tpebs_start when tpebs_recording is enabled. */ if (!tpebs_recording) return 0; + + mutex_lock(tpebs_mtx_get()); + if (tpebs_stopping) { + mutex_unlock(tpebs_mtx_get()); + return -EBUSY; + } /* Only start the events once. */ if (tpebs_cmd.pid != 0) { struct tpebs_retire_lat *t; bool valid; - mutex_lock(tpebs_mtx_get()); t = tpebs_retire_lat__find(evsel); valid = t && t->started; mutex_unlock(tpebs_mtx_get()); /* May fail as the event wasn't started. */ return valid ? 0 : -EBUSY; } + mutex_unlock(tpebs_mtx_get()); ret = evsel__tpebs_prepare(evsel); if (ret) return ret; mutex_lock(tpebs_mtx_get()); + if (tpebs_stopping || tpebs_cmd.pid != 0) { + ret = -EBUSY; + goto out; + } tpebs_empty = list_empty(&tpebs_results); if (!tpebs_empty) { + started_process = true; /*Create control and ack fd for --control*/ if (pipe(control_fd) < 0) { pr_err("tpebs: Failed to create control fifo"); @@ -530,7 +569,6 @@ int evsel__tpebs_open(struct evsel *evsel) if (pthread_create(&tpebs_reader_thread, /*attr=*/NULL, __sample_reader, /*arg=*/NULL)) { kill(tpebs_cmd.pid, SIGTERM); - close(tpebs_cmd.out); pr_err("Could not create thread to process sample data.\n"); ret = -1; goto out; @@ -541,8 +579,38 @@ int evsel__tpebs_open(struct evsel *evsel) if (ret) { struct tpebs_retire_lat *t = tpebs_retire_lat__find(evsel); - list_del_init(&t->nd); - tpebs_retire_lat__delete(t); + if (t) { + list_del_init(&t->nd); + tpebs_retire_lat__delete(t); + } + + if (started_process) { + if (tpebs_cmd.pid > 0) { + kill(tpebs_cmd.pid, SIGTERM); + finish_command(&tpebs_cmd); + tpebs_cmd.pid = 0; + } + if (tpebs_cmd.out >= 0) { + close(tpebs_cmd.out); + tpebs_cmd.out = -1; + } + if (control_fd[0] >= 0) { + close(control_fd[0]); + control_fd[0] = -1; + } + if (control_fd[1] >= 0) { + close(control_fd[1]); + control_fd[1] = -1; + } + if (ack_fd[0] >= 0) { + close(ack_fd[0]); + ack_fd[0] = -1; + } + if (ack_fd[1] >= 0) { + close(ack_fd[1]); + ack_fd[1] = -1; + } + } } mutex_unlock(tpebs_mtx_get()); return ret; diff --git a/tools/perf/util/iostat.c b/tools/perf/util/iostat.c index b770bd473af7..c9d5028a47f3 100644 --- a/tools/perf/util/iostat.c +++ b/tools/perf/util/iostat.c @@ -4,7 +4,7 @@ enum iostat_mode_t iostat_mode = IOSTAT_NONE; -__weak int iostat_prepare(struct evlist *evlist __maybe_unused, +__weak int iostat_prepare(struct evlist **evlist __maybe_unused, struct perf_stat_config *config __maybe_unused) { return -1; diff --git a/tools/perf/util/iostat.h b/tools/perf/util/iostat.h index a4e7299c5c2f..df8a241fbc32 100644 --- a/tools/perf/util/iostat.h +++ b/tools/perf/util/iostat.h @@ -30,7 +30,7 @@ extern enum iostat_mode_t iostat_mode; typedef void (*iostat_print_counter_t)(struct perf_stat_config *, struct evsel *, void *); -int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config); +int iostat_prepare(struct evlist **evlist, struct perf_stat_config *config); int iostat_parse(const struct option *opt, const char *str, int unset __maybe_unused); void iostat_list(struct evlist *evlist, struct perf_stat_config *config); diff --git a/tools/perf/util/jitdump.c b/tools/perf/util/jitdump.c index e0ce8b904729..83005b30b9bf 100644 --- a/tools/perf/util/jitdump.c +++ b/tools/perf/util/jitdump.c @@ -234,7 +234,7 @@ jit_open(struct jit_buf_desc *jd, const char *name) /* * keep dirname for generating files and mmap records */ - strncpy(jd->dir, name, PATH_MAX); + strncpy(jd->dir, name, PATH_MAX - 1); jd->dir[PATH_MAX - 1] = '\0'; dirname(jd->dir); free(buf); @@ -409,7 +409,7 @@ static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) * checks the event size and assigns these extended fields if these * fields are contained in the event. */ - if (event_contains(*time_conv, time_cycles)) { + if (event_contains(*time_conv, cap_user_time_short)) { tc.time_cycles = time_conv->time_cycles; tc.time_mask = time_conv->time_mask; tc.cap_user_time_zero = time_conv->cap_user_time_zero; @@ -642,7 +642,7 @@ static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) ret = jit_inject_event(jd, event); if (!ret) - build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); + build_id__mark_dso_hit(tool, event, &sample, jd->machine); out: perf_sample__exit(&sample); return ret; diff --git a/tools/perf/util/jitdump.h b/tools/perf/util/jitdump.h index ab2842def83d..f57bfebb20ff 100644 --- a/tools/perf/util/jitdump.h +++ b/tools/perf/util/jitdump.h @@ -11,9 +11,8 @@ #ifndef JITDUMP_H #define JITDUMP_H -#include -#include #include +#include /* JiTD */ #define JITHEADER_MAGIC 0x4A695444 diff --git a/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h b/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h index 2011376c7ab5..2688ca7d0399 100644 --- a/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h +++ b/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h @@ -26,8 +26,10 @@ {0xe00, "H_DATA_STORAGE"}, \ {0xe20, "H_INST_STORAGE"}, \ {0xe40, "H_EMUL_ASSIST"}, \ + {0xea0, "H_VIRT"}, \ {0xf00, "PERFMON"}, \ {0xf20, "ALTIVEC"}, \ - {0xf40, "VSX"} + {0xf40, "VSX"}, \ + {0xf80, "H_FAC_UNAVAIL"} #endif diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c b/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c index c640dcd8af7c..018b0db0e6e7 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c @@ -17,12 +17,11 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static void event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_get_key(struct perf_sample *sample, struct event_key *key) { key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(EM_AARCH64)); + key->key = perf_sample__intval(sample, kvm_exit_reason(EM_AARCH64)); key->exit_reasons = arm64_exit_reasons; /* @@ -31,24 +30,22 @@ static void event_get_key(struct evsel *evsel, * properly decode event's est_ec. */ if (key->key == ARM_EXCEPTION_TRAP) { - key->key = evsel__intval(evsel, sample, kvm_trap_exit_reason); + key->key = perf_sample__intval(sample, kvm_trap_exit_reason); key->exit_reasons = arm64_trap_exit_reasons; } } -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_begin(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, kvm_entry_trace(EM_AARCH64)); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_AARCH64)); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample, +static bool event_end(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, kvm_exit_trace(EM_AARCH64))) { - event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, kvm_exit_trace(EM_AARCH64))) { + event_get_key(sample, key); return true; } return false; diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c b/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c index b802e516b138..a04cd09e3361 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c @@ -53,14 +53,12 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool event_begin(struct perf_sample *sample, struct event_key *key) { - return exit_event_begin(evsel, sample, key); + return exit_event_begin(sample, key); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { /* @@ -71,17 +69,16 @@ static bool event_end(struct evsel *evsel, * kvm:kvm_enter means returning to vmm and then to guest * kvm:kvm_reenter means returning to guest immediately */ - return evsel__name_is(evsel, kvm_entry_trace(EM_LOONGARCH)) || - evsel__name_is(evsel, kvm_reenter_trace); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_LOONGARCH)) || + evsel__name_is(sample->evsel, kvm_reenter_trace); } -static void event_gspr_get_key(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static void event_gspr_get_key(struct perf_sample *sample, struct event_key *key) { unsigned int insn; key->key = LOONGARCH_EXCEPTION_OTHERS; - insn = evsel__intval(evsel, sample, "inst_word"); + insn = perf_sample__intval(sample, "inst_word"); switch (insn >> 24) { case 0: diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c index 42182d70beb6..315b24f9a592 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c @@ -9,7 +9,6 @@ #include "book3s_hv_exits.h" #include "book3s_hcalls.h" -#include #define NR_TPS 4 @@ -28,12 +27,11 @@ static const char * const ppc_book3s_hv_kvm_tp[] = { /* 1 extra placeholder for NULL */ static const char *__kvm_events_tp[NR_TPS + 1]; -static void hcall_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void hcall_event_get_key(struct perf_sample *sample, struct event_key *key) { key->info = 0; - key->key = evsel__intval(evsel, sample, "req"); + key->key = perf_sample__intval(sample, "req"); } static const char *get_hcall_exit_reason(u64 exit_code) @@ -51,18 +49,16 @@ static const char *get_hcall_exit_reason(u64 exit_code) return "UNKNOWN"; } -static bool hcall_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool hcall_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, __kvm_events_tp[3]); + return evsel__name_is(sample->evsel, __kvm_events_tp[3]); } -static bool hcall_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool hcall_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, __kvm_events_tp[2])) { - hcall_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, __kvm_events_tp[2])) { + hcall_event_get_key(sample, key); return true; } @@ -180,35 +176,15 @@ int __cpu_isa_init_powerpc(struct perf_kvm_stat *kvm) */ int __kvm_add_default_arch_event_powerpc(int *argc, const char **argv) { - const char **tmp; - bool event = false; - int i, j = *argc; + int j = *argc; - const struct option event_options[] = { - OPT_BOOLEAN('e', "event", &event, NULL), - OPT_END() - }; - - tmp = calloc(j + 1, sizeof(char *)); - if (!tmp) + if (!perf_pmus__have_event("trace_imc", "trace_cycles")) return -EINVAL; - for (i = 0; i < j; i++) - tmp[i] = argv[i]; - - parse_options(j, tmp, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN); - if (!event) { - if (perf_pmus__have_event("trace_imc", "trace_cycles")) { - argv[j++] = strdup("-e"); - argv[j++] = strdup("trace_imc/trace_cycles/"); - *argc += 2; - } else { - free(tmp); - return -EINVAL; - } - } + argv[j++] = "-e"; + argv[j++] = "trace_imc/trace_cycles/"; + *argc += 2; - free(tmp); return 0; } diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c b/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c index 8d4d5d6ce720..967bba261a47 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c @@ -20,30 +20,27 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static void event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_get_key(struct perf_sample *sample, struct event_key *key) { int xlen = 64; // TODO: 32-bit support. key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(EM_RISCV)) & ~CAUSE_IRQ_FLAG(xlen); + key->key = perf_sample__intval(sample, kvm_exit_reason(EM_RISCV)) & ~CAUSE_IRQ_FLAG(xlen); key->exit_reasons = riscv_exit_reasons; } -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_begin(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, kvm_entry_trace(EM_RISCV)); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_RISCV)); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample, +static bool event_end(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, kvm_exit_trace(EM_RISCV))) { - event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, kvm_exit_trace(EM_RISCV))) { + event_get_key(sample, key); return true; } return false; diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c b/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c index 7e29169f5bb0..4771fc69fa39 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c @@ -18,38 +18,34 @@ define_exit_reasons_table(sie_sigp_order_codes, sigp_order_codes); define_exit_reasons_table(sie_diagnose_codes, diagnose_codes); define_exit_reasons_table(sie_icpt_prog_codes, icpt_prog_codes); -static void event_icpt_insn_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_icpt_insn_get_key(struct perf_sample *sample, struct event_key *key) { u64 insn; - insn = evsel__intval(evsel, sample, "instruction"); + insn = perf_sample__intval(sample, "instruction"); key->key = icpt_insn_decoder(insn); key->exit_reasons = sie_icpt_insn_codes; } -static void event_sigp_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_sigp_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "order_code"); + key->key = perf_sample__intval(sample, "order_code"); key->exit_reasons = sie_sigp_order_codes; } -static void event_diag_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_diag_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "code"); + key->key = perf_sample__intval(sample, "code"); key->exit_reasons = sie_diagnose_codes; } -static void event_icpt_prog_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_icpt_prog_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "code"); + key->key = perf_sample__intval(sample, "code"); key->exit_reasons = sie_icpt_prog_codes; } diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c index 0f626db3a439..97041f5100fc 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c @@ -7,7 +7,6 @@ #include "../../../arch/x86/include/uapi/asm/svm.h" #include "../../../arch/x86/include/uapi/asm/vmx.h" #include "../../../arch/x86/include/uapi/asm/kvm.h" -#include define_exit_reasons_table(vmx_exit_reasons, VMX_EXIT_REASONS); define_exit_reasons_table(svm_exit_reasons, SVM_EXIT_REASONS); @@ -24,45 +23,43 @@ static const struct kvm_events_ops exit_events = { * the time of MMIO write: kvm_mmio(KVM_TRACE_MMIO_WRITE...) -> kvm_entry * the time of MMIO read: kvm_exit -> kvm_mmio(KVM_TRACE_MMIO_READ...). */ -static void mmio_event_get_key(struct evsel *evsel, struct perf_sample *sample, +static void mmio_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "gpa"); - key->info = evsel__intval(evsel, sample, "type"); + key->key = perf_sample__intval(sample, "gpa"); + key->info = perf_sample__intval(sample, "type"); } #define KVM_TRACE_MMIO_READ_UNSATISFIED 0 #define KVM_TRACE_MMIO_READ 1 #define KVM_TRACE_MMIO_WRITE 2 -static bool mmio_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool mmio_event_begin(struct perf_sample *sample, struct event_key *key) { /* MMIO read begin event in kernel. */ - if (kvm_exit_event(evsel)) + if (kvm_exit_event(sample->evsel)) return true; /* MMIO write begin event in kernel. */ - if (evsel__name_is(evsel, "kvm:kvm_mmio") && - evsel__intval(evsel, sample, "type") == KVM_TRACE_MMIO_WRITE) { - mmio_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_mmio") && + perf_sample__intval(sample, "type") == KVM_TRACE_MMIO_WRITE) { + mmio_event_get_key(sample, key); return true; } return false; } -static bool mmio_event_end(struct evsel *evsel, struct perf_sample *sample, - struct event_key *key) +static bool mmio_event_end(struct perf_sample *sample, struct event_key *key) { /* MMIO write end event in kernel. */ - if (kvm_entry_event(evsel)) + if (kvm_entry_event(sample->evsel)) return true; /* MMIO read end event in kernel.*/ - if (evsel__name_is(evsel, "kvm:kvm_mmio") && - evsel__intval(evsel, sample, "type") == KVM_TRACE_MMIO_READ) { - mmio_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_mmio") && + perf_sample__intval(sample, "type") == KVM_TRACE_MMIO_READ) { + mmio_event_get_key(sample, key); return true; } @@ -86,31 +83,27 @@ static const struct kvm_events_ops mmio_events = { }; /* The time of emulation pio access is from kvm_pio to kvm_entry. */ -static void ioport_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void ioport_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "port"); - key->info = evsel__intval(evsel, sample, "rw"); + key->key = perf_sample__intval(sample, "port"); + key->info = perf_sample__intval(sample, "rw"); } -static bool ioport_event_begin(struct evsel *evsel, - struct perf_sample *sample, +static bool ioport_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, "kvm:kvm_pio")) { - ioport_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_pio")) { + ioport_event_get_key(sample, key); return true; } return false; } -static bool ioport_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, - struct event_key *key __maybe_unused) +static bool ioport_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static void ioport_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused, @@ -130,31 +123,25 @@ static const struct kvm_events_ops ioport_events = { }; /* The time of emulation msr is from kvm_msr to kvm_entry. */ -static void msr_event_get_key(struct evsel *evsel, - struct perf_sample *sample, - struct event_key *key) +static void msr_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "ecx"); - key->info = evsel__intval(evsel, sample, "write"); + key->key = perf_sample__intval(sample, "ecx"); + key->info = perf_sample__intval(sample, "write"); } -static bool msr_event_begin(struct evsel *evsel, - struct perf_sample *sample, - struct event_key *key) +static bool msr_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, "kvm:kvm_msr")) { - msr_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_msr")) { + msr_event_get_key(sample, key); return true; } return false; } -static bool msr_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, - struct event_key *key __maybe_unused) +static bool msr_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static void msr_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused, @@ -223,39 +210,13 @@ int __cpu_isa_init_x86(struct perf_kvm_stat *kvm, const char *cpuid) */ int __kvm_add_default_arch_event_x86(int *argc, const char **argv) { - const char **tmp; - bool event = false; - int ret = 0, i, j = *argc; - - const struct option event_options[] = { - OPT_BOOLEAN('e', "event", &event, NULL), - OPT_BOOLEAN(0, "pfm-events", &event, NULL), - OPT_END() - }; - - if (!x86__is_intel_cpu()) - return 0; - - tmp = calloc(j + 1, sizeof(char *)); - if (!tmp) - return -ENOMEM; - - for (i = 0; i < j; i++) - tmp[i] = argv[i]; - - parse_options(j, tmp, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN); - if (!event) { - argv[j++] = STRDUP_FAIL_EXIT("-e"); - argv[j++] = STRDUP_FAIL_EXIT("cycles"); - *argc += 2; - } + int j = *argc; - free(tmp); - return 0; + argv[j++] = "-e"; + argv[j++] = "cycles"; + *argc += 2; -EXIT: - free(tmp); - return ret; + return 0; } const char * const *__kvm_events_tp_x86(void) diff --git a/tools/perf/util/kvm-stat.c b/tools/perf/util/kvm-stat.c index 27f16810498c..c571d2ed711a 100644 --- a/tools/perf/util/kvm-stat.c +++ b/tools/perf/util/kvm-stat.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 #include "debug.h" +#include "env.h" #include "evsel.h" #include "kvm-stat.h" #include +#include bool kvm_exit_event(struct evsel *evsel) { @@ -11,22 +13,20 @@ bool kvm_exit_event(struct evsel *evsel) return evsel__name_is(evsel, kvm_exit_trace(e_machine)); } -void exit_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +void exit_event_get_key(struct perf_sample *sample, struct event_key *key) { - uint16_t e_machine = evsel__e_machine(evsel, /*e_flags=*/NULL); + uint16_t e_machine = evsel__e_machine(sample->evsel, /*e_flags=*/NULL); key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(e_machine)); + key->key = perf_sample__intval(sample, kvm_exit_reason(e_machine)); } -bool exit_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +bool exit_event_begin(struct perf_sample *sample, struct event_key *key) { - if (kvm_exit_event(evsel)) { - exit_event_get_key(evsel, sample, key); + if (kvm_exit_event(sample->evsel)) { + exit_event_get_key(sample, key); return true; } @@ -40,11 +40,10 @@ bool kvm_entry_event(struct evsel *evsel) return evsel__name_is(evsel, kvm_entry_trace(e_machine)); } -bool exit_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +bool exit_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static const char *get_exit_reason(struct perf_kvm_stat *kvm, @@ -271,3 +270,42 @@ int kvm_add_default_arch_event(uint16_t e_machine, int *argc, const char **argv) return 0; } } + +bool kvm_need_default_arch_event(uint16_t e_machine, int argc, const char **argv) +{ + const char **tmp_argv; + bool event = false; + int i; + + const struct option event_options[] = { + OPT_BOOLEAN('e', "event", &event, NULL), + OPT_BOOLEAN(0, "pfm-events", &event, NULL), + OPT_END() + }; + + switch (e_machine) { + case EM_PPC: + case EM_PPC64: + break; + case EM_X86_64: + case EM_386: + if (!x86__is_intel_cpu()) + return false; + break; + default: + return false; + } + + /* parse_options() may change the argv, let's make a copy */ + tmp_argv = calloc(argc + 1, sizeof(char *)); + if (!tmp_argv) + return false; + + for (i = 0; i < argc; i++) + tmp_argv[i] = argv[i]; + + parse_options(argc, tmp_argv, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN); + free(tmp_argv); + + return !event; +} diff --git a/tools/perf/util/kvm-stat.h b/tools/perf/util/kvm-stat.h index 4a998aaece5d..1db31c075187 100644 --- a/tools/perf/util/kvm-stat.h +++ b/tools/perf/util/kvm-stat.h @@ -53,18 +53,15 @@ struct kvm_event { }; struct child_event_ops { - void (*get_key)(struct evsel *evsel, - struct perf_sample *sample, + void (*get_key)(struct perf_sample *sample, struct event_key *key); const char *name; }; struct kvm_events_ops { - bool (*is_begin_event)(struct evsel *evsel, - struct perf_sample *sample, + bool (*is_begin_event)(struct perf_sample *sample, struct event_key *key); - bool (*is_end_event)(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key); + bool (*is_end_event)(struct perf_sample *sample, struct event_key *key); const struct child_event_ops *child_ops; void (*decode_key)(struct perf_kvm_stat *kvm, struct event_key *key, char *decode); @@ -116,14 +113,11 @@ struct kvm_reg_events_ops { #ifdef HAVE_LIBTRACEEVENT -void exit_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +void exit_event_get_key(struct perf_sample *sample, struct event_key *key); -bool exit_event_begin(struct evsel *evsel, - struct perf_sample *sample, +bool exit_event_begin(struct perf_sample *sample, struct event_key *key); -bool exit_event_end(struct evsel *evsel, - struct perf_sample *sample, +bool exit_event_end(struct perf_sample *sample, struct event_key *key); void exit_event_decode_key(struct perf_kvm_stat *kvm, struct event_key *key, @@ -180,12 +174,20 @@ const char * const *__kvm_skip_events_riscv(void); const char * const *__kvm_skip_events_s390(void); const char * const *__kvm_skip_events_x86(void); +bool kvm_need_default_arch_event(uint16_t e_machine, int argc, const char **argv); int kvm_add_default_arch_event(uint16_t e_machine, int *argc, const char **argv); int __kvm_add_default_arch_event_powerpc(int *argc, const char **argv); int __kvm_add_default_arch_event_x86(int *argc, const char **argv); #else /* !HAVE_LIBTRACEEVENT */ +static inline bool kvm_need_default_arch_event(uint16_t e_machine __maybe_unused, + int argc __maybe_unused, + const char **argv __maybe_unused) +{ + return false; +} + static inline int kvm_add_default_arch_event(uint16_t e_machine __maybe_unused, int *argc __maybe_unused, const char **argv __maybe_unused) @@ -227,14 +229,4 @@ static inline struct kvm_info *kvm_info__new(void) return ki; } -#define STRDUP_FAIL_EXIT(s) \ - ({ char *_p; \ - _p = strdup(s); \ - if (!_p) { \ - ret = -ENOMEM; \ - goto EXIT; \ - } \ - _p; \ - }) - #endif /* __PERF_KVM_STAT_H */ diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index db00269b73f2..6ec70dcc4157 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -5,6 +5,7 @@ #include "util/tool.h" #include "util/time-utils.h" +#include #include #include #include @@ -157,7 +158,6 @@ struct kwork_class { struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); @@ -165,21 +165,29 @@ struct kwork_class { char *buf, int len); }; +static inline void work_exit(struct kwork_work *work) +{ + if (work) { + free(work->name); + work->name = NULL; + } +} + struct trace_kwork_handler { int (*raise_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*entry_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*exit_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*sched_switch_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); }; @@ -194,6 +202,7 @@ struct __top_cpus_runtime { struct kwork_top_stat { DECLARE_BITMAP(all_cpus_bitmap, MAX_NR_CPUS); struct __top_cpus_runtime *cpus_runtime; + unsigned int nr_skipped_cpu; }; struct perf_kwork { diff --git a/tools/perf/util/libdw.c b/tools/perf/util/libdw.c index 216977884103..d5d2958902c0 100644 --- a/tools/perf/util/libdw.c +++ b/tools/perf/util/libdw.c @@ -4,6 +4,7 @@ #include "srcline.h" #include "symbol.h" #include "dwarf-aux.h" +#include "callchain.h" #include #include #include @@ -60,7 +61,10 @@ struct Dwfl *dso__libdw_dwfl(struct dso *dso) return NULL; } - dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL); + if (dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL) != 0) { + dwfl_end(dwfl); + return NULL; + } dso__set_libdw(dso, dwfl); return dwfl; @@ -72,43 +76,65 @@ struct libdw_a2l_cb_args { struct inline_node *node; char *leaf_srcline; bool leaf_srcline_used; + int err; }; static int libdw_a2l_cb(Dwarf_Die *die, void *_args) { struct libdw_a2l_cb_args *args = _args; - struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, dwarf_diename(die)); + struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, die_name(die)); const char *call_fname = die_get_call_file(die); + int call_lineno = die_get_call_lineno(die); char *call_srcline = srcline__unknown; - struct inline_list *ilist; if (!inline_sym) - return -ENOMEM; + goto abort_enomem; /* Assign caller information to the parent. */ if (call_fname) - call_srcline = srcline_from_fileline(call_fname, die_get_call_lineno(die)); + call_srcline = srcline_from_fileline(call_fname, call_lineno >= 0 ? call_lineno : 0); + + if (!list_empty(&args->node->val)) { + struct inline_list *parent; - list_for_each_entry(ilist, &args->node->val, list) { - if (args->leaf_srcline == ilist->srcline) + if (callchain_param.order == ORDER_CALLEE) + parent = list_first_entry(&args->node->val, struct inline_list, list); + else + parent = list_last_entry(&args->node->val, struct inline_list, list); + + if (args->leaf_srcline == parent->srcline) args->leaf_srcline_used = false; - else if (ilist->srcline != srcline__unknown) - free(ilist->srcline); - ilist->srcline = call_srcline; + else if (parent->srcline != srcline__unknown) + free(parent->srcline); + parent->srcline = call_srcline; call_srcline = NULL; - break; } if (call_srcline && call_srcline != srcline__unknown) free(call_srcline); /* Add this symbol to the chain as the leaf. */ if (!args->leaf_srcline_used) { - inline_list__append_tail(inline_sym, args->leaf_srcline, args->node); + if (inline_list__append_tail(inline_sym, args->leaf_srcline, args->node) != 0) + goto abort_delete_sym; args->leaf_srcline_used = true; } else { - inline_list__append_tail(inline_sym, strdup(args->leaf_srcline), args->node); + char *srcline = strdup(args->leaf_srcline); + + if (!srcline) + goto abort_delete_sym; + if (inline_list__append_tail(inline_sym, srcline, args->node) != 0) { + free(srcline); + goto abort_delete_sym; + } } return 0; + +abort_delete_sym: + if (symbol__inlined(inline_sym)) + symbol__delete(inline_sym); +abort_enomem: + args->err = -ENOMEM; + return DWARF_CB_ABORT; } int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, @@ -162,11 +188,29 @@ int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, .leaf_srcline = srcline_from_fileline(src ?: "", lineno), }; + if (!args.leaf_srcline) { + if (file && *file) { + free(*file); + *file = NULL; + } + return 0; + } + /* Walk from the parent down to the leaf. */ - cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); + if (cudie) + cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); if (!args.leaf_srcline_used) free(args.leaf_srcline); + + if (args.err) { + if (file && *file) { + free(*file); + *file = NULL; + } + inline_node__clear_frames(node); + return 0; + } } return 1; } diff --git a/tools/perf/util/libunwind-arch/Build b/tools/perf/util/libunwind-arch/Build new file mode 100644 index 000000000000..80d3571918b1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/Build @@ -0,0 +1,11 @@ +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arm64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arm.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-loongarch.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-mips.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-ppc32.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-ppc64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-riscv.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-s390.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-i386.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-x86_64.o diff --git a/tools/perf/util/libunwind-arch/libunwind-arch.c b/tools/perf/util/libunwind-arch/libunwind-arch.c new file mode 100644 index 000000000000..9a74cf3c8729 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arch.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include +#include + +int get_perf_regnum_for_unw_regnum(unsigned int e_machine, int unw_regnum) +{ + switch (e_machine) { + case EM_ARM: + return __get_perf_regnum_for_unw_regnum_arm(unw_regnum); + case EM_AARCH64: + return __get_perf_regnum_for_unw_regnum_arm64(unw_regnum); + case EM_LOONGARCH: + return __get_perf_regnum_for_unw_regnum_loongarch(unw_regnum); + case EM_MIPS: + return __get_perf_regnum_for_unw_regnum_mips(unw_regnum); + case EM_PPC: + return __get_perf_regnum_for_unw_regnum_ppc32(unw_regnum); + case EM_PPC64: + return __get_perf_regnum_for_unw_regnum_ppc64(unw_regnum); + case EM_RISCV: + return __get_perf_regnum_for_unw_regnum_riscv(unw_regnum); + case EM_S390: + return __get_perf_regnum_for_unw_regnum_s390(unw_regnum); + case EM_386: + return __get_perf_regnum_for_unw_regnum_i386(unw_regnum); + case EM_X86_64: + return __get_perf_regnum_for_unw_regnum_x86_64(unw_regnum); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + + +void libunwind_arch__flush_access(struct maps *maps) +{ + unsigned int e_machine = maps__e_machine(maps); + + switch (e_machine) { + case EM_NONE: + break; // No libunwind info on the maps. + case EM_ARM: + __libunwind_arch__flush_access_arm(maps); + break; + case EM_AARCH64: + __libunwind_arch__flush_access_arm64(maps); + break; + case EM_LOONGARCH: + __libunwind_arch__flush_access_loongarch(maps); + break; + case EM_MIPS: + __libunwind_arch__flush_access_mips(maps); + break; + case EM_PPC: + __libunwind_arch__flush_access_ppc32(maps); + break; + case EM_PPC64: + __libunwind_arch__flush_access_ppc64(maps); + break; + case EM_RISCV: + __libunwind_arch__flush_access_riscv(maps); + break; + case EM_S390: + __libunwind_arch__flush_access_s390(maps); + break; + case EM_386: + __libunwind_arch__flush_access_i386(maps); + break; + case EM_X86_64: + __libunwind_arch__flush_access_x86_64(maps); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + break; + } +} + +void libunwind_arch__finish_access(struct maps *maps) +{ + unsigned int e_machine = maps__e_machine(maps); + + switch (e_machine) { + case EM_NONE: + break; // No libunwind info on the maps. + case EM_ARM: + __libunwind_arch__finish_access_arm(maps); + break; + case EM_AARCH64: + __libunwind_arch__finish_access_arm64(maps); + break; + case EM_LOONGARCH: + __libunwind_arch__finish_access_loongarch(maps); + break; + case EM_MIPS: + __libunwind_arch__finish_access_mips(maps); + break; + case EM_PPC: + __libunwind_arch__finish_access_ppc32(maps); + break; + case EM_PPC64: + __libunwind_arch__finish_access_ppc64(maps); + break; + case EM_RISCV: + __libunwind_arch__finish_access_riscv(maps); + break; + case EM_S390: + __libunwind_arch__finish_access_s390(maps); + break; + case EM_386: + __libunwind_arch__finish_access_i386(maps); + break; + case EM_X86_64: + __libunwind_arch__finish_access_x86_64(maps); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + break; + } +} + +void *libunwind_arch__create_addr_space(unsigned int e_machine) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__create_addr_space_arm(); + case EM_AARCH64: + return __libunwind_arch__create_addr_space_arm64(); + case EM_LOONGARCH: + return __libunwind_arch__create_addr_space_loongarch(); + case EM_MIPS: + return __libunwind_arch__create_addr_space_mips(); + case EM_PPC: + return __libunwind_arch__create_addr_space_ppc32(); + case EM_PPC64: + return __libunwind_arch__create_addr_space_ppc64(); + case EM_RISCV: + return __libunwind_arch__create_addr_space_riscv(); + case EM_S390: + return __libunwind_arch__create_addr_space_s390(); + case EM_386: + return __libunwind_arch__create_addr_space_i386(); + case EM_X86_64: + return __libunwind_arch__create_addr_space_x86_64(); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return NULL; + } +} + +int libunwind_arch__dwarf_search_unwind_table(unsigned int e_machine, + void *as, + uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__dwarf_search_unwind_table_arm(as, ip, di, pi, + need_unwind_info, arg); + case EM_AARCH64: + return __libunwind_arch__dwarf_search_unwind_table_arm64(as, ip, di, pi, + need_unwind_info, arg); + case EM_LOONGARCH: + return __libunwind_arch__dwarf_search_unwind_table_loongarch(as, ip, di, pi, + need_unwind_info, arg); + case EM_MIPS: + return __libunwind_arch__dwarf_search_unwind_table_mips(as, ip, di, pi, + need_unwind_info, arg); + case EM_PPC: + return __libunwind_arch__dwarf_search_unwind_table_ppc32(as, ip, di, pi, + need_unwind_info, arg); + case EM_PPC64: + return __libunwind_arch__dwarf_search_unwind_table_ppc64(as, ip, di, pi, + need_unwind_info, arg); + case EM_RISCV: + return __libunwind_arch__dwarf_search_unwind_table_riscv(as, ip, di, pi, + need_unwind_info, arg); + case EM_S390: + return __libunwind_arch__dwarf_search_unwind_table_s390(as, ip, di, pi, + need_unwind_info, arg); + case EM_386: + return __libunwind_arch__dwarf_search_unwind_table_i386(as, ip, di, pi, + need_unwind_info, arg); + case EM_X86_64: + return __libunwind_arch__dwarf_search_unwind_table_x86_64(as, ip, di, pi, + need_unwind_info, arg); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + +int libunwind_arch__dwarf_find_debug_frame(unsigned int e_machine, + int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__dwarf_find_debug_frame_arm(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_AARCH64: + return __libunwind_arch__dwarf_find_debug_frame_arm64(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_LOONGARCH: + return __libunwind_arch__dwarf_find_debug_frame_loongarch(found, di_debug, ip, + segbase, obj_name, + start, end); + case EM_MIPS: + return __libunwind_arch__dwarf_find_debug_frame_mips(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_PPC: + return __libunwind_arch__dwarf_find_debug_frame_ppc32(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_PPC64: + return __libunwind_arch__dwarf_find_debug_frame_ppc64(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_RISCV: + return __libunwind_arch__dwarf_find_debug_frame_riscv(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_S390: + return __libunwind_arch__dwarf_find_debug_frame_s390(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_386: + return __libunwind_arch__dwarf_find_debug_frame_i386(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_X86_64: + return __libunwind_arch__dwarf_find_debug_frame_x86_64(found, di_debug, ip, segbase, + obj_name, start, end); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + +struct unwind_info *libunwind_arch_unwind_info__new(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool best_effort, uint16_t e_machine, + uint64_t first_ip) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch_unwind_info__new_arm(thread, sample, max_stack, + best_effort, first_ip); + case EM_AARCH64: + return __libunwind_arch_unwind_info__new_arm64(thread, sample, max_stack, + best_effort, first_ip); + case EM_LOONGARCH: + return __libunwind_arch_unwind_info__new_loongarch(thread, sample, max_stack, + best_effort, first_ip); + case EM_MIPS: + return __libunwind_arch_unwind_info__new_mips(thread, sample, max_stack, + best_effort, first_ip); + case EM_PPC: + return __libunwind_arch_unwind_info__new_ppc32(thread, sample, max_stack, + best_effort, first_ip); + case EM_PPC64: + return __libunwind_arch_unwind_info__new_ppc64(thread, sample, max_stack, + best_effort, first_ip); + case EM_RISCV: + return __libunwind_arch_unwind_info__new_riscv(thread, sample, max_stack, + best_effort, first_ip); + case EM_S390: + return __libunwind_arch_unwind_info__new_s390(thread, sample, max_stack, + best_effort, first_ip); + case EM_386: + return __libunwind_arch_unwind_info__new_i386(thread, sample, max_stack, + best_effort, first_ip); + case EM_X86_64: + return __libunwind_arch_unwind_info__new_x86_64(thread, sample, max_stack, + best_effort, first_ip); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return NULL; + } +} + +void libunwind_arch_unwind_info__delete(struct unwind_info *ui) +{ + free(ui); +} + +int libunwind_arch__unwind_step(struct unwind_info *ui) +{ + switch (ui->e_machine) { + case EM_ARM: + return __libunwind_arch__unwind_step_arm(ui); + case EM_AARCH64: + return __libunwind_arch__unwind_step_arm64(ui); + case EM_LOONGARCH: + return __libunwind_arch__unwind_step_loongarch(ui); + case EM_MIPS: + return __libunwind_arch__unwind_step_mips(ui); + case EM_PPC: + return __libunwind_arch__unwind_step_ppc32(ui); + case EM_PPC64: + return __libunwind_arch__unwind_step_ppc64(ui); + case EM_RISCV: + return __libunwind_arch__unwind_step_riscv(ui); + case EM_S390: + return __libunwind_arch__unwind_step_s390(ui); + case EM_386: + return __libunwind_arch__unwind_step_i386(ui); + case EM_X86_64: + return __libunwind_arch__unwind_step_x86_64(ui); + default: + pr_err("ELF MACHINE %x is not supported.\n", ui->e_machine); + return -EINVAL; + } +} diff --git a/tools/perf/util/libunwind-arch/libunwind-arch.h b/tools/perf/util/libunwind-arch/libunwind-arch.h new file mode 100644 index 000000000000..74a09cd58f38 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arch.h @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LIBUNWIND_ARCH_H +#define __LIBUNWIND_ARCH_H + +#include +#include +#include + +struct machine; +struct maps; +struct perf_sample; +struct thread; + +struct unwind_info { + struct machine *machine; + struct thread *thread; + struct perf_sample *sample; + void *cursor; + uint64_t *ips; + int cur_ip; + int max_ips; + unsigned int unw_word_t_size; + uint16_t e_machine; + bool best_effort; +}; + +struct libarch_unwind__dyn_info { + uint64_t start_ip; + uint64_t end_ip; + uint64_t segbase; + uint64_t table_data; + uint64_t table_len; +}; +struct libarch_unwind__proc_info; + +int __get_perf_regnum_for_unw_regnum_arm(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_arm64(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_loongarch(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_mips(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_ppc32(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_ppc64(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_riscv(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_s390(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_i386(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_x86_64(int unw_regnum); +int get_perf_regnum_for_unw_regnum(unsigned int e_machine, int unw_regnum); + +void __libunwind_arch__flush_access_arm(struct maps *maps); +void __libunwind_arch__flush_access_arm64(struct maps *maps); +void __libunwind_arch__flush_access_loongarch(struct maps *maps); +void __libunwind_arch__flush_access_mips(struct maps *maps); +void __libunwind_arch__flush_access_ppc32(struct maps *maps); +void __libunwind_arch__flush_access_ppc64(struct maps *maps); +void __libunwind_arch__flush_access_riscv(struct maps *maps); +void __libunwind_arch__flush_access_s390(struct maps *maps); +void __libunwind_arch__flush_access_i386(struct maps *maps); +void __libunwind_arch__flush_access_x86_64(struct maps *maps); +void libunwind_arch__flush_access(struct maps *maps); + +void __libunwind_arch__finish_access_arm(struct maps *maps); +void __libunwind_arch__finish_access_arm64(struct maps *maps); +void __libunwind_arch__finish_access_loongarch(struct maps *maps); +void __libunwind_arch__finish_access_mips(struct maps *maps); +void __libunwind_arch__finish_access_ppc32(struct maps *maps); +void __libunwind_arch__finish_access_ppc64(struct maps *maps); +void __libunwind_arch__finish_access_riscv(struct maps *maps); +void __libunwind_arch__finish_access_s390(struct maps *maps); +void __libunwind_arch__finish_access_i386(struct maps *maps); +void __libunwind_arch__finish_access_x86_64(struct maps *maps); +void libunwind_arch__finish_access(struct maps *maps); + +void *__libunwind_arch__create_addr_space_arm(void); +void *__libunwind_arch__create_addr_space_arm64(void); +void *__libunwind_arch__create_addr_space_loongarch(void); +void *__libunwind_arch__create_addr_space_mips(void); +void *__libunwind_arch__create_addr_space_ppc32(void); +void *__libunwind_arch__create_addr_space_ppc64(void); +void *__libunwind_arch__create_addr_space_riscv(void); +void *__libunwind_arch__create_addr_space_s390(void); +void *__libunwind_arch__create_addr_space_i386(void); +void *__libunwind_arch__create_addr_space_x86_64(void); +void *libunwind_arch__create_addr_space(unsigned int e_machine); + +int __libunwind__find_proc_info(void *as, uint64_t ip, void *pi, int need_unwind_info, void *arg); +int __libunwind__access_mem(void *as, uint64_t addr, void *valp_word, int __write, void *arg); +int __libunwind__access_reg(void *as, int regnum, void *valp_word, int __write, void *arg); + +int __libunwind_arch__dwarf_search_unwind_table_arm(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_arm64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_loongarch(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_mips(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_ppc32(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_ppc64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_riscv(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_s390(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_i386(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_x86_64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int libunwind_arch__dwarf_search_unwind_table(unsigned int e_machine, + void *as, + uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); + +int __libunwind_arch__dwarf_find_debug_frame_arm(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_arm64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_loongarch(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_mips(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_ppc32(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_ppc64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_riscv(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_s390(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_i386(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_x86_64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int libunwind_arch__dwarf_find_debug_frame(unsigned int e_machine, + int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); + +struct unwind_info *__libunwind_arch_unwind_info__new_arm(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_arm64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_loongarch(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_mips(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_ppc32(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_ppc64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_riscv(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_s390(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_i386(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_x86_64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *libunwind_arch_unwind_info__new(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint16_t e_machine, + uint64_t first_ip); + +void libunwind_arch_unwind_info__delete(struct unwind_info *ui); + +int __libunwind_arch__unwind_step_arm(struct unwind_info *ui); +int __libunwind_arch__unwind_step_arm64(struct unwind_info *ui); +int __libunwind_arch__unwind_step_loongarch(struct unwind_info *ui); +int __libunwind_arch__unwind_step_mips(struct unwind_info *ui); +int __libunwind_arch__unwind_step_ppc32(struct unwind_info *ui); +int __libunwind_arch__unwind_step_ppc64(struct unwind_info *ui); +int __libunwind_arch__unwind_step_riscv(struct unwind_info *ui); +int __libunwind_arch__unwind_step_s390(struct unwind_info *ui); +int __libunwind_arch__unwind_step_i386(struct unwind_info *ui); +int __libunwind_arch__unwind_step_x86_64(struct unwind_info *ui); +int libunwind_arch__unwind_step(struct unwind_info *ui); + +#endif /* __LIBUNWIND_ARCH_H */ diff --git a/tools/perf/util/libunwind-arch/libunwind-arm.c b/tools/perf/util/libunwind-arch/libunwind-arm.c new file mode 100644 index 000000000000..61b1c5a70b7f --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arm.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/arm/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_arm(int unw_regnum) +{ + if (unw_regnum < 0 || unw_regnum >= PERF_REG_ARM_MAX) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return unw_regnum; +} + +void __libunwind_arch__flush_access_arm(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_arm(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_arm(void) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_arm(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_ARM_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_arm(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_ARM_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_arm(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_ARM; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_arm(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-arm64.c b/tools/perf/util/libunwind-arch/libunwind-arm64.c new file mode 100644 index 000000000000..4d467dd440a1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arm64.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/arm64/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_arm64(int unw_regnum) +{ + if (unw_regnum < 0 || unw_regnum >= PERF_REG_ARM64_EXTENDED_MAX) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return unw_regnum; +} + +void __libunwind_arch__flush_access_arm64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_arm64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_arm64(void) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_arm64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_ARM64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_arm64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_ARM64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_arm64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_AARCH64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_arm64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-i386.c b/tools/perf/util/libunwind-arch/libunwind-i386.c new file mode 100644 index 000000000000..3423be45096a --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-i386.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_i386(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_X86_SUPPORT + return -EINVAL; +#else + static const int perf_i386_regnums[] = { +#define REGNUM(reg) [UNW_X86_E ## reg] = PERF_REG_X86_ ## reg + REGNUM(AX), + REGNUM(DX), + REGNUM(CX), + REGNUM(BX), + REGNUM(SI), + REGNUM(DI), + REGNUM(BP), + REGNUM(SP), + REGNUM(IP), +#undef REGNUM + }; + + if (unw_regnum == UNW_X86_EAX) + return PERF_REG_X86_AX; + + if (unw_regnum < 0 || unw_regnum >= (int)ARRAY_SIZE(perf_i386_regnums) || + perf_i386_regnums[unw_regnum] == 0) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + + return perf_i386_regnums[unw_regnum]; +#endif // HAVE_LIBUNWIND_X86_SUPPORT +} + +void __libunwind_arch__flush_access_i386(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_i386(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_i386(void) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_i386(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_X86_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_i386(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_X86_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_i386(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_I386; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_i386(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-loongarch.c b/tools/perf/util/libunwind-arch/libunwind-loongarch.c new file mode 100644 index 000000000000..ff225a540b43 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-loongarch.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/loongarch/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_loongarch(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_LOONGARCH64_R1 ... UNW_LOONGARCH64_R31: + return unw_regnum - UNW_LOONGARCH64_R1 + PERF_REG_LOONGARCH_R1; + case UNW_LOONGARCH64_PC: + return PERF_REG_LOONGARCH_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +} + +void __libunwind_arch__flush_access_loongarch(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_loongarch(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_loongarch(void) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_loongarch(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_LOONGARCH_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_LOONGARCH) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_loongarch(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_LOONGARCH_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_LOONGARCH) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_loongarch(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_LOONGARCH; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_loongarch(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-mips.c b/tools/perf/util/libunwind-arch/libunwind-mips.c new file mode 100644 index 000000000000..ed14a8f94fa7 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-mips.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/mips/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_mips(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_MIPS_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_MIPS_R1 ... UNW_MIPS_R25: + return unw_regnum - UNW_MIPS_R1 + PERF_REG_MIPS_R1; + case UNW_MIPS_R28 ... UNW_MIPS_R31: + return unw_regnum - UNW_MIPS_R28 + PERF_REG_MIPS_R28; + case UNW_MIPS_PC: + return PERF_REG_MIPS_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_MIPS_SUPPORT +} + +void __libunwind_arch__flush_access_mips(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_mips(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_mips(void) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_mips(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_MIPS_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_MIPS) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_mips(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_MIPS_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_MIPS) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_mips(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_MIPS; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_mips(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-ppc32.c b/tools/perf/util/libunwind-arch/libunwind-ppc32.c new file mode 100644 index 000000000000..289a3b8cde75 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-ppc32.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_ppc32(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_PPC32_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_PPC32_R0 ... UNW_PPC32_R31: + return unw_regnum - UNW_PPC32_R0 + PERF_REG_POWERPC_R0; + case UNW_PPC32_LR: + return PERF_REG_POWERPC_LINK; + case UNW_PPC32_CTR: + return PERF_REG_POWERPC_CTR; + case UNW_PPC32_XER: + return PERF_REG_POWERPC_XER; + case UNW_PPC32_NIP: + return PERF_REG_POWERPC_NIP; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_PPC32_SUPPORT +} + +void __libunwind_arch__flush_access_ppc32(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_ppc32(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_ppc32(void) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_ppc32(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_PPC32_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC32) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_ppc32(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_PPC32_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC32) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_ppc32(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_PPC; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_ppc32(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-ppc64.c b/tools/perf/util/libunwind-arch/libunwind-ppc64.c new file mode 100644 index 000000000000..a4c9b6cc3c00 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-ppc64.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_ppc64(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_PPC64_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_PPC64_R0 ... UNW_PPC64_R31: + return unw_regnum - UNW_PPC64_R0 + PERF_REG_POWERPC_R0; + case UNW_PPC64_LR: + return PERF_REG_POWERPC_LINK; + case UNW_PPC64_CTR: + return PERF_REG_POWERPC_CTR; + case UNW_PPC64_XER: + return PERF_REG_POWERPC_XER; + case UNW_PPC64_NIP: + return PERF_REG_POWERPC_NIP; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_PPC64_SUPPORT +} + +void __libunwind_arch__flush_access_ppc64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_ppc64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_ppc64(void) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_ppc64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_PPC64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_ppc64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_PPC64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_ppc64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_PPC64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_ppc64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-riscv.c b/tools/perf/util/libunwind-arch/libunwind-riscv.c new file mode 100644 index 000000000000..3220690cd7d1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-riscv.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/riscv/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_riscv(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_RISCV_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_RISCV_X1 ... UNW_RISCV_X31: + return unw_regnum - UNW_RISCV_X1 + PERF_REG_RISCV_RA; + case UNW_RISCV_PC: + return PERF_REG_RISCV_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_RISCV_SUPPORT +} + +void __libunwind_arch__flush_access_riscv(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_riscv(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_riscv(void) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_riscv(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_RISCV_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_RISCV) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_riscv(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_RISCV_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_RISCV) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_riscv(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_RISCV; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_riscv(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-s390.c b/tools/perf/util/libunwind-arch/libunwind-s390.c new file mode 100644 index 000000000000..3e57cfc451c6 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-s390.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/s390/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_s390(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_S390X_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_S390X_R0 ... UNW_S390X_R15: + return unw_regnum - UNW_S390X_R0 + PERF_REG_S390_R0; + case UNW_S390X_F0 ... UNW_S390X_F15: + return unw_regnum - UNW_S390X_F0 + PERF_REG_S390_FP0; + case UNW_S390X_IP: + return PERF_REG_S390_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_S390X_SUPPORT +} + +void __libunwind_arch__flush_access_s390(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_s390(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_s390(void) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_s390(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_S390X_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_S390X) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_s390(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_S390X_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_S390X) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_s390(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_S390; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_s390(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-x86_64.c b/tools/perf/util/libunwind-arch/libunwind-x86_64.c new file mode 100644 index 000000000000..b13e9b1cc7a3 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-x86_64.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +#include +#endif + +int __get_perf_regnum_for_unw_regnum_x86_64(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_X86_64_SUPPORT + return -EINVAL; +#else + static const int perf_x86_64_regnums[] = { +#define REGNUM(reg) [UNW_X86_64_R ## reg] = PERF_REG_X86_ ## reg + REGNUM(AX), + REGNUM(DX), + REGNUM(CX), + REGNUM(BX), + REGNUM(SI), + REGNUM(DI), + REGNUM(BP), + REGNUM(SP), + REGNUM(IP), +#undef REGNUM +#define REGNUM(reg) [UNW_X86_64_ ## reg] = PERF_REG_X86_ ## reg + REGNUM(R8), + REGNUM(R9), + REGNUM(R10), + REGNUM(R11), + REGNUM(R12), + REGNUM(R13), + REGNUM(R14), + REGNUM(R15), +#undef REGNUM + }; + + if (unw_regnum == UNW_X86_64_RAX) + return PERF_REG_X86_AX; + + if (unw_regnum < 0 || unw_regnum >= (int)ARRAY_SIZE(perf_x86_64_regnums) || + perf_x86_64_regnums[unw_regnum] == 0) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return perf_x86_64_regnums[unw_regnum]; +#endif // HAVE_LIBUNWIND_X86_64_SUPPORT +} + +void __libunwind_arch__flush_access_x86_64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_x86_64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_x86_64(void) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_x86_64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_X86_64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86_64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_x86_64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_X86_64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86_64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_x86_64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_X86_64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_x86_64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind/arm64.c b/tools/perf/util/libunwind/arm64.c deleted file mode 100644 index 37ecef0c53b9..000000000000 --- a/tools/perf/util/libunwind/arm64.c +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This file setups defines to compile arch specific binary from the - * generic one. - * - * The function 'LIBUNWIND__ARCH_REG_ID' name is set according to arch - * name and the definition of this function is included directly from - * 'arch/arm64/util/unwind-libunwind.c', to make sure that this function - * is defined no matter what arch the host is. - * - * Finally, the arch specific unwind methods are exported which will - * be assigned to each arm64 thread. - */ - -#define REMOTE_UNWIND_LIBUNWIND - -/* Define arch specific functions & regs for libunwind, should be - * defined before including "unwind.h" - */ -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__arm64_reg_id(regnum) - -#include "unwind.h" -#include "libunwind-aarch64.h" -#define perf_event_arm_regs perf_event_arm64_regs -#include <../../../arch/arm64/include/uapi/asm/perf_regs.h> -#undef perf_event_arm_regs -#include "../../arch/arm64/util/unwind-libunwind.c" - -/* NO_LIBUNWIND_DEBUG_FRAME is a feature flag for local libunwind, - * assign NO_LIBUNWIND_DEBUG_FRAME_AARCH64 to it for compiling arm64 - * unwind methods. - */ -#undef NO_LIBUNWIND_DEBUG_FRAME -#ifdef NO_LIBUNWIND_DEBUG_FRAME_AARCH64 -#define NO_LIBUNWIND_DEBUG_FRAME -#endif -#include "util/unwind-libunwind-local.c" - -struct unwind_libunwind_ops * -arm64_unwind_libunwind_ops = &_unwind_libunwind_ops; diff --git a/tools/perf/util/libunwind/x86_32.c b/tools/perf/util/libunwind/x86_32.c deleted file mode 100644 index 1697dece1b74..000000000000 --- a/tools/perf/util/libunwind/x86_32.c +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This file setups defines to compile arch specific binary from the - * generic one. - * - * The function 'LIBUNWIND__ARCH_REG_ID' name is set according to arch - * name and the definition of this function is included directly from - * 'arch/x86/util/unwind-libunwind.c', to make sure that this function - * is defined no matter what arch the host is. - * - * Finally, the arch specific unwind methods are exported which will - * be assigned to each x86 thread. - */ - -#define REMOTE_UNWIND_LIBUNWIND - -/* Define arch specific functions & regs for libunwind, should be - * defined before including "unwind.h" - */ -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__x86_reg_id(regnum) - -#include "unwind.h" -#include "libunwind-x86.h" -#include <../../../../arch/x86/include/uapi/asm/perf_regs.h> - -/* HAVE_ARCH_X86_64_SUPPORT is used in'arch/x86/util/unwind-libunwind.c' - * for x86_32, we undef it to compile code for x86_32 only. - */ -#undef HAVE_ARCH_X86_64_SUPPORT -#include "../../arch/x86/util/unwind-libunwind.c" - -/* Explicitly define NO_LIBUNWIND_DEBUG_FRAME, because non-ARM has no - * dwarf_find_debug_frame() function. - */ -#ifndef NO_LIBUNWIND_DEBUG_FRAME -#define NO_LIBUNWIND_DEBUG_FRAME -#endif -#include "util/unwind-libunwind-local.c" - -struct unwind_libunwind_ops * -x86_32_unwind_libunwind_ops = &_unwind_libunwind_ops; diff --git a/tools/perf/util/lock-contention.c b/tools/perf/util/lock-contention.c index 92e7b7b572a2..119a7206f3cd 100644 --- a/tools/perf/util/lock-contention.c +++ b/tools/perf/util/lock-contention.c @@ -104,7 +104,8 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack, int max_sta struct map *kmap; struct symbol *sym; u64 ip; - const char *arch = perf_env__arch(machine->env); + uint16_t e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL); + bool is_powerpc = e_machine == EM_PPC64 || e_machine == EM_PPC; if (list_empty(&callstack_filters)) return true; @@ -125,8 +126,7 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack, int max_sta * incase first or second callstack index entry has 0 * address for powerpc. */ - if (!callstack || (!callstack[i] && (strcmp(arch, "powerpc") || - (i != 1 && i != 2)))) + if (!callstack || (!callstack[i] && (!is_powerpc || (i != 1 && i != 2)))) break; ip = callstack[i]; diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index e76f8c86e62a..0d2ebf6a84bc 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -4,10 +4,12 @@ #include #include #include +#include #include "callchain.h" #include "debug.h" #include "dso.h" #include "env.h" +#include "dwarf-regs.h" #include "event.h" #include "evsel.h" #include "hist.h" @@ -77,15 +79,14 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) int err = -ENOMEM; memset(machine, 0, sizeof(*machine)); - machine->kmaps = maps__new(machine); - if (machine->kmaps == NULL) - return -ENOMEM; - RB_CLEAR_NODE(&machine->rb_node); dsos__init(&machine->dsos); - threads__init(&machine->threads); + machine->kmaps = maps__new(machine); + if (machine->kmaps == NULL) + goto out; + machine->vdso_info = NULL; machine->env = NULL; @@ -122,11 +123,11 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) out: if (err) { - zfree(&machine->kmaps); + maps__zput(machine->kmaps); zfree(&machine->root_dir); zfree(&machine->mmap_name); } - return 0; + return err; } static struct machine *__machine__new_host(struct perf_env *host_env, bool kernel_maps) @@ -136,7 +137,10 @@ static struct machine *__machine__new_host(struct perf_env *host_env, bool kerne if (!machine) return NULL; - machine__init(machine, "", HOST_KERNEL_ID); + if (machine__init(machine, "", HOST_KERNEL_ID) != 0) { + free(machine); + return NULL; + } if (kernel_maps && machine__create_kernel_maps(machine) < 0) { free(machine); @@ -229,10 +233,12 @@ void machine__delete(struct machine *machine) } } -void machines__init(struct machines *machines) +int machines__init(struct machines *machines) { - machine__init(&machines->host, "", HOST_KERNEL_ID); + int err = machine__init(&machines->host, "", HOST_KERNEL_ID); + machines->guests = RB_ROOT_CACHED; + return err; } void machines__exit(struct machines *machines) @@ -327,7 +333,7 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - sprintf(path, "%s/%d", symbol_conf.guestmount, pid); + snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); if (access(path, R_OK)) { static struct strlist *seen; @@ -729,9 +735,15 @@ static int machine__process_ksymbol_register(struct machine *machine, { struct symbol *sym; struct dso *dso = NULL; - struct map *map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); + struct map *map; int err = 0; + /* Ignore mapping symbols in ksymbol events - check early before any state mutation */ + if (is_ignored_kernel_symbol(event->ksymbol.name)) + return 0; + + map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); + if (!map) { dso = dso__new(event->ksymbol.name); @@ -790,6 +802,10 @@ static int machine__process_ksymbol_unregister(struct machine *machine, struct symbol *sym; struct map *map; + /* Ignore mapping symbols in ksymbol events */ + if (is_ignored_kernel_symbol(event->ksymbol.name)) + return 0; + map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); if (!map) return 0; @@ -814,6 +830,11 @@ int machine__process_ksymbol(struct machine *machine __maybe_unused, if (dump_trace) perf_event__fprintf_ksymbol(event, stdout); + if (event->header.size < offsetof(struct perf_record_ksymbol, name) + 2 || + !memchr(event->ksymbol.name, '\0', + event->header.size - offsetof(struct perf_record_ksymbol, name))) + return -EINVAL; + /* no need to process non-JIT BPF as it cannot get samples */ if (event->ksymbol.len == 0) return 0; @@ -1078,7 +1099,7 @@ static u64 find_entry_trampoline(struct dso *dso) unsigned int i; for (; sym; sym = dso__next_symbol(sym)) { - if (sym->binding != STB_GLOBAL) + if (symbol__binding(sym) != STB_GLOBAL) continue; for (i = 0; i < ARRAY_SIZE(syms); i++) { if (!strcmp(sym->name, syms[i])) @@ -1239,9 +1260,9 @@ int machines__create_guest_kernel_maps(struct machines *machines) namelist[i]->d_name); continue; } - sprintf(path, "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); + snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name); ret = access(path, R_OK); if (ret) { pr_debug("Can't access file %s\n", path); @@ -1319,7 +1340,7 @@ static char *get_kernel_version(const char *root_dir) char *name, *tmp; const char *prefix = "Linux version "; - sprintf(version, "%s/proc/version", root_dir); + snprintf(version, sizeof(version), "%s/proc/version", root_dir); file = fopen(version, "r"); if (!file) return NULL; @@ -1522,22 +1543,30 @@ static void machine__set_kernel_mmap(struct machine *machine, map__set_end(machine->vmlinux_map, ~0ULL); } -static int machine__update_kernel_mmap(struct machine *machine, - u64 start, u64 end) +struct kernel_mmap_mutation_ctx { + u64 start; + u64 end; +}; + +static int kernel_mmap_mutate_cb(struct map *map, void *data) { - struct map *orig, *updated; - int err; + struct kernel_mmap_mutation_ctx *ctx = data; - orig = machine->vmlinux_map; - updated = map__get(orig); + map__set_start(map, ctx->start); + map__set_end(map, ctx->end); + if (ctx->start == 0 && ctx->end == 0) + map__set_end(map, ~0ULL); + return 0; +} - machine->vmlinux_map = updated; - maps__remove(machine__kernel_maps(machine), orig); - machine__set_kernel_mmap(machine, start, end); - err = maps__insert(machine__kernel_maps(machine), updated); - map__put(orig); +static int machine__update_kernel_mmap(struct machine *machine, + u64 start, u64 end) +{ + struct kernel_mmap_mutation_ctx ctx = { .start = start, .end = end }; - return err; + return maps__mutate_mapping(machine__kernel_maps(machine), + machine->vmlinux_map, + kernel_mmap_mutate_cb, &ctx); } int machine__create_kernel_maps(struct machine *machine) @@ -1611,10 +1640,24 @@ static bool machine__uses_kcore(struct machine *machine) return dsos__for_each_dso(&machine->dsos, machine__uses_kcore_cb, NULL) != 0 ? true : false; } +static bool machine__is(struct machine *machine, uint16_t e_machine) +{ + if (!machine) + return false; + + if (!machine->env) { + if (machine__is_host(machine)) + return e_machine == EM_HOST; + return false; + } + + return perf_env__e_machine(machine->env, NULL) == e_machine; +} + static bool perf_event__is_extra_kernel_mmap(struct machine *machine, struct extra_kernel_map *xm) { - return machine__is(machine, "x86_64") && + return machine__is(machine, EM_X86_64) && is_entry_trampoline(xm->name); } @@ -2770,7 +2813,7 @@ static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread, static u64 get_leaf_frame_caller(struct perf_sample *sample, struct thread *thread, int usr_idx) { - if (machine__normalized_is(maps__machine(thread__maps(thread)), "arm64")) + if (thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL) == EM_AARCH64) return get_leaf_frame_caller_aarch64(sample, thread, usr_idx); else return 0; @@ -2778,13 +2821,13 @@ static u64 get_leaf_frame_caller(struct perf_sample *sample, static int thread__resolve_callchain_sample(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, int max_stack, bool symbols) { + struct evsel *evsel = sample->evsel; struct branch_stack *branch = sample->branch_stack; struct branch_entry *entries = perf_sample__branch_entries(sample); struct ip_callchain *chain = sample->callchain; @@ -2986,10 +3029,11 @@ static int unwind_entry(struct unwind_entry *entry, void *arg) static int thread__resolve_callchain_unwind(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, int max_stack, bool symbols) { + struct evsel *evsel = sample->evsel; + /* Can we do dwarf post unwind? */ if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) && (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER))) @@ -3009,7 +3053,6 @@ static int thread__resolve_callchain_unwind(struct thread *thread, int __thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -3025,22 +3068,22 @@ int __thread__resolve_callchain(struct thread *thread, if (callchain_param.order == ORDER_CALLEE) { ret = thread__resolve_callchain_sample(thread, cursor, - evsel, sample, + sample, parent, root_al, max_stack, symbols); if (ret) return ret; ret = thread__resolve_callchain_unwind(thread, cursor, - evsel, sample, + sample, max_stack, symbols); } else { ret = thread__resolve_callchain_unwind(thread, cursor, - evsel, sample, + sample, max_stack, symbols); if (ret) return ret; ret = thread__resolve_callchain_sample(thread, cursor, - evsel, sample, + sample, parent, root_al, max_stack, symbols); } @@ -3141,20 +3184,6 @@ int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid, return 0; } -/* - * Compares the raw arch string. N.B. see instead perf_env__arch() or - * machine__normalized_is() if a normalized arch is needed. - */ -bool machine__is(struct machine *machine, const char *arch) -{ - return machine && !strcmp(perf_env__raw_arch(machine->env), arch); -} - -bool machine__normalized_is(struct machine *machine, const char *arch) -{ - return machine && !strcmp(perf_env__arch(machine->env), arch); -} - int machine__nr_cpus_avail(struct machine *machine) { return machine ? perf_env__nr_cpus_avail(machine->env) : 0; @@ -3181,7 +3210,7 @@ int machine__get_kernel_start(struct machine *machine) * start of kernel text, but still above 2^63. So leave * kernel_start = 1ULL << 63 for x86_64. */ - if (!err && !machine__is(machine, "x86_64")) + if (!err && !machine__is(machine, EM_X86_64)) machine->kernel_start = map__start(map); } return err; diff --git a/tools/perf/util/machine.h b/tools/perf/util/machine.h index 22a42c5825fa..26f9827062f5 100644 --- a/tools/perf/util/machine.h +++ b/tools/perf/util/machine.h @@ -152,7 +152,7 @@ struct machines { struct rb_root_cached guests; }; -void machines__init(struct machines *machines); +int machines__init(struct machines *machines); void machines__exit(struct machines *machines); void machines__process_guests(struct machines *machines, @@ -187,7 +187,6 @@ struct callchain_cursor; int __thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -196,7 +195,6 @@ int __thread__resolve_callchain(struct thread *thread, static inline int thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -204,7 +202,6 @@ static inline int thread__resolve_callchain(struct thread *thread, { return __thread__resolve_callchain(thread, cursor, - evsel, sample, parent, root_al, @@ -227,8 +224,6 @@ static inline bool machine__is_host(struct machine *machine) } bool machine__is_lock_function(struct machine *machine, u64 addr); -bool machine__is(struct machine *machine, const char *arch); -bool machine__normalized_is(struct machine *machine, const char *arch); int machine__nr_cpus_avail(struct machine *machine); struct thread *machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid); diff --git a/tools/perf/util/map.h b/tools/perf/util/map.h index 979b3e11b9bc..fb0279810ae9 100644 --- a/tools/perf/util/map.h +++ b/tools/perf/util/map.h @@ -2,14 +2,13 @@ #ifndef __PERF_MAP_H #define __PERF_MAP_H -#include -#include -#include -#include +#include #include #include -#include + +#include #include + #include struct dso; diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 81a97ac34077..f808df2fe77b 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -40,6 +40,7 @@ DECLARE_RC_STRUCT(maps) { #ifdef HAVE_LIBUNWIND_SUPPORT void *addr_space; const struct unwind_libunwind_ops *unwind_libunwind_ops; + uint16_t e_machine; #endif #ifdef HAVE_LIBDW_SUPPORT void *libdw_addr_space_dwfl; @@ -197,14 +198,14 @@ void maps__set_addr_space(struct maps *maps, void *addr_space) RC_CHK_ACCESS(maps)->addr_space = addr_space; } -const struct unwind_libunwind_ops *maps__unwind_libunwind_ops(const struct maps *maps) +uint16_t maps__e_machine(const struct maps *maps) { - return RC_CHK_ACCESS(maps)->unwind_libunwind_ops; + return RC_CHK_ACCESS(maps)->e_machine; } -void maps__set_unwind_libunwind_ops(struct maps *maps, const struct unwind_libunwind_ops *ops) +void maps__set_e_machine(struct maps *maps, uint16_t e_machine) { - RC_CHK_ACCESS(maps)->unwind_libunwind_ops = ops; + RC_CHK_ACCESS(maps)->e_machine = e_machine; } #endif #ifdef HAVE_LIBDW_SUPPORT @@ -575,6 +576,48 @@ void maps__remove(struct maps *maps, struct map *map) #endif } +/** + * maps__mutate_mapping - Apply write-protected mutations to a map. + * @maps: The maps collection containing the map. + * @map: The map to mutate. + * @mutate_cb: Callback function that performs the actual mutations. + * @data: Private data passed to the callback. + * + * This acquires the write lock on the maps semaphore to safely protect + * concurrent readers from seeing partially mutated or unsorted map boundaries. + * + * WARNING: Acquiring down_write() here can trigger a recursive self-deadlock if + * the caller already holds the read lock (e.g., during maps__for_each_map() or + * maps__find() iteration paths that trigger lazy symbol loading). To completely + * avoid this deadlock, all kernel/module maps must be pre-loaded up-front (via + * maps__load_maps()) under a clean, single-threaded context before entering + * multi-threaded event processing loops. + */ +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data) +{ + int err = 0; + + if (maps) { + down_write(maps__lock(maps)); + + err = mutate_cb(map, data); + + RC_CHK_ACCESS(maps)->maps_by_address_sorted = false; + RC_CHK_ACCESS(maps)->maps_by_name_sorted = false; + + up_write(maps__lock(maps)); + +#ifdef HAVE_LIBDW_SUPPORT + libdw__invalidate_dwfl(maps, maps__libdw_addr_space_dwfl(maps)); +#endif + } else { + err = mutate_cb(map, data); + } + + return err; +} + bool maps__empty(struct maps *maps) { bool res; @@ -625,6 +668,41 @@ int maps__for_each_map(struct maps *maps, int (*cb)(struct map *map, void *data) return ret; } +int maps__load_maps(struct maps *maps) +{ + struct map **maps_copy; + unsigned int nr_maps; + int err = 0; + + if (!maps) + return 0; + + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + if (nr_maps == 0) { + up_read(maps__lock(maps)); + return 0; + } + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (!maps_copy) { + up_read(maps__lock(maps)); + return -ENOMEM; + } + for (unsigned int i = 0; i < nr_maps; i++) + maps_copy[i] = map__get(maps__maps_by_address(maps)[i]); + up_read(maps__lock(maps)); + + for (unsigned int i = 0; i < nr_maps; i++) { + if (map__load(maps_copy[i]) < 0) { + pr_warning("Failed to load map %s\n", dso__name(map__dso(maps_copy[i]))); + err = -1; + } + map__put(maps_copy[i]); + } + free(maps_copy); + return err; +} + void maps__remove_maps(struct maps *maps, bool (*cb)(struct map *map, void *data), void *data) { struct map **maps_by_address; @@ -667,40 +745,57 @@ struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp) return result; } -struct maps__find_symbol_by_name_args { - struct map **mapp; - const char *name; - struct symbol *sym; -}; - -static int maps__find_symbol_by_name_cb(struct map *map, void *data) +struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) { - struct maps__find_symbol_by_name_args *args = data; + struct map **maps_copy; + unsigned int nr_maps; + struct symbol *sym = NULL; - args->sym = map__find_symbol_by_name(map, args->name); - if (!args->sym) - return 0; + if (!maps) + return NULL; - if (!map__contains_symbol(map, args->sym)) { - args->sym = NULL; - return 0; + /* + * First, ensure all maps are loaded. We pre-load them outside of any + * read-to-write locks to avoid deadlocks. Even if some fail, we proceed. + */ + maps__load_maps(maps); + + /* + * Create a local snapshot of the maps while holding the read lock. + * This prevents deadlocking if iteration triggers further map insertions. + */ + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (maps_copy) { + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps__maps_by_address(maps)[i]; + + maps_copy[i] = map__get(map); + } } + up_read(maps__lock(maps)); - if (args->mapp != NULL) - *args->mapp = map__get(map); - return 1; -} + if (!maps_copy) + return NULL; -struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) -{ - struct maps__find_symbol_by_name_args args = { - .mapp = mapp, - .name = name, - .sym = NULL, - }; + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps_copy[i]; + + sym = map__find_symbol_by_name(map, name); + if (sym && map__contains_symbol(map, sym)) { + if (mapp) + *mapp = map__get(map); + break; + } + sym = NULL; + } + + for (unsigned int i = 0; i < nr_maps; i++) + map__put(maps_copy[i]); - maps__for_each_map(maps, maps__find_symbol_by_name_cb, &args); - return args.sym; + free(maps_copy); + return sym; } int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) @@ -1039,6 +1134,14 @@ int maps__copy_from(struct maps *dest, struct maps *parent) down_write(maps__lock(dest)); down_read(maps__lock(parent)); +#ifdef HAVE_LIBUNWIND_SUPPORT + err = unwind__prepare_access(dest, maps__e_machine(parent)); + if (err) { + up_read(maps__lock(parent)); + up_write(maps__lock(dest)); + return err; + } +#endif parent_maps_by_address = maps__maps_by_address(parent); n = maps__nr_maps(parent); if (maps__nr_maps(dest) == 0) { @@ -1068,14 +1171,11 @@ int maps__copy_from(struct maps *dest, struct maps *parent) if (!new) err = -ENOMEM; else { - err = unwind__prepare_access(dest, new, NULL); - if (!err) { - dest_maps_by_address[i] = new; - map__set_kmap_maps(new, dest); - if (dest_maps_by_name) - dest_maps_by_name[i] = map__get(new); - RC_CHK_ACCESS(dest)->nr_maps = i + 1; - } + dest_maps_by_address[i] = new; + map__set_kmap_maps(new, dest); + if (dest_maps_by_name) + dest_maps_by_name[i] = map__get(new); + RC_CHK_ACCESS(dest)->nr_maps = i + 1; } if (err) map__put(new); @@ -1093,9 +1193,7 @@ int maps__copy_from(struct maps *dest, struct maps *parent) if (!new) err = -ENOMEM; else { - err = unwind__prepare_access(dest, new, NULL); - if (!err) - err = __maps__insert(dest, new); + err = __maps__insert(dest, new); } map__put(new); } diff --git a/tools/perf/util/maps.h b/tools/perf/util/maps.h index 20c52084ba9e..4ec9b7453a3b 100644 --- a/tools/perf/util/maps.h +++ b/tools/perf/util/maps.h @@ -49,8 +49,8 @@ refcount_t *maps__refcnt(struct maps *maps); /* Test only. */ #ifdef HAVE_LIBUNWIND_SUPPORT void *maps__addr_space(const struct maps *maps); void maps__set_addr_space(struct maps *maps, void *addr_space); -const struct unwind_libunwind_ops *maps__unwind_libunwind_ops(const struct maps *maps); -void maps__set_unwind_libunwind_ops(struct maps *maps, const struct unwind_libunwind_ops *ops); +uint16_t maps__e_machine(const struct maps *maps); +void maps__set_e_machine(struct maps *maps, uint16_t e_machine); #endif #ifdef HAVE_LIBDW_SUPPORT void *maps__libdw_addr_space_dwfl(const struct maps *maps); @@ -59,8 +59,11 @@ void maps__set_libdw_addr_space_dwfl(struct maps *maps, void *dwfl); size_t maps__fprintf(struct maps *maps, FILE *fp); +int maps__load_maps(struct maps *maps); int maps__insert(struct maps *maps, struct map *map); void maps__remove(struct maps *maps, struct map *map); +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data); struct map *maps__find(struct maps *maps, u64 addr); struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp); diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 4db9578efd81..69bfa2a723b2 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -214,7 +214,7 @@ static void metric__free(struct metric *m) zfree(&m->metric_refs); expr__ctx_free(m->pctx); zfree(&m->modifier); - evlist__delete(m->evlist); + evlist__put(m->evlist); free(m); } @@ -295,7 +295,8 @@ static int setup_metric_events(const char *pmu, struct hashmap *ids, const char *metric_id; struct evsel *ev; size_t ids_size, matched_events, i; - bool all_pmus = !strcmp(pmu, "all") || perf_pmus__num_core_pmus() == 1 || !is_pmu_core(pmu); + bool all_pmus = !strcmp(pmu, "all") || !strcmp(pmu, "default_core") || + perf_pmus__num_core_pmus() == 1 || !is_pmu_core(pmu); *out_metric_events = NULL; ids_size = hashmap__size(ids); @@ -415,14 +416,9 @@ static int metricgroup__sys_event_iter(const struct pmu_metric *pm, if (!pm->metric_expr || !pm->compat) return 0; - while ((pmu = perf_pmus__scan(pmu))) { - - if (!pmu->id || !pmu_uncore_identifier_match(pm->compat, pmu->id)) - continue; - - return d->fn(pm, table, d->data); - } - return 0; + /* Only process with the iterator if there is a a PMU that matches the ID. */ + pmu = perf_pmus__scan_for_uncore_id(pmu, pm->compat); + return pmu ? d->fn(pm, table, d->data) : 0; } int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, @@ -1266,7 +1262,8 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, struct expr_parse_ctx *ids, const char *modifier, bool group_events, const bool tool_events[TOOL_PMU__EVENT_MAX], struct evlist **out_evlist, - const char *filter_pmu) + const char *filter_pmu, + bool cputype_filter) { struct parse_events_error parse_error; struct evlist *parsed_evlist; @@ -1321,7 +1318,9 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, pr_debug("Parsing metric events '%s'\n", events.buf); parse_events_error__init(&parse_error); ret = __parse_events(parsed_evlist, events.buf, filter_pmu, - &parse_error, fake_pmu, /*warn_if_reordered=*/false, + cputype_filter, + &parse_error, fake_pmu, + /*warn_if_reordered=*/false, /*fake_tp=*/false); if (ret) { parse_events_error__print(&parse_error, events.buf); @@ -1335,7 +1334,7 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu, parsed_evlist = NULL; err_out: parse_events_error__exit(&parse_error); - evlist__delete(parsed_evlist); + evlist__put(parsed_evlist); strbuf_release(&events); return ret; } @@ -1386,7 +1385,7 @@ static struct evsel *pick_display_evsel(struct list_head *metric_list, } static int parse_groups(struct evlist *perf_evlist, - const char *pmu, const char *str, + const char *pmu, bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, bool metric_no_threshold, @@ -1424,7 +1423,8 @@ static int parse_groups(struct evlist *perf_evlist, /*group_events=*/false, tool_events, &combined_evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); } if (combined) expr__ctx_free(combined); @@ -1480,7 +1480,8 @@ static int parse_groups(struct evlist *perf_evlist, if (!metric_evlist) { ret = parse_ids(metric_no_merge, fake_pmu, m->pctx, m->modifier, m->group_events, tool_events, &m->evlist, - (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu); + (pmu && strcmp(pmu, "all") == 0) ? NULL : pmu, + cputype_filter); if (ret) goto out; @@ -1494,7 +1495,7 @@ static int parse_groups(struct evlist *perf_evlist, goto out; } - me = metricgroup__lookup(&perf_evlist->metric_events, + me = metricgroup__lookup(evlist__metric_events(perf_evlist), pick_display_evsel(&metric_list, metric_events), /*create=*/true); @@ -1545,22 +1546,26 @@ static int parse_groups(struct evlist *perf_evlist, if (combined_evlist) { - evlist__splice_list_tail(perf_evlist, &combined_evlist->core.entries); - evlist__delete(combined_evlist); + evlist__splice_list_tail(perf_evlist, &evlist__core(combined_evlist)->entries); + evlist__put(combined_evlist); + combined_evlist = NULL; } list_for_each_entry(m, &metric_list, nd) { if (m->evlist) - evlist__splice_list_tail(perf_evlist, &m->evlist->core.entries); + evlist__splice_list_tail(perf_evlist, &evlist__core(m->evlist)->entries); } out: + if (combined_evlist) + evlist__put(combined_evlist); metricgroup__free_metrics(&metric_list); return ret; } int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -1574,16 +1579,17 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, if (hardware_aware_grouping) pr_debug("Use hardware aware grouping instead of traditional metric grouping method\n"); - return parse_groups(perf_evlist, pmu, str, metric_no_group, metric_no_merge, + return parse_groups(perf_evlist, pmu, cputype_filter, str, metric_no_group, metric_no_merge, metric_no_threshold, user_requested_cpu_list, system_wide, /*fake_pmu=*/false, table); } int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str) + const char *str, + bool cputype_filter) { - return parse_groups(evlist, "all", str, + return parse_groups(evlist, "all", cputype_filter, str, /*metric_no_group=*/false, /*metric_no_merge=*/false, /*metric_no_threshold=*/false, diff --git a/tools/perf/util/metricgroup.h b/tools/perf/util/metricgroup.h index 4be6bfc13c46..6a66f14dd01b 100644 --- a/tools/perf/util/metricgroup.h +++ b/tools/perf/util/metricgroup.h @@ -71,6 +71,7 @@ struct metric_event *metricgroup__lookup(struct rblist *metric_events, bool create); int metricgroup__parse_groups(struct evlist *perf_evlist, const char *pmu, + bool cputype_filter, const char *str, bool metric_no_group, bool metric_no_merge, @@ -80,7 +81,8 @@ int metricgroup__parse_groups(struct evlist *perf_evlist, bool hardware_aware_grouping); int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_metrics_table *table, - const char *str); + const char *str, + bool cputype_filter); int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, void *data); diff --git a/tools/perf/util/mmap.c b/tools/perf/util/mmap.c index b69f926d314b..358e70c4f3ed 100644 --- a/tools/perf/util/mmap.c +++ b/tools/perf/util/mmap.c @@ -89,10 +89,10 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { - if (map->aio.data[idx]) { - munmap(map->aio.data[idx], mmap__mmap_len(map)); - map->aio.data[idx] = NULL; - } + if (!map->aio.data || !map->aio.data[idx]) + return; + munmap(map->aio.data[idx], mmap__mmap_len(map)); + map->aio.data[idx] = NULL; } static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity) @@ -104,9 +104,15 @@ static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, i int err = 0; if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) { + int node; + data = map->aio.data[idx]; mmap_len = mmap__mmap_len(map); - node_index = cpu__get_node(cpu); + node = cpu__get_node(cpu); + /* -1 sign-extends to ULONG_MAX, wrapping bitmap_zalloc(0) and OOB __set_bit */ + if (node < 0) + return 0; + node_index = node; node_mask = bitmap_zalloc(node_index + 1); if (!node_mask) { pr_err("Failed to allocate node mask for mbind: error %m\n"); @@ -135,6 +141,8 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { + if (!map->aio.data) + return; zfree(&(map->aio.data[idx])); } @@ -230,6 +238,8 @@ static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused) void mmap__munmap(struct mmap *map) { bitmap_free(map->affinity_mask.bits); + map->affinity_mask.bits = NULL; + map->affinity_mask.nbits = 0; zstd_fini(&map->zstd_data); diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 1497e1f2a08c..cc7ad331a49f 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -429,6 +429,9 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state, if (parse_state->pmu_filter == NULL) return false; + if (parse_state->cputype_filter && !pmu->is_core) + return false; + return perf_pmu__wildcard_match(pmu, parse_state->pmu_filter) == 0; } @@ -2251,6 +2254,33 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) } last_event_was_forced_leader = (force_grouped_leader == pos); } + + /* + * Make sure the first wildcard match is the earliest entry in the list. + * Since list_sort might have reordered the aliases, the original leader + * might not be at the head of the list anymore. We find the first + * alias in the sorted list and make it the new leader, and redirect + * all other aliases to it. + */ + list_for_each_entry(pos, list, core.node) { + struct evsel *orig_leader = pos->first_wildcard_match; + + if (!orig_leader) + continue; + + if (orig_leader->first_wildcard_match) { + /* Original leader was redirected to a new leader */ + pos->first_wildcard_match = orig_leader->first_wildcard_match; + } else if (pos->core.idx < orig_leader->core.idx) { + /* + * We are earlier than the original leader in sorted order, + * and no earlier alias has claimed leadership yet. + */ + orig_leader->first_wildcard_match = pos; + pos->first_wildcard_match = NULL; + } + } + list_for_each_entry(pos, list, core.node) { struct evsel *pos_leader = evsel__leader(pos); @@ -2261,18 +2291,20 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) return (idx_changed || num_leaders != orig_num_leaders) ? 1 : 0; } -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, +int __parse_events(struct evlist *evlist, const char *str, + const char *pmu_filter, bool cputype_filter, struct parse_events_error *err, bool fake_pmu, bool warn_if_reordered, bool fake_tp) { struct parse_events_state parse_state = { - .list = LIST_HEAD_INIT(parse_state.list), - .idx = evlist->core.nr_entries, - .error = err, - .stoken = PE_START_EVENTS, - .fake_pmu = fake_pmu, - .fake_tp = fake_tp, - .pmu_filter = pmu_filter, + .list = LIST_HEAD_INIT(parse_state.list), + .idx = evlist__nr_entries(evlist), + .error = err, + .stoken = PE_START_EVENTS, + .fake_pmu = fake_pmu, + .fake_tp = fake_tp, + .pmu_filter = pmu_filter, + .cputype_filter = cputype_filter, .match_legacy_cache_terms = true, }; int ret, ret2; @@ -2285,15 +2317,15 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte } ret2 = parse_events__sort_events_and_fix_groups(&parse_state.list); - if (ret2 < 0) - return ret; + if (ret2 < 0 && !ret) + ret = ret2; /* * Add list to the evlist even with errors to allow callers to clean up. */ evlist__splice_list_tail(evlist, &parse_state.list); - if (ret2 && warn_if_reordered && !parse_state.wild_card_pmus) { + if (ret2 > 0 && warn_if_reordered && !parse_state.wild_card_pmus) { evlist__uniquify_evsel_names(evlist, &stat_config); pr_warning("WARNING: events were regrouped to match PMUs\n"); @@ -2316,7 +2348,7 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte /* * There are 2 users - builtin-record and builtin-test objects. - * Both call evlist__delete in case of error, so we dont + * Both call evlist__put in case of error, so we dont * need to bother. */ return ret; @@ -2491,8 +2523,9 @@ int parse_events_option(const struct option *opt, const char *str, int ret; parse_events_error__init(&err); - ret = __parse_events(*args->evlistp, str, args->pmu_filter, &err, - /*fake_pmu=*/false, /*warn_if_reordered=*/true, + ret = __parse_events(*args->evlistp, str, args->pmu_filter, + args->cputype_filter, &err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, /*fake_tp=*/false); if (ret) { @@ -2519,7 +2552,7 @@ int parse_events_option_new_evlist(const struct option *opt, const char *str, in } ret = parse_events_option(opt, str, unset); if (ret) { - evlist__delete(*args->evlistp); + evlist__put(*args->evlistp); *args->evlistp = NULL; } @@ -2541,7 +2574,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, * * So no need to WARN here, let *func do this. */ - if (evlist->core.nr_entries > 0) + if (evlist__nr_entries(evlist) > 0) last = evlist__last(evlist); do { @@ -2551,7 +2584,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist, if (!last) return 0; - if (last->core.node.prev == &evlist->core.entries) + if (last->core.node.prev == &evlist__core(evlist)->entries) return 0; last = list_entry(last->core.node.prev, struct evsel, core.node); } while (!last->cmdline_group_boundary); diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 3577ab213730..b14c832b03a1 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -26,20 +26,23 @@ const char *event_type(size_t type); struct parse_events_option_args { struct evlist **evlistp; const char *pmu_filter; + bool cputype_filter; }; int parse_events_option(const struct option *opt, const char *str, int unset); int parse_events_option_new_evlist(const struct option *opt, const char *str, int unset); -__attribute__((nonnull(1, 2, 4))) -int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, - struct parse_events_error *error, bool fake_pmu, - bool warn_if_reordered, bool fake_tp); +__attribute__((nonnull(1, 2, 5))) int +__parse_events(struct evlist *evlist, const char *str, const char *pmu_filter, + bool cputype_filter, struct parse_events_error *error, + bool fake_pmu, bool warn_if_reordered, bool fake_tp); __attribute__((nonnull(1, 2, 3))) static inline int parse_events(struct evlist *evlist, const char *str, struct parse_events_error *err) { - return __parse_events(evlist, str, /*pmu_filter=*/NULL, err, /*fake_pmu=*/false, - /*warn_if_reordered=*/true, /*fake_tp=*/false); + return __parse_events(evlist, str, /*pmu_filter=*/NULL, + /*cputype_filter=*/false, err, /*fake_pmu=*/false, + /*warn_if_reordered=*/true, + /*fake_tp=*/false); } int parse_event(struct evlist *evlist, const char *str); @@ -161,6 +164,8 @@ struct parse_events_state { bool fake_tp; /* If non-null, when wildcard matching only match the given PMU. */ const char *pmu_filter; + /* If true, the pmu_filter was set by --cputype option. */ + bool cputype_filter; /* Should PE_LEGACY_NAME tokens be generated for config terms? */ bool match_legacy_cache_terms; /* Were multiple PMUs scanned to find events? */ diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index c194de5ec1ec..b531b1f0ceb3 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -47,7 +47,7 @@ static void free_list_evsel(struct list_head* list_evsel) list_for_each_entry_safe(evsel, tmp, list_evsel, core.node) { list_del_init(&evsel->core.node); - evsel__delete(evsel); + evsel__put(evsel); } free(list_evsel); } diff --git a/tools/perf/util/perf-regs-arch/perf_regs_riscv.c b/tools/perf/util/perf-regs-arch/perf_regs_riscv.c index 5b5f21fcba8c..bf769304c97c 100644 --- a/tools/perf/util/perf-regs-arch/perf_regs_riscv.c +++ b/tools/perf/util/perf-regs-arch/perf_regs_riscv.c @@ -1,8 +1,136 @@ // SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include "../debug.h" #include "../perf_regs.h" #include "../../arch/riscv/include/perf_regs.h" +/* + * RISC-V SDT argument formats (GCC 'nor' constraint): + * + * Register: REG e.g. a0, t1, s0, sp + * Memory: NUM(REG) e.g. 8(a0), -20(s0) + * Constant: NUM e.g. 99 (not supported by uprobe, skip) + * + * Note: 'zero' (x0) is hardwired to 0 and not in pt_regs; skip it. + * + * Uprobe target format: + * Register: %REG e.g. %a0 + * Memory: +NUM(%REG) or -NUM(%REG) + */ + +/* RISC-V register ABI names: ra, sp, gp, tp, t0-t6, s0-s11, a0-a7 */ +#define SDT_OP_REGEX1 "^(ra|sp|gp|tp|t[0-6]|s[0-9]|s1[01]|a[0-7])$" + +/* RISC-V memory operand: [-]NUM(REG) */ +#define SDT_OP_REGEX2 "^(\\-)?([0-9]+)\\((ra|sp|gp|tp|t[0-6]|s[0-9]|s1[01]|a[0-7])\\)$" + +static regex_t sdt_op_regex1, sdt_op_regex2; + +static int sdt_init_op_regex(void) +{ + static int initialized; + int ret = 0; + + if (initialized) + return 0; + + ret = regcomp(&sdt_op_regex1, SDT_OP_REGEX1, REG_EXTENDED); + if (ret) + goto error; + + ret = regcomp(&sdt_op_regex2, SDT_OP_REGEX2, REG_EXTENDED); + if (ret) + goto free_regex1; + + initialized = 1; + return 0; + +free_regex1: + regfree(&sdt_op_regex1); +error: + pr_debug4("Regex compilation error.\n"); + return -ret; +} + +/* + * Parse OP and convert it into uprobe format. + * Possible variants of OP (RISC-V, GCC 'nor' constraint): + * + * Format Example Uprobe + * ---------------------------------------- + * REG a0 %a0 + * NUM(REG) 8(a0) +8(%a0) + * -NUM(REG) -20(s0) -20(%s0) + * NUM 99 (skip, constant not supported) + */ +int __perf_sdt_arg_parse_op_riscv(char *old_op, char **new_op) +{ + int ret, new_len; + regmatch_t rm[4]; + char prefix; + + /* + * Constant argument: pure integer with no trailing '(' (e.g. "99", "-1"). + * uprobe does not support immediate values, so skip them. + * Memory operands like "8(a0)" or "-20(s0)" contain '(' so are NOT + * treated as constants here; they will be matched by REGEX2 below. + */ + if (strchr(old_op, '(') == NULL && + ((*old_op >= '0' && *old_op <= '9') || + (*old_op == '-' && old_op[1] >= '0' && old_op[1] <= '9'))) { + pr_debug4("Skipping unsupported SDT argument: %s\n", old_op); + return SDT_ARG_SKIP; + } + + ret = sdt_init_op_regex(); + if (ret < 0) + return ret; + + if (!regexec(&sdt_op_regex1, old_op, 2, rm, 0)) { + /* REG --> %REG */ + new_len = 2; /* % NULL */ + new_len += (int)(rm[1].rm_eo - rm[1].rm_so); + + *new_op = zalloc(new_len); + if (!*new_op) + return -ENOMEM; + + scnprintf(*new_op, new_len, "%%%.*s", + (int)(rm[1].rm_eo - rm[1].rm_so), old_op + rm[1].rm_so); + } else if (!regexec(&sdt_op_regex2, old_op, 4, rm, 0)) { + /* + * NUM(REG) or -NUM(REG) --> +NUM(%REG) or -NUM(%REG) + * rm[1]: optional '-' + * rm[2]: decimal offset + * rm[3]: register name + */ + prefix = (rm[1].rm_so == -1) ? '+' : '-'; + + new_len = 5; /* sign ( % ) NULL */ + new_len += (int)(rm[2].rm_eo - rm[2].rm_so); + new_len += (int)(rm[3].rm_eo - rm[3].rm_so); + + *new_op = zalloc(new_len); + if (!*new_op) + return -ENOMEM; + + scnprintf(*new_op, new_len, "%c%.*s(%%%.*s)", prefix, + (int)(rm[2].rm_eo - rm[2].rm_so), old_op + rm[2].rm_so, + (int)(rm[3].rm_eo - rm[3].rm_so), old_op + rm[3].rm_so); + } else { + pr_debug4("Skipping unsupported SDT argument: %s\n", old_op); + return SDT_ARG_SKIP; + } + + return SDT_ARG_VALID; +} + uint64_t __perf_reg_mask_riscv(bool intr __maybe_unused) { return PERF_REGS_MASK; diff --git a/tools/perf/util/perf_api_probe.c b/tools/perf/util/perf_api_probe.c index 6ecf38314f01..f61c4ec52827 100644 --- a/tools/perf/util/perf_api_probe.c +++ b/tools/perf/util/perf_api_probe.c @@ -1,14 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include "perf_api_probe.h" -#include "perf-sys.h" -#include "util/cloexec.h" -#include "util/evlist.h" -#include "util/evsel.h" -#include "util/parse-events.h" -#include "util/perf_api_probe.h" -#include #include +#include + +#include "cloexec.h" +#include "evlist.h" +#include "evsel.h" +#include "parse-events.h" +#include "perf-sys.h" +#include "pmu.h" +#include "pmus.h" + typedef void (*setup_probe_fn_t)(struct evsel *evsel); static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const char *str) @@ -53,7 +57,7 @@ static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const cha err = 0; out_delete: - evlist__delete(evlist); + evlist__put(evlist); return err; } diff --git a/tools/perf/util/perf_event_attr_fprintf.c b/tools/perf/util/perf_event_attr_fprintf.c index 741c3d657a8b..3933639d76c5 100644 --- a/tools/perf/util/perf_event_attr_fprintf.c +++ b/tools/perf/util/perf_event_attr_fprintf.c @@ -275,24 +275,56 @@ static void __p_config_id(struct perf_pmu *pmu, char *buf, size_t size, u32 type #define p_type_id(val) __p_type_id(buf, BUF_SIZE, pmu, val) #define p_config_id(val) __p_config_id(pmu, buf, BUF_SIZE, attr->type, val) -#define PRINT_ATTRn(_n, _f, _p, _a) \ -do { \ - if (_a || attr->_f) { \ - _p(attr->_f); \ - ret += attr__fprintf(fp, _n, buf, priv);\ - } \ +#define PRINT_ATTRn(_n, _f, _p, _a) \ +do { \ + if (attr_size >= offsetof(struct perf_event_attr, _f) + \ + sizeof(attr->_f) && \ + (_a || attr->_f)) { \ + _p(attr->_f); \ + ret += attr__fprintf(fp, _n, buf, priv); \ + } \ +} while (0) + +/* bitfield members share an offset; most are within PERF_ATTR_SIZE_VER0 */ +#define PRINT_ATTRn_bf(_n, _f, _p, _a) \ +do { \ + if (_a || attr->_f) { \ + _p(attr->_f); \ + ret += attr__fprintf(fp, _n, buf, priv); \ + } \ } while (0) #define PRINT_ATTRf(_f, _p) PRINT_ATTRn(#_f, _f, _p, false) +#define PRINT_ATTRf_bf(_f, _p) PRINT_ATTRn_bf(#_f, _f, _p, false) int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, attr__fprintf_f attr__fprintf, void *priv) { struct perf_pmu *pmu = perf_pmus__find_by_type(attr->type); + /* + * size == 0 means ABI0 — the producer didn't set attr.size. + * perf_event__fprintf_attr() may pass the raw mmap'd event + * before the local copy, so default to PERF_ATTR_SIZE_VER0 + * (the ABI0 footprint) to avoid reading past the attr into + * the ID array that follows it in HEADER_ATTR events. + */ + u32 attr_size = attr->size ?: PERF_ATTR_SIZE_VER0; char buf[BUF_SIZE]; int ret = 0; - if (!pmu && (attr->type == PERF_TYPE_HARDWARE || attr->type == PERF_TYPE_HW_CACHE)) { + /* + * Cap to what we understand: all callers store the attr in a + * buffer of sizeof(*attr) bytes (perf.data read path copies + * min(attr.size, sizeof), BPF augmented path copies into a + * fixed-size value[] array). A spoofed attr->size larger + * than sizeof would cause PRINT_ATTRn to read past the + * actual buffer. + */ + if (attr_size > sizeof(*attr)) + attr_size = sizeof(*attr); + + if (!pmu && attr_size >= offsetof(struct perf_event_attr, config) + sizeof(attr->config) && + (attr->type == PERF_TYPE_HARDWARE || attr->type == PERF_TYPE_HW_CACHE)) { u32 extended_type = attr->config >> PERF_PMU_TYPE_SHIFT; if (extended_type) @@ -306,45 +338,53 @@ int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, PRINT_ATTRf(sample_type, p_sample_type); PRINT_ATTRf(read_format, p_read_format); - PRINT_ATTRf(disabled, p_unsigned); - PRINT_ATTRf(inherit, p_unsigned); - PRINT_ATTRf(pinned, p_unsigned); - PRINT_ATTRf(exclusive, p_unsigned); - PRINT_ATTRf(exclude_user, p_unsigned); - PRINT_ATTRf(exclude_kernel, p_unsigned); - PRINT_ATTRf(exclude_hv, p_unsigned); - PRINT_ATTRf(exclude_idle, p_unsigned); - PRINT_ATTRf(mmap, p_unsigned); - PRINT_ATTRf(comm, p_unsigned); - PRINT_ATTRf(freq, p_unsigned); - PRINT_ATTRf(inherit_stat, p_unsigned); - PRINT_ATTRf(enable_on_exec, p_unsigned); - PRINT_ATTRf(task, p_unsigned); - PRINT_ATTRf(watermark, p_unsigned); - PRINT_ATTRf(precise_ip, p_unsigned); - PRINT_ATTRf(mmap_data, p_unsigned); - PRINT_ATTRf(sample_id_all, p_unsigned); - PRINT_ATTRf(exclude_host, p_unsigned); - PRINT_ATTRf(exclude_guest, p_unsigned); - PRINT_ATTRf(exclude_callchain_kernel, p_unsigned); - PRINT_ATTRf(exclude_callchain_user, p_unsigned); - PRINT_ATTRf(mmap2, p_unsigned); - PRINT_ATTRf(comm_exec, p_unsigned); - PRINT_ATTRf(use_clockid, p_unsigned); - PRINT_ATTRf(context_switch, p_unsigned); - PRINT_ATTRf(write_backward, p_unsigned); - PRINT_ATTRf(namespaces, p_unsigned); - PRINT_ATTRf(ksymbol, p_unsigned); - PRINT_ATTRf(bpf_event, p_unsigned); - PRINT_ATTRf(aux_output, p_unsigned); - PRINT_ATTRf(cgroup, p_unsigned); - PRINT_ATTRf(text_poke, p_unsigned); - PRINT_ATTRf(build_id, p_unsigned); - PRINT_ATTRf(inherit_thread, p_unsigned); - PRINT_ATTRf(remove_on_exec, p_unsigned); - PRINT_ATTRf(sigtrap, p_unsigned); - PRINT_ATTRf(defer_callchain, p_unsigned); - PRINT_ATTRf(defer_output, p_unsigned); + /* + * All bitfields share a single __u64 right after read_format. + * BPF-captured attrs from perf trace may have a small size + * when the tracee passes a minimal struct, so skip the + * entire block when it's not covered. + */ + if (attr_size >= offsetof(struct perf_event_attr, wakeup_events)) { + PRINT_ATTRf_bf(disabled, p_unsigned); + PRINT_ATTRf_bf(inherit, p_unsigned); + PRINT_ATTRf_bf(pinned, p_unsigned); + PRINT_ATTRf_bf(exclusive, p_unsigned); + PRINT_ATTRf_bf(exclude_user, p_unsigned); + PRINT_ATTRf_bf(exclude_kernel, p_unsigned); + PRINT_ATTRf_bf(exclude_hv, p_unsigned); + PRINT_ATTRf_bf(exclude_idle, p_unsigned); + PRINT_ATTRf_bf(mmap, p_unsigned); + PRINT_ATTRf_bf(comm, p_unsigned); + PRINT_ATTRf_bf(freq, p_unsigned); + PRINT_ATTRf_bf(inherit_stat, p_unsigned); + PRINT_ATTRf_bf(enable_on_exec, p_unsigned); + PRINT_ATTRf_bf(task, p_unsigned); + PRINT_ATTRf_bf(watermark, p_unsigned); + PRINT_ATTRf_bf(precise_ip, p_unsigned); + PRINT_ATTRf_bf(mmap_data, p_unsigned); + PRINT_ATTRf_bf(sample_id_all, p_unsigned); + PRINT_ATTRf_bf(exclude_host, p_unsigned); + PRINT_ATTRf_bf(exclude_guest, p_unsigned); + PRINT_ATTRf_bf(exclude_callchain_kernel, p_unsigned); + PRINT_ATTRf_bf(exclude_callchain_user, p_unsigned); + PRINT_ATTRf_bf(mmap2, p_unsigned); + PRINT_ATTRf_bf(comm_exec, p_unsigned); + PRINT_ATTRf_bf(use_clockid, p_unsigned); + PRINT_ATTRf_bf(context_switch, p_unsigned); + PRINT_ATTRf_bf(write_backward, p_unsigned); + PRINT_ATTRf_bf(namespaces, p_unsigned); + PRINT_ATTRf_bf(ksymbol, p_unsigned); + PRINT_ATTRf_bf(bpf_event, p_unsigned); + PRINT_ATTRf_bf(aux_output, p_unsigned); + PRINT_ATTRf_bf(cgroup, p_unsigned); + PRINT_ATTRf_bf(text_poke, p_unsigned); + PRINT_ATTRf_bf(build_id, p_unsigned); + PRINT_ATTRf_bf(inherit_thread, p_unsigned); + PRINT_ATTRf_bf(remove_on_exec, p_unsigned); + PRINT_ATTRf_bf(sigtrap, p_unsigned); + PRINT_ATTRf_bf(defer_callchain, p_unsigned); + PRINT_ATTRf_bf(defer_output, p_unsigned); + } PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned, false); PRINT_ATTRf(bp_type, p_unsigned); @@ -359,9 +399,12 @@ int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, PRINT_ATTRf(sample_max_stack, p_unsigned); PRINT_ATTRf(aux_sample_size, p_unsigned); PRINT_ATTRf(sig_data, p_unsigned); - PRINT_ATTRf(aux_start_paused, p_unsigned); - PRINT_ATTRf(aux_pause, p_unsigned); - PRINT_ATTRf(aux_resume, p_unsigned); + /* aux_{start_paused,pause,resume} are at byte 116, past VER0 */ + if (attr_size >= offsetof(struct perf_event_attr, sig_data)) { + PRINT_ATTRf_bf(aux_start_paused, p_unsigned); + PRINT_ATTRf_bf(aux_pause, p_unsigned); + PRINT_ATTRf_bf(aux_resume, p_unsigned); + } return ret; } diff --git a/tools/perf/util/perf_regs.c b/tools/perf/util/perf_regs.c index f52b0e1f7fc7..558c143abbab 100644 --- a/tools/perf/util/perf_regs.c +++ b/tools/perf/util/perf_regs.c @@ -19,6 +19,9 @@ int perf_sdt_arg_parse_op(uint16_t e_machine, char *old_op, char **new_op) case EM_PPC64: ret = __perf_sdt_arg_parse_op_powerpc(old_op, new_op); break; + case EM_RISCV: + ret = __perf_sdt_arg_parse_op_riscv(old_op, new_op); + break; case EM_386: case EM_X86_64: ret = __perf_sdt_arg_parse_op_x86(old_op, new_op); diff --git a/tools/perf/util/perf_regs.h b/tools/perf/util/perf_regs.h index 573f0d1dfe04..79be2b791509 100644 --- a/tools/perf/util/perf_regs.h +++ b/tools/perf/util/perf_regs.h @@ -53,6 +53,7 @@ const char *__perf_reg_name_powerpc(int id); uint64_t __perf_reg_ip_powerpc(void); uint64_t __perf_reg_sp_powerpc(void); +int __perf_sdt_arg_parse_op_riscv(char *old_op, char **new_op); uint64_t __perf_reg_mask_riscv(bool intr); const char *__perf_reg_name_riscv(int id); uint64_t __perf_reg_ip_riscv(void); diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index d9043f4afbe7..f80d6b0df47a 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -85,7 +85,7 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, } pmu = perf_pmus__find_by_type((unsigned int)attr.type); - evsel = parse_events__add_event(evlist->core.nr_entries, + evsel = parse_events__add_event(evlist__nr_entries(evlist), &attr, q, /*metric_id=*/NULL, pmu); if (evsel == NULL) @@ -159,7 +159,7 @@ static bool is_libpfm_event_supported(const char *name, struct perf_cpu_map *cpu result = false; evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); return result; } diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 23337d2fa281..836e3b5615cd 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -314,7 +314,7 @@ static int perf_pmu__parse_scale(struct perf_pmu *pmu, struct perf_pmu_alias *al goto error; sret = read(fd, scale, sizeof(scale)-1); - if (sret < 0) + if (sret <= 0) goto error; if (scale[sret - 1] == '\n') @@ -346,7 +346,7 @@ static int perf_pmu__parse_unit(struct perf_pmu *pmu, struct perf_pmu_alias *ali return -1; sret = read(fd, alias->unit, UNIT_MAX_LEN); - if (sret < 0) + if (sret <= 0) goto error; close(fd); @@ -865,6 +865,12 @@ static char *pmu_id(const char *name) if (filename__read_str(path, &str, &len) < 0) return NULL; + /* empty identifier file — nothing useful */ + if (len == 0) { + free(str); + return NULL; + } + str[len - 1] = 0; /* remove line feed */ return str; @@ -2029,9 +2035,26 @@ int perf_pmu__for_each_format(struct perf_pmu *pmu, void *state, pmu_format_call return 0; } +/** + * is_pmu_core() - Check if the given PMU name corresponds to a core CPU PMU. + * @name: The PMU name to check. + * + * Core PMUs can be identified by: + * 1. Exact name match: + * - "cpu": Typically used on x86 architectures. + * - "cpum_cf": Typically used on s390 architectures (CPU Measurement Counter Facility). + * - "default_core": A generic name used to refer to the default core PMU. + * 2. Sysfs file existence check (is_sysfs_pmu_core): + * - Typically used on ARM systems or Intel hybrid architectures (e.g., "cpu_atom", + * "cpu_core"). This approach checks if the sysfs directory for the PMU + * contains a "cpus" file. + */ bool is_pmu_core(const char *name) { - return !strcmp(name, "cpu") || !strcmp(name, "cpum_cf") || is_sysfs_pmu_core(name); + return !strcmp(name, "cpu") || + !strcmp(name, "cpum_cf") || + !strcmp(name, "default_core") || + is_sysfs_pmu_core(name); } bool perf_pmu__supports_legacy_cache(const struct perf_pmu *pmu) @@ -2117,7 +2140,7 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, skip_duplicate_pmus); /* Paramemterized events have the parameters shown. */ - if (strstr(alias->terms, "=?")) { + if (!strstr(alias->terms, "=?")) { /* No parameters. */ snprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); return buf; @@ -2129,15 +2152,19 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, pr_err("Failure to parse '%s' terms '%s': %d\n", alias->name, alias->terms, ret); parse_events_terms__exit(&terms); - snprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); + scnprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); return buf; } - used = snprintf(buf, len, "%.*s/%s", (int)pmu_name_len, pmu->name, alias->name); + used = scnprintf(buf, len, "%.*s/%s", (int)pmu_name_len, pmu->name, alias->name); list_for_each_entry(term, &terms.terms, list) { + const char *name = term->config; + + if (!name) + name = parse_events__term_type_str(term->type_term); if (term->type_val == PARSE_EVENTS__TERM_TYPE_STR) - used += snprintf(buf + used, sub_non_neg(len, used), - ",%s=%s", term->config, + used += scnprintf(buf + used, sub_non_neg(len, used), + ",%s=%s", name, term->val.str); } parse_events_terms__exit(&terms); @@ -2201,6 +2228,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, int ret = 0; struct hashmap_entry *entry; size_t bkt; + size_t size_rem; if (perf_pmu__is_tracepoint(pmu)) return tp_pmu__for_each_event(pmu, state, cb); @@ -2234,17 +2262,30 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, } buf_used = strlen(buf) + 1; } + info.scale_unit = NULL; if (strlen(event->unit) || event->scale != 1.0) { - info.scale_unit = buf + buf_used; - buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, - "%G%s", event->scale, event->unit) + 1; + /* Check the remaining space */ + size_rem = sub_non_neg(sizeof(buf), buf_used); + + if (size_rem > 0) { + info.scale_unit = buf + buf_used; + buf_used += scnprintf(buf + buf_used, size_rem, "%G%s", + event->scale, event->unit) + 1; + } } info.desc = event->desc; info.long_desc = event->long_desc; - info.encoding_desc = buf + buf_used; - buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, - "%.*s/%s/", (int)pmu_name_len, info.pmu_name, event->terms) + 1; + info.encoding_desc = NULL; + + /* Check the remaining space */ + size_rem = sub_non_neg(sizeof(buf), buf_used); + if (size_rem > 0) { + info.encoding_desc = buf + buf_used; + buf_used += scnprintf(buf + buf_used, size_rem, "%.*s/%s/", + (int)pmu_name_len, info.pmu_name, event->terms) + 1; + } + info.str = event->terms; info.topic = event->topic; info.deprecated = perf_pmu_alias__check_deprecated(pmu, event); @@ -2254,7 +2295,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, } if (pmu->selectable) { info.name = buf; - snprintf(buf, sizeof(buf), "%s//", pmu->name); + scnprintf(buf, sizeof(buf), "%s//", pmu->name); info.alias = NULL; info.scale_unit = NULL; info.desc = NULL; @@ -2619,8 +2660,12 @@ bool perf_pmu__wildcard_match(const struct perf_pmu *pmu, const char *wildcard_t pmu->name, pmu->alias_name, }; - bool need_fnmatch = strisglob(wildcard_to_match); + bool need_fnmatch; + + if (pmu->is_core && !strcmp(wildcard_to_match, "default_core")) + return true; + need_fnmatch = strisglob(wildcard_to_match); if (!strncmp(wildcard_to_match, "uncore_", 7)) wildcard_to_match += 7; diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 9a2023ceeefd..e0a4cb2428ca 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -150,6 +150,8 @@ struct perf_pmu *perf_pmus__find(const char *name) bool core_pmu; unsigned int to_read_pmus = 0; + if (!strcmp(name, "default_core")) + return perf_pmus__find_core_pmu(); /* * Once PMU is loaded it stays in the list, * so we keep us from multiple reading/parsing @@ -409,7 +411,7 @@ struct perf_pmu *perf_pmus__scan_matching_wildcard(struct perf_pmu *pmu, const c if (!pmu) { /* * Core PMUs, other sysfs PMUs and tool PMU can have any name or - * aren't wother optimizing for. + * aren't worth optimizing for. */ unsigned int to_read_pmus = PERF_TOOL_PMU_TYPE_PE_CORE_MASK | PERF_TOOL_PMU_TYPE_PE_OTHER_MASK | @@ -486,6 +488,22 @@ static struct perf_pmu *perf_pmus__scan_skip_duplicates(struct perf_pmu *pmu) return NULL; } +struct perf_pmu *perf_pmus__scan_for_uncore_id(struct perf_pmu *pmu, const char *compat) +{ + if (!pmu) { + /* Only uncore PMUs can have identifiers. */ + unsigned int to_read_pmus = PERF_TOOL_PMU_TYPE_PE_OTHER_MASK; + + pmu_read_sysfs(to_read_pmus); + pmu = list_prepare_entry(pmu, &other_pmus, list); + } + list_for_each_entry_continue(pmu, &other_pmus, list) { + if (pmu->id && pmu_uncore_identifier_match(compat, pmu->id)) + return pmu; + } + return NULL; +} + const struct perf_pmu *perf_pmus__pmu_for_pmu_filter(const char *str) { struct perf_pmu *pmu = NULL; diff --git a/tools/perf/util/pmus.h b/tools/perf/util/pmus.h index 7cb36863711a..0d55edb3f2fc 100644 --- a/tools/perf/util/pmus.h +++ b/tools/perf/util/pmus.h @@ -23,6 +23,7 @@ struct perf_pmu *perf_pmus__scan(struct perf_pmu *pmu); struct perf_pmu *perf_pmus__scan_core(struct perf_pmu *pmu); struct perf_pmu *perf_pmus__scan_for_event(struct perf_pmu *pmu, const char *event); struct perf_pmu *perf_pmus__scan_matching_wildcard(struct perf_pmu *pmu, const char *wildcard); +struct perf_pmu *perf_pmus__scan_for_uncore_id(struct perf_pmu *pmu, const char *compat); const struct perf_pmu *perf_pmus__pmu_for_pmu_filter(const char *str); diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index cb27e2898aa0..0242243681b6 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -174,7 +174,7 @@ bool is_event_supported(u8 type, u64 config) } evsel__close(evsel); - evsel__delete(evsel); + evsel__put(evsel); } perf_thread_map__put(tmap); diff --git a/tools/perf/util/print_insn.c b/tools/perf/util/print_insn.c index 02e6fbb8ca04..4068436f26ea 100644 --- a/tools/perf/util/print_insn.c +++ b/tools/perf/util/print_insn.c @@ -4,19 +4,24 @@ * * Author(s): Changbin Du */ +#include "print_insn.h" + #include -#include #include +#include + +#include + #include "capstone.h" #include "debug.h" +#include "dso.h" +#include "dump-insn.h" +#include "env.h" +#include "machine.h" +#include "map.h" #include "sample.h" #include "symbol.h" -#include "machine.h" #include "thread.h" -#include "print_insn.h" -#include "dump-insn.h" -#include "map.h" -#include "dso.h" size_t sample__fprintf_insn_raw(struct perf_sample *sample, FILE *fp) { @@ -33,13 +38,13 @@ size_t sample__fprintf_insn_raw(struct perf_sample *sample, FILE *fp) static bool is64bitip(struct machine *machine, struct addr_location *al) { const struct dso *dso = al->map ? map__dso(al->map) : NULL; + uint16_t e_machine; if (dso) return dso__is_64_bit(dso); - return machine__is(machine, "x86_64") || - machine__normalized_is(machine, "arm64") || - machine__normalized_is(machine, "s390"); + e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL); + return e_machine == EM_X86_64 || e_machine == EM_AARCH64 || e_machine == EM_S390; } ssize_t fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, diff --git a/tools/perf/util/print_insn.h b/tools/perf/util/print_insn.h index 07d11af3fc1c..cefa5c5f246e 100644 --- a/tools/perf/util/print_insn.h +++ b/tools/perf/util/print_insn.h @@ -5,10 +5,13 @@ #include #include -struct perf_sample; -struct thread; +#include + +struct addr_location; struct machine; struct perf_insn; +struct perf_sample; +struct thread; #define PRINT_INSN_IMM_HEX (1<<0) diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index 34b4badd2c14..11ae4a09412c 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -416,7 +416,7 @@ static int find_alternative_probe_point(struct debuginfo *dinfo, map__for_each_symbol_by_name(map, pp->function, sym, idx) { if (uprobes) { address = sym->start; - if (sym->type == STT_GNU_IFUNC) + if (symbol__type(sym) == STT_GNU_IFUNC) pr_warning("Warning: The probe function (%s) is a GNU indirect function.\n" "Consider identifying the final function used at run time and set the probe directly on that.\n", pp->function); @@ -3189,7 +3189,7 @@ static int find_probe_trace_events_from_map(struct perf_probe_event *pev, for (j = 0; j < num_matched_functions; j++) { sym = syms[j]; - if (sym->type != STT_FUNC) + if (symbol__type(sym) != STT_FUNC) continue; /* There can be duplicated symbols in the map */ diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c index 4032572cbf55..4d12693a83b3 100644 --- a/tools/perf/util/probe-file.c +++ b/tools/perf/util/probe-file.c @@ -197,6 +197,8 @@ struct strlist *probe_file__get_rawlist(int fd) idx = strlen(p) - 1; if (p[idx] == '\n') p[idx] = '\0'; + if (buf[0] == '#') + continue; ret = strlist__add(sl, buf); if (ret < 0) { pr_debug("strlist__add failed (%d)\n", ret); diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 64328abeef8b..f3f9a1573502 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -79,7 +79,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, unsigned int regn; Dwarf_Word offs = 0; bool ref = false; - const char *regs; + const char *regs, *name; int ret, ret2 = 0; if (dwarf_attr(vr_die, DW_AT_external, &attr) != NULL) @@ -93,7 +93,8 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (!tvar) return 0; - dwarf_formsdata(&attr, &snum); + if (dwarf_formsdata(&attr, &snum) != 0) + return -ENOENT; ret = asprintf(&tvar->value, "\\%ld", (long)snum); return ret < 0 ? -ENOMEM : 0; @@ -103,8 +104,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) return -EINVAL; /* Broken DIE ? */ if (dwarf_getlocation_addr(&attr, addr, &op, &nops, 1) <= 0) { - ret = dwarf_entrypc(sp_die, &tmp); - if (ret) + if (dwarf_entrypc(sp_die, &tmp) != 0) return -ENOENT; if (probe_conf.show_location_range && @@ -115,8 +115,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, return -ENOENT; } - ret = dwarf_highpc(sp_die, &tmp); - if (ret) + if (dwarf_highpc(sp_die, &tmp) != 0) return -ENOENT; /* * This is fuzzed by fentry mcount. We try to find the @@ -138,12 +137,16 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, static_var: if (!tvar) return ret2; + /* Static variables on memory (not stack), make @varname */ - ret = strlen(dwarf_diename(vr_die)); + name = dwarf_diename(vr_die); + if (!name) + return -ENOENT; + ret = strlen(name); tvar->value = zalloc(ret + 2); if (tvar->value == NULL) return -ENOMEM; - snprintf(tvar->value, ret + 2, "@%s", dwarf_diename(vr_die)); + snprintf(tvar->value, ret + 2, "@%s", name); tvar->ref = alloc_trace_arg_ref((long)offs); if (tvar->ref == NULL) return -ENOMEM; @@ -234,13 +237,14 @@ static int convert_variable_type(Dwarf_Die *vr_die, } if (die_get_real_type(vr_die, &type) == NULL) { - pr_warning("Failed to get a type information of %s.\n", - dwarf_diename(vr_die)); + const char *name = dwarf_diename(vr_die); + + pr_warning("Failed to get a type information of %s.\n", name ?: ""); return -ENOENT; } pr_debug("%s type is %s.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); if (cast && (!strcmp(cast, "string") || !strcmp(cast, "ustring"))) { /* String type */ @@ -249,7 +253,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, ret != DW_TAG_array_type) { pr_warning("Failed to cast into string: " "%s(%s) is not a pointer nor array.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); return -EINVAL; } if (die_get_real_type(&type, &type) == NULL) { @@ -272,7 +276,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, !die_compare_name(&type, "unsigned char")) { pr_warning("Failed to cast into string: " "%s is not (unsigned) char *.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); return -EINVAL; } tvar->type = strdup(cast); @@ -299,7 +303,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, /* Check the bitwidth */ if (ret > MAX_BASIC_TYPE_BITS) { pr_info("%s exceeds max-bitwidth. Cut down to %d bits.\n", - dwarf_diename(&type), MAX_BASIC_TYPE_BITS); + die_name(&type), MAX_BASIC_TYPE_BITS); ret = MAX_BASIC_TYPE_BITS; } ret = snprintf(buf, 16, "%c%d", prefix, ret); @@ -333,12 +337,14 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Var real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Var real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); tag = dwarf_tag(&type); if (field->name[0] == '[' && (tag == DW_TAG_array_type || tag == DW_TAG_pointer_type)) { + int bsize; + /* Save original type for next field or type */ memcpy(die_mem, &type, sizeof(*die_mem)); /* Get the type of this array */ @@ -346,7 +352,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Array real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Array real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); if (tag == DW_TAG_pointer_type) { ref = zalloc(sizeof(struct probe_trace_arg_ref)); @@ -357,7 +363,15 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, else *ref_ptr = ref; } - ref->offset += dwarf_bytesize(&type) * field->index; + bsize = dwarf_bytesize(&type); + + if (bsize < 0) + return -EINVAL; + if (!ref) { + pr_warning("Array indexing not supported for variables in registers.\n"); + return -ENOTSUP; + } + ref->offset += bsize * field->index; ref->user_access = user_access; goto next; } else if (tag == DW_TAG_pointer_type) { @@ -414,7 +428,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, if (die_find_member(&type, field->name, die_mem) == NULL) { pr_warning("%s(type:%s) has no member %s.\n", varname, - dwarf_diename(&type), field->name); + die_name(&type), field->name); return -EINVAL; } @@ -461,7 +475,7 @@ static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) int ret; pr_debug("Converting variable %s into trace event.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops, &pf->sp_die, pf, pf->tvar); @@ -542,7 +556,7 @@ static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod, /* Verify the address is correct */ if (!dwarf_haspc(sp_die, paddr)) { pr_warning("Specified offset is out of %s\n", - dwarf_diename(sp_die)); + die_name(sp_die)); return -EINVAL; } @@ -599,7 +613,7 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) if (!die_find_realfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { if (die_find_tailfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { pr_warning("Ignoring tail call from %s\n", - dwarf_diename(&pf->sp_die)); + die_name(&pf->sp_die)); return 0; } else { pr_warning("Failed to find probe point in any " @@ -611,10 +625,16 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) memcpy(&pf->sp_die, sc_die, sizeof(Dwarf_Die)); /* Get the frame base attribute/ops from subprogram */ - dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr); - ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); - if (ret <= 0 || nops == 0) { + if (dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr) == NULL) { pf->fb_ops = NULL; + } else { + ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); + if (ret <= 0 || nops == 0) + pf->fb_ops = NULL; + } + + if (pf->fb_ops == NULL) { + /* Not supported */ } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa && (pf->cfi_eh != NULL || pf->cfi_dbg != NULL)) { if ((dwarf_cfi_addrframe(pf->cfi_eh, pf->addr, &frame) != 0 && @@ -667,8 +687,8 @@ static int find_best_scope_cb(Dwarf_Die *fn_die, void *data) } } else { /* With the line number, find the nearest declared DIE */ - dwarf_decl_line(fn_die, &lno); - if (lno < fsp->line && fsp->diff > fsp->line - lno) { + if (dwarf_decl_line(fn_die, &lno) == 0 && lno < fsp->line && + fsp->diff > fsp->line - lno) { /* Keep a candidate and continue */ fsp->diff = fsp->line - lno; memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die)); @@ -924,12 +944,12 @@ static int probe_point_inline_cb(Dwarf_Die *in_die, void *data) /* Get probe address */ if (die_entrypc(in_die, &addr) != 0) { pr_warning("Failed to get entry address of %s.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } if (addr == 0) { pr_debug("%s has no valid entry address. skipped.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } pf->addr = addr; @@ -971,12 +991,13 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) if (pp->file && fname && strtailcmp(pp->file, fname)) return DWARF_CB_OK; - pr_debug("Matched function: %s [%lx]\n", dwarf_diename(sp_die), + pr_debug("Matched function: %s [%lx]\n", die_name(sp_die), (unsigned long)dwarf_dieoffset(sp_die)); pf->fname = fname; pf->abstrace_dieoffset = dwarf_dieoffset(sp_die); if (pp->line) { /* Function relative line */ - dwarf_decl_line(sp_die, &pf->lno); + if (dwarf_decl_line(sp_die, &pf->lno) != 0) + return DWARF_CB_OK; pf->lno += pp->line; param->retval = find_probe_point_by_line(pf); } else if (die_is_func_instance(sp_die)) { @@ -985,7 +1006,7 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) /* But in some case the entry address is 0 */ if (pf->addr == 0) { pr_debug("%s has no entry PC. Skipped\n", - dwarf_diename(sp_die)); + die_name(sp_die)); param->retval = 0; /* Real function */ } else if (pp->lazy_line) @@ -1018,7 +1039,8 @@ static int find_probe_point_by_func(struct probe_finder *pf) { struct dwarf_callback_param _param = {.data = (void *)pf, .retval = 0}; - dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0); + if (dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return _param.retval; } @@ -1207,7 +1229,8 @@ static int copy_variables_cb(Dwarf_Die *die_mem, void *data) * points to correct die. */ if (dwarf_attr(die_mem, DW_AT_abstract_origin, &attr)) { - dwarf_formref_die(&attr, &var_die); + if (dwarf_formref_die(&attr, &var_die) == NULL) + goto out; if (pf->abstrace_dieoffset != dwarf_dieoffset(&var_die)) goto out; } @@ -1293,13 +1316,16 @@ static int add_probe_trace_event(Dwarf_Die *sc_die, struct probe_finder *pf) if (ret < 0) goto end; - tev->point.realname = strdup(dwarf_diename(sc_die)); + tev->point.realname = strdup(die_name(sc_die)); if (!tev->point.realname) { ret = -ENOMEM; goto end; } - tev->lang = dwarf_srclang(dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL)); + if (dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL) != NULL) + tev->lang = dwarf_srclang(&pf->cu_die); + else + tev->lang = DW_LANG_C; /* Fallback */ pr_debug("Probe point found: %s+%lu\n", tev->point.symbol, tev->point.offset); @@ -1794,7 +1820,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) if (die_match_name(sp_die, lr->function) && die_is_func_def(sp_die)) { lf->fname = die_get_decl_file(sp_die); - dwarf_decl_line(sp_die, &lr->offset); + if (dwarf_decl_line(sp_die, &lr->offset) != 0) + lr->offset = 0; /* Fallback if no line info */ pr_debug("fname: %s, lineno:%d\n", lf->fname, lr->offset); lf->lno_s = lr->offset + lr->start; if (lf->lno_s < 0) /* Overflow */ @@ -1818,7 +1845,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) static int find_line_range_by_func(struct line_finder *lf) { struct dwarf_callback_param param = {.data = (void *)lf, .retval = 0}; - dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0); + if (dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return param.retval; } diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index cc1019d29a5d..af85c0771c44 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -1,33 +1,62 @@ // SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN #include -#include + #include -#include + #include +#include +#include + +#include #include -#ifdef HAVE_LIBTRACEEVENT -#include -#endif #include +#include + +#include "addr_location.h" +#include "build-id.h" #include "callchain.h" +#include "comm.h" +#include "config.h" #include "counts.h" +#include "data.h" +#include "debug.h" +#include "dso.h" +#include "dwarf-regs.h" +#include "event.h" +#include "branch.h" #include "evlist.h" #include "evsel.h" -#include "event.h" #include "expr.h" +#include "map.h" +#include "metricgroup.h" +#include "mmap.h" +#include "pmus.h" #include "print_binary.h" #include "record.h" +#include "sample.h" +#include "session.h" +#include "srccode.h" +#include "srcline.h" #include "strbuf.h" +#include "symbol.h" +#include "stat.h" +#include "header.h" +#include "trace/beauty/syscalltbl.h" +#include "thread.h" #include "thread_map.h" +#include "tool.h" #include "tp_pmu.h" #include "trace-event.h" -#include "metricgroup.h" -#include "mmap.h" -#include "util/sample.h" -#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif PyMODINIT_FUNC PyInit_perf(void); +static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel); + #define member_def(type, member, ptype, help) \ { #member, ptype, \ offsetof(struct pyrf_event, event) + offsetof(struct type, member), \ @@ -38,30 +67,83 @@ PyMODINIT_FUNC PyInit_perf(void); offsetof(struct pyrf_event, sample) + offsetof(struct perf_sample, member), \ 0, help } +#define CHECK_INITIALIZED(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return NULL; \ + } \ + } while (0) + +#define CHECK_INITIALIZED_INT(ptr, msg) \ + do { \ + if (!(ptr)) { \ + PyErr_SetString(PyExc_ValueError, msg " not initialized"); \ + return -1; \ + } \ + } while (0) + struct pyrf_event { PyObject_HEAD - struct evsel *evsel; + /** @sample: The parsed sample from the event. */ struct perf_sample sample; - union perf_event event; + /** @al: The address location from machine__resolve, lazily computed. */ + struct addr_location al; + /** @al_resolved: True when machine__resolve been called. */ + bool al_resolved; + /** @callchain: Resolved callchain, eagerly computed if requested. */ + PyObject *callchain; + /** @brstack: Resolved branch stack, eagerly computed if requested. */ + PyObject *brstack; + /** @event: The underlying perf_event that may be in a file or ring buffer. */ + union perf_event event; }; #define sample_members \ - sample_member_def(sample_ip, ip, T_ULONGLONG, "event ip"), \ sample_member_def(sample_pid, pid, T_INT, "event pid"), \ sample_member_def(sample_tid, tid, T_INT, "event tid"), \ sample_member_def(sample_time, time, T_ULONGLONG, "event timestamp"), \ - sample_member_def(sample_addr, addr, T_ULONGLONG, "event addr"), \ sample_member_def(sample_id, id, T_ULONGLONG, "event id"), \ sample_member_def(sample_stream_id, stream_id, T_ULONGLONG, "event stream id"), \ sample_member_def(sample_period, period, T_ULONGLONG, "event period"), \ sample_member_def(sample_cpu, cpu, T_UINT, "event cpu"), +static PyObject *pyrf_event__get_evsel(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (!pevent->sample.evsel) + Py_RETURN_NONE; + + return pyrf_evsel__from_evsel(pevent->sample.evsel); +} + +static PyGetSetDef pyrf_event__getset[] = { + { + .name = "evsel", + .get = pyrf_event__get_evsel, + .set = NULL, + .doc = "tracking event.", + }, + { .name = NULL, }, +}; + +static void pyrf_event__delete(struct pyrf_event *pevent) +{ + if (pevent->al_resolved) + addr_location__exit(&pevent->al); + Py_XDECREF(pevent->callchain); + Py_XDECREF(pevent->brstack); + perf_sample__exit(&pevent->sample); + Py_TYPE(pevent)->tp_free((PyObject *)pevent); +} + static const char pyrf_mmap_event__doc[] = PyDoc_STR("perf mmap event object."); static PyMemberDef pyrf_mmap_event__members[] = { sample_members member_def(perf_event_header, type, T_UINT, "event type"), - member_def(perf_event_header, misc, T_UINT, "event misc"), + member_def(perf_event_header, misc, T_USHORT, "event misc"), member_def(perf_record_mmap, pid, T_UINT, "event pid"), member_def(perf_record_mmap, tid, T_UINT, "event tid"), member_def(perf_record_mmap, start, T_ULONGLONG, "start of the map"), @@ -94,12 +176,155 @@ static PyTypeObject pyrf_mmap_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.mmap_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_mmap_event__doc, .tp_members = pyrf_mmap_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_mmap_event__repr, }; +static const char pyrf_mmap2_event__doc[] = PyDoc_STR("perf mmap2 event object."); + +static PyObject *pyrf_mmap2_event__get_maj(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLong(pevent->event.mmap2.maj); +} + +static PyObject *pyrf_mmap2_event__get_min(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLong(pevent->event.mmap2.min); +} + +static PyObject *pyrf_mmap2_event__get_ino(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLongLong(pevent->event.mmap2.ino); +} + +static PyObject *pyrf_mmap2_event__get_ino_generation(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLongLong(pevent->event.mmap2.ino_generation); +} + +static PyObject *pyrf_mmap2_event__get_build_id(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (!(pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)) + Py_RETURN_NONE; + + int size = pevent->event.mmap2.build_id_size; + + if (size > 20) + size = 20; + + return PyBytes_FromStringAndSize((const char *)pevent->event.mmap2.build_id, size); +} + +static PyGetSetDef pyrf_mmap2_event__getset[] = { + { + .name = "evsel", + .get = pyrf_event__get_evsel, + .set = NULL, + .doc = "tracking event.", + }, + { + .name = "maj", + .get = pyrf_mmap2_event__get_maj, + .set = NULL, + .doc = "major number.", + }, + { + .name = "min", + .get = pyrf_mmap2_event__get_min, + .set = NULL, + .doc = "minor number.", + }, + { + .name = "ino", + .get = pyrf_mmap2_event__get_ino, + .set = NULL, + .doc = "inode number.", + }, + { + .name = "ino_generation", + .get = pyrf_mmap2_event__get_ino_generation, + .set = NULL, + .doc = "inode generation.", + }, + { + .name = "build_id", + .get = pyrf_mmap2_event__get_build_id, + .set = NULL, + .doc = "binary build ID.", + }, + { .name = NULL, }, +}; + +static PyMemberDef pyrf_mmap2_event__members[] = { + sample_members + member_def(perf_event_header, type, T_UINT, "event type"), + member_def(perf_event_header, misc, T_USHORT, "event misc"), + member_def(perf_record_mmap2, pid, T_UINT, "event pid"), + member_def(perf_record_mmap2, tid, T_UINT, "event tid"), + member_def(perf_record_mmap2, start, T_ULONGLONG, "start of the map"), + member_def(perf_record_mmap2, len, T_ULONGLONG, "map length"), + member_def(perf_record_mmap2, pgoff, T_ULONGLONG, "page offset"), + member_def(perf_record_mmap2, prot, T_UINT, "protection"), + member_def(perf_record_mmap2, flags, T_UINT, "flags"), + member_def(perf_record_mmap2, filename, T_STRING_INPLACE, "backing store"), + { .name = NULL, }, +}; + +static PyObject *pyrf_mmap2_event__repr(const struct pyrf_event *pevent) +{ + PyObject *ret; + char *s; + + if (asprintf(&s, "{ type: mmap2, pid: %u, tid: %u, start: %#" PRI_lx64 ", length: %#" PRI_lx64 ", offset: %#" PRI_lx64 ", flags: %#x, prot: %#x, filename: %s }", + pevent->event.mmap2.pid, pevent->event.mmap2.tid, + pevent->event.mmap2.start, pevent->event.mmap2.len, + pevent->event.mmap2.pgoff, pevent->event.mmap2.flags, + pevent->event.mmap2.prot, pevent->event.mmap2.filename) < 0) + return PyErr_NoMemory(); + + ret = PyUnicode_FromString(s); + free(s); + return ret; +} + +static PyTypeObject pyrf_mmap2_event__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.mmap2_event", + .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_mmap2_event__doc, + .tp_members = pyrf_mmap2_event__members, + .tp_getset = pyrf_mmap2_event__getset, + .tp_repr = (reprfunc)pyrf_mmap2_event__repr, +}; + static const char pyrf_task_event__doc[] = PyDoc_STR("perf task (fork/exit) event object."); static PyMemberDef pyrf_task_event__members[] = { @@ -129,9 +354,11 @@ static PyTypeObject pyrf_task_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.task_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_task_event__doc, .tp_members = pyrf_task_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_task_event__repr, }; @@ -158,9 +385,11 @@ static PyTypeObject pyrf_comm_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.comm_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_comm_event__doc, .tp_members = pyrf_comm_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_comm_event__repr, }; @@ -190,9 +419,11 @@ static PyTypeObject pyrf_throttle_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.throttle_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_throttle_event__doc, .tp_members = pyrf_throttle_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_throttle_event__repr, }; @@ -200,6 +431,7 @@ static const char pyrf_lost_event__doc[] = PyDoc_STR("perf lost event object."); static PyMemberDef pyrf_lost_event__members[] = { sample_members + member_def(perf_event_header, type, T_UINT, "event type"), member_def(perf_record_lost, id, T_ULONGLONG, "event id"), member_def(perf_record_lost, lost, T_ULONGLONG, "number of lost events"), { .name = NULL, }, @@ -225,16 +457,90 @@ static PyTypeObject pyrf_lost_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.lost_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_lost_event__doc, .tp_members = pyrf_lost_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_lost_event__repr, }; +static const char pyrf_stat_event__doc[] = PyDoc_STR("perf stat event object."); + +static PyMemberDef pyrf_stat_event__members[] = { + sample_members + member_def(perf_event_header, type, T_UINT, "event type"), + member_def(perf_record_stat, id, T_ULONGLONG, "event id"), + member_def(perf_record_stat, cpu, T_UINT, "event cpu"), + member_def(perf_record_stat, thread, T_UINT, "event thread"), + member_def(perf_record_stat, val, T_ULONGLONG, "counter value"), + member_def(perf_record_stat, ena, T_ULONGLONG, "enabled time"), + member_def(perf_record_stat, run, T_ULONGLONG, "running time"), + { .name = NULL, }, +}; + +static PyObject *pyrf_stat_event__repr(const struct pyrf_event *pevent) +{ + return PyUnicode_FromFormat( + "{ type: stat, id: %llu, cpu: %u, thread: %u, val: %llu, ena: %llu, run: %llu }", + pevent->event.stat.id, + pevent->event.stat.cpu, + pevent->event.stat.thread, + pevent->event.stat.val, + pevent->event.stat.ena, + pevent->event.stat.run); +} + +static PyTypeObject pyrf_stat_event__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.stat_event", + .tp_basicsize = sizeof(struct pyrf_event), + .tp_new = PyType_GenericNew, + .tp_dealloc = (destructor)pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_stat_event__doc, + .tp_members = pyrf_stat_event__members, + .tp_getset = pyrf_event__getset, + .tp_repr = (reprfunc)pyrf_stat_event__repr, +}; + +static const char pyrf_stat_round_event__doc[] = PyDoc_STR("perf stat round event object."); + +static PyMemberDef pyrf_stat_round_event__members[] = { + sample_members + member_def(perf_event_header, type, T_UINT, "event type"), + { .name = "stat_round_type", .type = T_ULONGLONG, + .offset = offsetof(struct pyrf_event, event) + offsetof(struct perf_record_stat_round, type), + .doc = "round type" }, + member_def(perf_record_stat_round, time, T_ULONGLONG, "round time"), + { .name = NULL, }, +}; + +static PyObject *pyrf_stat_round_event__repr(const struct pyrf_event *pevent) +{ + return PyUnicode_FromFormat("{ type: stat_round, type: %llu, time: %llu }", + pevent->event.stat_round.type, + pevent->event.stat_round.time); +} + +static PyTypeObject pyrf_stat_round_event__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.stat_round_event", + .tp_basicsize = sizeof(struct pyrf_event), + .tp_new = PyType_GenericNew, + .tp_dealloc = (destructor)pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_stat_round_event__doc, + .tp_members = pyrf_stat_round_event__members, + .tp_getset = pyrf_event__getset, + .tp_repr = (reprfunc)pyrf_stat_round_event__repr, +}; + static const char pyrf_read_event__doc[] = PyDoc_STR("perf read event object."); static PyMemberDef pyrf_read_event__members[] = { sample_members + member_def(perf_event_header, type, T_UINT, "event type"), member_def(perf_record_read, pid, T_UINT, "event pid"), member_def(perf_record_read, tid, T_UINT, "event tid"), { .name = NULL, }, @@ -255,9 +561,11 @@ static PyTypeObject pyrf_read_event__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.read_event", .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_read_event__doc, .tp_members = pyrf_read_event__members, + .tp_getset = pyrf_event__getset, .tp_repr = (reprfunc)pyrf_read_event__repr, }; @@ -265,16 +573,17 @@ static const char pyrf_sample_event__doc[] = PyDoc_STR("perf sample event object static PyMemberDef pyrf_sample_event__members[] = { sample_members + sample_member_def(sample_ip, ip, T_ULONGLONG, "event ip"), + sample_member_def(sample_addr, addr, T_ULONGLONG, "event addr"), + sample_member_def(sample_phys_addr, phys_addr, T_ULONGLONG, "event physical addr"), + sample_member_def(sample_weight, weight, T_ULONGLONG, "event weight"), + sample_member_def(sample_data_src, data_src, T_ULONGLONG, "event data source"), + sample_member_def(sample_insn_count, insn_cnt, T_ULONGLONG, "event instruction count"), + sample_member_def(sample_cyc_count, cyc_cnt, T_ULONGLONG, "event cycle count"), member_def(perf_event_header, type, T_UINT, "event type"), { .name = NULL, }, }; -static void pyrf_sample_event__delete(struct pyrf_event *pevent) -{ - perf_sample__exit(&pevent->sample); - Py_TYPE(pevent)->tp_free((PyObject*)pevent); -} - static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent) { PyObject *ret; @@ -292,7 +601,9 @@ static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent) #ifdef HAVE_LIBTRACEEVENT static bool is_tracepoint(const struct pyrf_event *pevent) { - return pevent->evsel->core.attr.type == PERF_TYPE_TRACEPOINT; + if (!pevent->sample.evsel) + return false; + return pevent->sample.evsel->core.attr.type == PERF_TYPE_TRACEPOINT; } static PyObject* @@ -339,7 +650,7 @@ tracepoint_field(const struct pyrf_event *pe, struct tep_format_field *field) static PyObject* get_tracepoint_field(struct pyrf_event *pevent, PyObject *attr_name) { - struct evsel *evsel = pevent->evsel; + struct evsel *evsel = pevent->sample.evsel; struct tep_event *tp_format = evsel__tp_format(evsel); struct tep_format_field *field; @@ -362,163 +673,861 @@ get_tracepoint_field(struct pyrf_event *pevent, PyObject *attr_name) } #endif /* HAVE_LIBTRACEEVENT */ -static PyObject* -pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name) +static int pyrf_sample_event__resolve_al(struct pyrf_event *pevent) { - PyObject *obj = NULL; - -#ifdef HAVE_LIBTRACEEVENT - if (is_tracepoint(pevent)) - obj = get_tracepoint_field(pevent, attr_name); -#endif + struct evsel *evsel = pevent->sample.evsel; + struct evlist *evlist = evsel ? evsel->evlist : NULL; + struct perf_session *session = evlist ? evlist__session(evlist) : NULL; - return obj ?: PyObject_GenericGetAttr((PyObject *) pevent, attr_name); -} + if (pevent->al_resolved) + return 0; -static PyTypeObject pyrf_sample_event__type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "perf.sample_event", - .tp_basicsize = sizeof(struct pyrf_event), - .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - .tp_doc = pyrf_sample_event__doc, - .tp_members = pyrf_sample_event__members, - .tp_repr = (reprfunc)pyrf_sample_event__repr, - .tp_getattro = (getattrofunc) pyrf_sample_event__getattro, -}; + if (!session) + return -1; -static const char pyrf_context_switch_event__doc[] = PyDoc_STR("perf context_switch event object."); + addr_location__init(&pevent->al); + if (machine__resolve(&session->machines.host, &pevent->al, &pevent->sample) < 0) { + addr_location__exit(&pevent->al); + return -1; + } -static PyMemberDef pyrf_context_switch_event__members[] = { - sample_members - member_def(perf_event_header, type, T_UINT, "event type"), - member_def(perf_record_switch, next_prev_pid, T_UINT, "next/prev pid"), - member_def(perf_record_switch, next_prev_tid, T_UINT, "next/prev tid"), - { .name = NULL, }, -}; + pevent->al_resolved = true; + return 0; +} -static PyObject *pyrf_context_switch_event__repr(const struct pyrf_event *pevent) +static PyObject *pyrf_sample_event__get_dso(struct pyrf_event *pevent, + void *closure __maybe_unused) { - PyObject *ret; - char *s; + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; - if (asprintf(&s, "{ type: context_switch, next_prev_pid: %u, next_prev_tid: %u, switch_out: %u }", - pevent->event.context_switch.next_prev_pid, - pevent->event.context_switch.next_prev_tid, - !!(pevent->event.header.misc & PERF_RECORD_MISC_SWITCH_OUT)) < 0) { - ret = PyErr_NoMemory(); - } else { - ret = PyUnicode_FromString(s); - free(s); - } - return ret; + return PyUnicode_FromString(dso__name(map__dso(pevent->al.map))); } -static PyTypeObject pyrf_context_switch_event__type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "perf.context_switch_event", - .tp_basicsize = sizeof(struct pyrf_event), - .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, - .tp_doc = pyrf_context_switch_event__doc, - .tp_members = pyrf_context_switch_event__members, - .tp_repr = (reprfunc)pyrf_context_switch_event__repr, -}; +static PyObject *pyrf_sample_event__get_dso_long_name(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; -static int pyrf_event__setup_types(void) + return PyUnicode_FromString(dso__long_name(map__dso(pevent->al.map))); +} + +static PyObject *pyrf_sample_event__get_dso_bid(struct pyrf_event *pevent, + void *closure __maybe_unused) { - int err; - pyrf_mmap_event__type.tp_new = - pyrf_task_event__type.tp_new = - pyrf_comm_event__type.tp_new = - pyrf_lost_event__type.tp_new = - pyrf_read_event__type.tp_new = - pyrf_sample_event__type.tp_new = - pyrf_context_switch_event__type.tp_new = - pyrf_throttle_event__type.tp_new = PyType_GenericNew; + char sbuild_id[SBUILD_ID_SIZE]; - pyrf_sample_event__type.tp_dealloc = (destructor)pyrf_sample_event__delete, + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; - err = PyType_Ready(&pyrf_mmap_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_lost_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_task_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_comm_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_throttle_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_read_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_sample_event__type); - if (err < 0) - goto out; - err = PyType_Ready(&pyrf_context_switch_event__type); - if (err < 0) - goto out; -out: - return err; + build_id__snprintf(dso__bid(map__dso(pevent->al.map)), sbuild_id, sizeof(sbuild_id)); + return PyUnicode_FromString(sbuild_id); } -static PyTypeObject *pyrf_event__type[] = { - [PERF_RECORD_MMAP] = &pyrf_mmap_event__type, - [PERF_RECORD_LOST] = &pyrf_lost_event__type, - [PERF_RECORD_COMM] = &pyrf_comm_event__type, - [PERF_RECORD_EXIT] = &pyrf_task_event__type, - [PERF_RECORD_THROTTLE] = &pyrf_throttle_event__type, - [PERF_RECORD_UNTHROTTLE] = &pyrf_throttle_event__type, - [PERF_RECORD_FORK] = &pyrf_task_event__type, - [PERF_RECORD_READ] = &pyrf_read_event__type, - [PERF_RECORD_SAMPLE] = &pyrf_sample_event__type, - [PERF_RECORD_SWITCH] = &pyrf_context_switch_event__type, - [PERF_RECORD_SWITCH_CPU_WIDE] = &pyrf_context_switch_event__type, -}; - -static PyObject *pyrf_event__new(const union perf_event *event) +static PyObject *pyrf_sample_event__get_map_start(struct pyrf_event *pevent, + void *closure __maybe_unused) { - struct pyrf_event *pevent; - PyTypeObject *ptype; + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; - if ((event->header.type < PERF_RECORD_MMAP || - event->header.type > PERF_RECORD_SAMPLE) && - !(event->header.type == PERF_RECORD_SWITCH || - event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)) { - PyErr_Format(PyExc_TypeError, "Unexpected header type %u", - event->header.type); - return NULL; - } + return PyLong_FromUnsignedLong(map__start(pevent->al.map)); +} - // FIXME this better be dynamic or we need to parse everything - // before calling perf_mmap__consume(), including tracepoint fields. - if (sizeof(pevent->event) < event->header.size) { - PyErr_Format(PyExc_TypeError, "Unexpected event size: %zd < %u", - sizeof(pevent->event), event->header.size); - return NULL; - } +static PyObject *pyrf_sample_event__get_map_end(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; - ptype = pyrf_event__type[event->header.type]; - pevent = PyObject_New(struct pyrf_event, ptype); - if (pevent != NULL) - memcpy(&pevent->event, event, event->header.size); - return (PyObject *)pevent; + return PyLong_FromUnsignedLong(map__end(pevent->al.map)); } -struct pyrf_cpu_map { - PyObject_HEAD +static PyObject *pyrf_sample_event__get_map_pgoff(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map) + Py_RETURN_NONE; - struct perf_cpu_map *cpus; -}; + return PyLong_FromUnsignedLongLong(map__pgoff(pevent->al.map)); +} -static int pyrf_cpu_map__init(struct pyrf_cpu_map *pcpus, - PyObject *args, PyObject *kwargs) +static PyObject *pyrf_sample_event__get_symbol(struct pyrf_event *pevent, + void *closure __maybe_unused) { - static char *kwlist[] = { "cpustr", NULL }; - char *cpustr = NULL; + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym) + Py_RETURN_NONE; + + return PyUnicode_FromString(pevent->al.sym->name); +} + +static PyObject *pyrf_sample_event__get_sym_start(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLongLong(pevent->al.sym->start); +} + +static PyObject *pyrf_sample_event__get_sym_end(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym) + Py_RETURN_NONE; + + return PyLong_FromUnsignedLongLong(pevent->al.sym->end); +} + +static PyObject *pyrf_sample_event__get_raw_buf(struct pyrf_event *pevent, + void *closure __maybe_unused) +{ + if (pevent->event.header.type != PERF_RECORD_SAMPLE) + Py_RETURN_NONE; + + return PyBytes_FromStringAndSize((const char *)pevent->sample.raw_data, + pevent->sample.raw_size); +} + +static PyObject *pyrf_sample_event__srccode(PyObject *self, PyObject *args) +{ + struct pyrf_event *pevent = (void *)self; + u64 addr = pevent->sample.ip; + char *srcfile = NULL; + char *srccode = NULL; + unsigned int line = 0; + int len = 0; + PyObject *result; + struct addr_location al; + + if (!PyArg_ParseTuple(args, "|K", &addr)) + return NULL; + + if (pyrf_sample_event__resolve_al(pevent) < 0) + Py_RETURN_NONE; + + if (addr != pevent->sample.ip) { + addr_location__init(&al); + thread__find_symbol_fb(pevent->al.thread, pevent->sample.cpumode, addr, &al); + } else { + addr_location__init(&al); + al.thread = thread__get(pevent->al.thread); + al.map = map__get(pevent->al.map); + al.sym = pevent->al.sym; + al.addr = pevent->al.addr; + } + + if (al.map) { + struct dso *dso = map__dso(al.map); + + if (dso) { + srcfile = get_srcline_split(dso, map__rip_2objdump(al.map, addr), + &line); + } + } + addr_location__exit(&al); + + if (srcfile) { + srccode = find_sourceline(srcfile, line, &len); + result = Py_BuildValue("(sIs#)", srcfile, line, srccode, (Py_ssize_t)len); + free(srcfile); + } else { + result = Py_BuildValue("(sIs#)", NULL, 0, NULL, (Py_ssize_t)0); + } + + return result; +} + +static PyObject *pyrf_sample_event__insn(PyObject *self, PyObject *args __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + struct thread *thread; + struct machine *machine; + + if (pyrf_sample_event__resolve_al(pevent) < 0) + Py_RETURN_NONE; + + thread = pevent->al.thread; + + if (!thread || !thread__maps(thread)) + Py_RETURN_NONE; + + machine = maps__machine(thread__maps(thread)); + if (!machine) + Py_RETURN_NONE; + + if (pevent->sample.ip && !pevent->sample.insn_len) + perf_sample__fetch_insn(&pevent->sample, thread, machine); + + if (!pevent->sample.insn_len) + Py_RETURN_NONE; + + return PyBytes_FromStringAndSize((const char *)pevent->sample.insn, + pevent->sample.insn_len); +} + +struct pyrf_callchain_node { + PyObject_HEAD + u64 ip; + struct map *map; + struct symbol *sym; +}; + +static void pyrf_callchain_node__delete(struct pyrf_callchain_node *pnode) +{ + map__put(pnode->map); + Py_TYPE(pnode)->tp_free((PyObject *)pnode); +} + +static PyObject *pyrf_callchain_node__get_ip(struct pyrf_callchain_node *pnode, + void *closure __maybe_unused) +{ + return PyLong_FromUnsignedLongLong(pnode->ip); +} + +static PyObject *pyrf_callchain_node__get_symbol(struct pyrf_callchain_node *pnode, + void *closure __maybe_unused) +{ + if (pnode->sym) + return PyUnicode_FromString(pnode->sym->name); + return PyUnicode_FromString("[unknown]"); +} + +static PyObject *pyrf_callchain_node__get_dso(struct pyrf_callchain_node *pnode, + void *closure __maybe_unused) +{ + const char *dsoname = "[unknown]"; + + if (pnode->map) { + struct dso *dso = map__dso(pnode->map); + + if (dso) { + if (symbol_conf.show_kernel_path && dso__long_name(dso)) + dsoname = dso__long_name(dso); + else + dsoname = dso__name(dso); + } + } + return PyUnicode_FromString(dsoname); +} + +static PyGetSetDef pyrf_callchain_node__getset[] = { + { .name = "ip", .get = (getter)pyrf_callchain_node__get_ip, }, + { .name = "symbol", .get = (getter)pyrf_callchain_node__get_symbol, }, + { .name = "dso", .get = (getter)pyrf_callchain_node__get_dso, }, + { .name = NULL, }, +}; + +static PyTypeObject pyrf_callchain_node__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.callchain_node", + .tp_basicsize = sizeof(struct pyrf_callchain_node), + .tp_dealloc = (destructor)pyrf_callchain_node__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = "perf callchain node object.", + .tp_getset = pyrf_callchain_node__getset, +}; + +struct pyrf_callchain_frame { + u64 ip; + struct map *map; + struct symbol *sym; +}; + +struct pyrf_callchain { + PyObject_HEAD + struct pyrf_callchain_frame *frames; + u64 nr_frames; +}; + +static void pyrf_callchain__delete(struct pyrf_callchain *pchain) +{ + if (pchain->frames) { + for (u64 i = 0; i < pchain->nr_frames; i++) + map__put(pchain->frames[i].map); + free(pchain->frames); + } + Py_TYPE(pchain)->tp_free((PyObject *)pchain); +} + +static Py_ssize_t pyrf_callchain__length(PyObject *obj) +{ + struct pyrf_callchain *pchain = (void *)obj; + + return pchain->nr_frames; +} + +static PyObject *pyrf_callchain__item(PyObject *obj, Py_ssize_t i) +{ + struct pyrf_callchain *pchain = (void *)obj; + struct pyrf_callchain_node *pnode; + + if (i < 0 || i >= (Py_ssize_t)pchain->nr_frames) { + PyErr_SetString(PyExc_IndexError, "Index out of range"); + return NULL; + } + + pnode = PyObject_New(struct pyrf_callchain_node, &pyrf_callchain_node__type); + if (!pnode) + return NULL; + + pnode->ip = pchain->frames[i].ip; + pnode->map = map__get(pchain->frames[i].map); + pnode->sym = pchain->frames[i].sym; + + return (PyObject *)pnode; +} + +static PySequenceMethods pyrf_callchain__sequence_methods = { + .sq_length = pyrf_callchain__length, + .sq_item = pyrf_callchain__item, +}; + +static PyTypeObject pyrf_callchain__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.callchain", + .tp_basicsize = sizeof(struct pyrf_callchain), + .tp_dealloc = (destructor)pyrf_callchain__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = "perf callchain object.", + .tp_as_sequence = &pyrf_callchain__sequence_methods, +}; + +static PyObject *pyrf_sample_event__get_callchain(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (!pevent->callchain) + Py_RETURN_NONE; + + Py_INCREF(pevent->callchain); + return pevent->callchain; +} + +struct pyrf_branch_entry { + PyObject_HEAD + u64 from; + u64 to; + struct branch_flags flags; +}; + +static void pyrf_branch_entry__delete(struct pyrf_branch_entry *pentry) +{ + Py_TYPE(pentry)->tp_free((PyObject *)pentry); +} + +static PyObject *pyrf_branch_entry__get_from(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyLong_FromUnsignedLongLong(pentry->from); +} + +static PyObject *pyrf_branch_entry__get_to(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyLong_FromUnsignedLongLong(pentry->to); +} + +static PyObject *pyrf_branch_entry__get_mispred(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyBool_FromLong(pentry->flags.mispred); +} + +static PyObject *pyrf_branch_entry__get_predicted(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyBool_FromLong(pentry->flags.predicted); +} + +static PyObject *pyrf_branch_entry__get_in_tx(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyBool_FromLong(pentry->flags.in_tx); +} + +static PyObject *pyrf_branch_entry__get_abort(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyBool_FromLong(pentry->flags.abort); +} + +static PyObject *pyrf_branch_entry__get_cycles(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyLong_FromUnsignedLongLong(pentry->flags.cycles); +} + +static PyObject *pyrf_branch_entry__get_type(struct pyrf_branch_entry *pentry, + void *closure __maybe_unused) +{ + return PyLong_FromUnsignedLongLong((unsigned long long)pentry->flags.type); +} + +static PyGetSetDef pyrf_branch_entry__getset[] = { + { .name = "from_ip", .get = (getter)pyrf_branch_entry__get_from, }, + { .name = "to_ip", .get = (getter)pyrf_branch_entry__get_to, }, + { .name = "mispred", .get = (getter)pyrf_branch_entry__get_mispred, }, + { .name = "predicted", .get = (getter)pyrf_branch_entry__get_predicted, }, + { .name = "in_tx", .get = (getter)pyrf_branch_entry__get_in_tx, }, + { .name = "abort", .get = (getter)pyrf_branch_entry__get_abort, }, + { .name = "cycles", .get = (getter)pyrf_branch_entry__get_cycles, }, + { .name = "type", .get = (getter)pyrf_branch_entry__get_type, }, + { .name = NULL, }, +}; + +static PyTypeObject pyrf_branch_entry__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.branch_entry", + .tp_basicsize = sizeof(struct pyrf_branch_entry), + .tp_dealloc = (destructor)pyrf_branch_entry__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = "perf branch entry object.", + .tp_getset = pyrf_branch_entry__getset, +}; + +struct pyrf_branch_stack { + PyObject_HEAD + struct branch_entry *entries; + u64 nr; +}; + +static void pyrf_branch_stack__delete(struct pyrf_branch_stack *pstack) +{ + free(pstack->entries); + Py_TYPE(pstack)->tp_free((PyObject *)pstack); +} + +static Py_ssize_t pyrf_branch_stack__length(PyObject *obj) +{ + struct pyrf_branch_stack *pstack = (void *)obj; + + return pstack->nr; +} + +static PyObject *pyrf_branch_stack__item(PyObject *obj, Py_ssize_t i) +{ + struct pyrf_branch_stack *pstack = (void *)obj; + struct pyrf_branch_entry *pentry; + + if (i < 0 || i >= (Py_ssize_t)pstack->nr) { + PyErr_SetString(PyExc_IndexError, "Index out of range"); + return NULL; + } + + pentry = PyObject_New(struct pyrf_branch_entry, &pyrf_branch_entry__type); + if (!pentry) + return NULL; + + pentry->from = pstack->entries[i].from; + pentry->to = pstack->entries[i].to; + pentry->flags = pstack->entries[i].flags; + + return (PyObject *)pentry; +} + +static PySequenceMethods pyrf_branch_stack__sequence_methods = { + .sq_length = pyrf_branch_stack__length, + .sq_item = pyrf_branch_stack__item, +}; + +static PyTypeObject pyrf_branch_stack__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.branch_stack", + .tp_basicsize = sizeof(struct pyrf_branch_stack), + .tp_dealloc = (destructor)pyrf_branch_stack__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = "perf branch stack object.", + .tp_as_sequence = &pyrf_branch_stack__sequence_methods, +}; + +static PyObject *pyrf_sample_event__get_brstack(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_event *pevent = (void *)self; + + if (!pevent->brstack) + Py_RETURN_NONE; + + Py_INCREF(pevent->brstack); + return pevent->brstack; +} + +static PyObject* +pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name) +{ + PyObject *obj = NULL; + +#ifdef HAVE_LIBTRACEEVENT + if (is_tracepoint(pevent)) + obj = get_tracepoint_field(pevent, attr_name); +#endif + + return obj ?: PyObject_GenericGetAttr((PyObject *) pevent, attr_name); +} + +static PyGetSetDef pyrf_sample_event__getset[] = { + { + .name = "callchain", + .get = pyrf_sample_event__get_callchain, + .set = NULL, + .doc = "event callchain.", + }, + { + .name = "brstack", + .get = pyrf_sample_event__get_brstack, + .set = NULL, + .doc = "event branch stack.", + }, + { + .name = "raw_buf", + .get = (getter)pyrf_sample_event__get_raw_buf, + .set = NULL, + .doc = "event raw buffer.", + }, + { + .name = "evsel", + .get = pyrf_event__get_evsel, + .set = NULL, + .doc = "tracking event.", + }, + { + .name = "dso", + .get = (getter)pyrf_sample_event__get_dso, + .set = NULL, + .doc = "event dso short name.", + }, + { + .name = "dso_long_name", + .get = (getter)pyrf_sample_event__get_dso_long_name, + .set = NULL, + .doc = "event dso long name.", + }, + { + .name = "dso_bid", + .get = (getter)pyrf_sample_event__get_dso_bid, + .set = NULL, + .doc = "event dso build id.", + }, + { + .name = "map_start", + .get = (getter)pyrf_sample_event__get_map_start, + .set = NULL, + .doc = "event map start address.", + }, + { + .name = "map_end", + .get = (getter)pyrf_sample_event__get_map_end, + .set = NULL, + .doc = "event map end address.", + }, + { + .name = "map_pgoff", + .get = (getter)pyrf_sample_event__get_map_pgoff, + .set = NULL, + .doc = "event map page offset.", + }, + { + .name = "symbol", + .get = (getter)pyrf_sample_event__get_symbol, + .set = NULL, + .doc = "event symbol name.", + }, + { + .name = "sym_start", + .get = (getter)pyrf_sample_event__get_sym_start, + .set = NULL, + .doc = "event symbol start address.", + }, + { + .name = "sym_end", + .get = (getter)pyrf_sample_event__get_sym_end, + .set = NULL, + .doc = "event symbol end address.", + }, + { .name = NULL, }, +}; + +static PyMethodDef pyrf_sample_event__methods[] = { + { + .ml_name = "srccode", + .ml_meth = (PyCFunction)pyrf_sample_event__srccode, + .ml_flags = METH_VARARGS, + .ml_doc = PyDoc_STR("Get source code for an address.") + }, + { + .ml_name = "insn", + .ml_meth = (PyCFunction)pyrf_sample_event__insn, + .ml_flags = METH_NOARGS, + .ml_doc = PyDoc_STR("Get instruction bytes for a sample.") + }, + { .ml_name = NULL, } +}; + +static PyTypeObject pyrf_sample_event__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.sample_event", + .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_sample_event__doc, + .tp_members = pyrf_sample_event__members, + .tp_getset = pyrf_sample_event__getset, + .tp_methods = pyrf_sample_event__methods, + .tp_repr = (reprfunc)pyrf_sample_event__repr, + .tp_getattro = (getattrofunc) pyrf_sample_event__getattro, +}; + +static const char pyrf_context_switch_event__doc[] = PyDoc_STR("perf context_switch event object."); + +static PyMemberDef pyrf_context_switch_event__members[] = { + sample_members + member_def(perf_event_header, type, T_UINT, "event type"), + member_def(perf_record_switch, next_prev_pid, T_UINT, "next/prev pid"), + member_def(perf_record_switch, next_prev_tid, T_UINT, "next/prev tid"), + { .name = NULL, }, +}; + +static PyObject *pyrf_context_switch_event__repr(const struct pyrf_event *pevent) +{ + PyObject *ret; + char *s; + + if (asprintf(&s, "{ type: context_switch, next_prev_pid: %u, next_prev_tid: %u, switch_out: %u }", + pevent->event.context_switch.next_prev_pid, + pevent->event.context_switch.next_prev_tid, + !!(pevent->event.header.misc & PERF_RECORD_MISC_SWITCH_OUT)) < 0) { + ret = PyErr_NoMemory(); + } else { + ret = PyUnicode_FromString(s); + free(s); + } + return ret; +} + +static PyTypeObject pyrf_context_switch_event__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.context_switch_event", + .tp_basicsize = sizeof(struct pyrf_event), + .tp_dealloc = (destructor)pyrf_event__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_context_switch_event__doc, + .tp_members = pyrf_context_switch_event__members, + .tp_getset = pyrf_event__getset, + .tp_repr = (reprfunc)pyrf_context_switch_event__repr, +}; + +static int pyrf_event__setup_types(void) +{ + int err; + + err = PyType_Ready(&pyrf_mmap_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_mmap2_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_lost_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_task_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_comm_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_throttle_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_read_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_sample_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_context_switch_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_stat_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_stat_round_event__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_callchain_node__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_callchain__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_branch_entry__type); + if (err < 0) + goto out; + err = PyType_Ready(&pyrf_branch_stack__type); + if (err < 0) + goto out; +out: + return err; +} + +static PyTypeObject *pyrf_event__type[] = { + [PERF_RECORD_MMAP] = &pyrf_mmap_event__type, + [PERF_RECORD_MMAP2] = &pyrf_mmap2_event__type, + [PERF_RECORD_LOST] = &pyrf_lost_event__type, + [PERF_RECORD_COMM] = &pyrf_comm_event__type, + [PERF_RECORD_EXIT] = &pyrf_task_event__type, + [PERF_RECORD_THROTTLE] = &pyrf_throttle_event__type, + [PERF_RECORD_UNTHROTTLE] = &pyrf_throttle_event__type, + [PERF_RECORD_FORK] = &pyrf_task_event__type, + [PERF_RECORD_READ] = &pyrf_read_event__type, + [PERF_RECORD_SAMPLE] = &pyrf_sample_event__type, + [PERF_RECORD_SWITCH] = &pyrf_context_switch_event__type, + [PERF_RECORD_SWITCH_CPU_WIDE] = &pyrf_context_switch_event__type, + [PERF_RECORD_STAT] = &pyrf_stat_event__type, + [PERF_RECORD_STAT_ROUND] = &pyrf_stat_round_event__type, +}; + +static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evsel, + struct perf_session *session, + struct machine *machine) +{ + struct pyrf_event *pevent; + struct perf_sample *sample; + int err; + u32 min_size; + bool needs_swap; + + if (!machine) + machine = session ? &session->machines.host : NULL; + + if (event->header.type >= ARRAY_SIZE(pyrf_event__type) || + pyrf_event__type[event->header.type] == NULL) { + return PyErr_Format(PyExc_TypeError, "Unexpected header type %u", + event->header.type); + } + + if (perf_event__too_small(event, &min_size)) { + return PyErr_Format(PyExc_ValueError, "Event size %u too small for type %u", + event->header.size, event->header.type); + } + + size_t copy_size = event->header.size; + + if (copy_size > sizeof(pevent->event)) { + return PyErr_Format(PyExc_TypeError, "Unexpected event size: %zd < %zu", + sizeof(pevent->event), copy_size); + } + + pevent = PyObject_New(struct pyrf_event, pyrf_event__type[event->header.type]); + if (pevent == NULL) + return PyErr_NoMemory(); + + /* Copy the event for memory safety and initialize variables. */ + memcpy(&pevent->event, event, copy_size); + if (copy_size < sizeof(pevent->event)) + memset((char *)&pevent->event + copy_size, 0, sizeof(pevent->event) - copy_size); + + if (event->header.type == PERF_RECORD_MMAP2) + pevent->event.mmap2.filename[sizeof(pevent->event.mmap2.filename) - 1] = '\0'; + + perf_sample__init(&pevent->sample, /*all=*/true); + pevent->callchain = NULL; + pevent->brstack = NULL; + pevent->al_resolved = false; + addr_location__init(&pevent->al); + + if (!evsel) + return (PyObject *)pevent; + + /* Parse the sample again so that pointers are within the copied event. */ + needs_swap = evsel->needs_swap; + + evsel->needs_swap = false; + err = evsel__parse_sample(evsel, &pevent->event, &pevent->sample); + evsel->needs_swap = needs_swap; + if (err < 0) { + Py_DECREF(pevent); + return PyErr_Format(PyExc_OSError, + "perf: can't parse sample, err=%d", err); + } + sample = &pevent->sample; + if (machine && sample->callchain) { + struct addr_location al; + struct callchain_cursor *cursor; + u64 i; + struct pyrf_callchain *pchain; + + addr_location__init(&al); + if (machine__resolve(machine, &al, sample) >= 0) { + cursor = get_tls_callchain_cursor(); + if (thread__resolve_callchain(al.thread, cursor, sample, + NULL, NULL, PERF_MAX_STACK_DEPTH) == 0) { + callchain_cursor_commit(cursor); + + pchain = PyObject_New(struct pyrf_callchain, &pyrf_callchain__type); + if (!pchain) { + addr_location__exit(&al); + Py_DECREF(pevent); + return NULL; + } + pchain->nr_frames = cursor->nr; + pchain->frames = calloc(pchain->nr_frames, + sizeof(*pchain->frames)); + if (!pchain->frames) { + Py_DECREF(pchain); + addr_location__exit(&al); + Py_DECREF(pevent); + return PyErr_NoMemory(); + } + struct callchain_cursor_node *node; + + for (i = 0; i < pchain->nr_frames; i++) { + node = callchain_cursor_current(cursor); + pchain->frames[i].ip = node->ip; + pchain->frames[i].map = + map__get(node->ms.map); + pchain->frames[i].sym = node->ms.sym; + callchain_cursor_advance(cursor); + } + pevent->callchain = (PyObject *)pchain; + } + addr_location__exit(&al); + } + } + if (sample->branch_stack) { + struct branch_stack *bs = sample->branch_stack; + struct branch_entry *entries = perf_sample__branch_entries(sample); + struct pyrf_branch_stack *pstack; + + pstack = PyObject_New(struct pyrf_branch_stack, &pyrf_branch_stack__type); + if (!pstack) { + Py_DECREF(pevent); + return NULL; + } + pstack->nr = bs->nr; + pstack->entries = calloc(bs->nr, sizeof(struct branch_entry)); + if (!pstack->entries) { + Py_DECREF(pstack); + Py_DECREF(pevent); + return PyErr_NoMemory(); + } + memcpy(pstack->entries, entries, + bs->nr * sizeof(struct branch_entry)); + pevent->brstack = (PyObject *)pstack; + } + return (PyObject *)pevent; +} + +struct pyrf_cpu_map { + PyObject_HEAD + + struct perf_cpu_map *cpus; +}; + +static int pyrf_cpu_map__init(struct pyrf_cpu_map *pcpus, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = { "cpustr", NULL }; + char *cpustr = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|s", kwlist, &cpustr)) @@ -942,7 +1951,7 @@ static int pyrf_counts_values__setup_types(void) struct pyrf_evsel { PyObject_HEAD - struct evsel evsel; + struct evsel *evsel; }; static int pyrf_evsel__init(struct pyrf_evsel *pevsel, @@ -983,6 +1992,7 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, "bp_type", "bp_addr", "bp_len", + "idx", NULL }; u64 sample_period = 0; @@ -1004,11 +2014,11 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, watermark = 0, precise_ip = 0, mmap_data = 0, - sample_id_all = 1; - int idx = 0; + sample_id_all = 1, + idx = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKK", kwlist, + "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKKi", kwlist, &attr.type, &attr.config, &attr.sample_freq, &sample_period, &attr.sample_type, &attr.read_format, &disabled, &inherit, @@ -1050,34 +2060,41 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel, attr.sample_id_all = sample_id_all; attr.size = sizeof(attr); - evsel__init(&pevsel->evsel, &attr, idx); + evsel__put(pevsel->evsel); + pevsel->evsel = evsel__new(&attr); + if (!pevsel->evsel) { + PyErr_NoMemory(); + return -1; + } return 0; } static void pyrf_evsel__delete(struct pyrf_evsel *pevsel) { - evsel__exit(&pevsel->evsel); + evsel__put(pevsel->evsel); Py_TYPE(pevsel)->tp_free((PyObject*)pevsel); } static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, PyObject *args, PyObject *kwargs) { - struct evsel *evsel = &pevsel->evsel; + struct evsel *evsel = pevsel->evsel; struct perf_cpu_map *cpus = NULL; struct perf_thread_map *threads = NULL; PyObject *pcpus = NULL, *pthreads = NULL; int group = 0, inherit = 0; static char *kwlist[] = { "cpus", "threads", "group", "inherit", NULL }; + CHECK_INITIALIZED(evsel, "evsel"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOii", kwlist, &pcpus, &pthreads, &group, &inherit)) return NULL; - if (pthreads != NULL) + if (pthreads != NULL && pthreads != Py_None) threads = ((struct pyrf_thread_map *)pthreads)->threads; - if (pcpus != NULL) + if (pcpus != NULL && pcpus != Py_None) cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; evsel->core.attr.inherit = inherit; @@ -1090,161 +2107,407 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, return NULL; } - Py_INCREF(Py_None); - return Py_None; + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel) +{ + struct pyrf_cpu_map *pcpu_map; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + if (pcpu_map) + pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel->core.cpus); + + return (PyObject *)pcpu_map; +} + +static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel) +{ + struct pyrf_thread_map *pthread_map; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + pthread_map = PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); + if (pthread_map) + pthread_map->threads = perf_thread_map__get(pevsel->evsel->core.threads); + + return (PyObject *)pthread_map; +} + +/* + * Ensure evsel's counts and prev_raw_counts are allocated, the latter + * used by tool PMUs to compute the cumulative count as expected by + * stat's process_counter_values. + */ +static int evsel__ensure_counts(struct evsel *evsel) +{ + int nthreads, ncpus; + + if (evsel->counts != NULL) + return 0; + + nthreads = perf_thread_map__nr(evsel->core.threads); + ncpus = perf_cpu_map__nr(evsel->core.cpus); + + evsel->counts = perf_counts__new(ncpus, nthreads); + if (evsel->counts == NULL) + return -ENOMEM; + + evsel->prev_raw_counts = perf_counts__new(ncpus, nthreads); + if (evsel->prev_raw_counts == NULL) + return -ENOMEM; + + return 0; +} + +static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, + PyObject *args, PyObject *kwargs) +{ + struct evsel *evsel = pevsel->evsel; + int cpu = 0, cpu_idx, thread = 0, thread_idx; + struct perf_counts_values *old_count, *new_count; + struct pyrf_counts_values *count_values; + + CHECK_INITIALIZED(evsel, "evsel"); + + count_values = PyObject_New(struct pyrf_counts_values, + &pyrf_counts_values__type); + if (!count_values) + return NULL; + + if (!PyArg_ParseTuple(args, "ii", &cpu, &thread)) + return NULL; + + cpu_idx = perf_cpu_map__idx(evsel->core.cpus, (struct perf_cpu){.cpu = cpu}); + if (cpu_idx < 0) { + PyErr_Format(PyExc_TypeError, "CPU %d is not part of evsel's CPUs", cpu); + return NULL; + } + thread_idx = perf_thread_map__idx(evsel->core.threads, thread); + if (thread_idx < 0) { + PyErr_Format(PyExc_TypeError, "Thread %d is not part of evsel's threads", + thread); + return NULL; + } + + if (evsel__ensure_counts(evsel)) + return PyErr_NoMemory(); + + /* Set up pointers to the old and newly read counter values. */ + old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx); + new_count = perf_counts(evsel->counts, cpu_idx, thread_idx); + /* Update the value in evsel->counts. */ + evsel__read_counter(evsel, cpu_idx, thread_idx); + /* Copy the value and turn it into the delta from old_count. */ + count_values->values = *new_count; + count_values->values.val -= old_count->val; + count_values->values.ena -= old_count->ena; + count_values->values.run -= old_count->run; + /* Save the new count over the old_count for the next read. */ + *old_count = *new_count; + return (PyObject *)count_values; +} + +static PyObject *pyrf_evsel__str(PyObject *self) +{ + struct pyrf_evsel *pevsel = (void *)self; + struct evsel *evsel = pevsel->evsel; + + if (!evsel) + return PyUnicode_FromString("evsel(uninitialized)"); + + return PyUnicode_FromFormat("evsel(%s)", evsel__name(evsel)); +} + +static PyMethodDef pyrf_evsel__methods[] = { + { + .ml_name = "open", + .ml_meth = (PyCFunction)pyrf_evsel__open, + .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("open the event selector file descriptor table.") + }, + { + .ml_name = "cpus", + .ml_meth = (PyCFunction)pyrf_evsel__cpus, + .ml_flags = METH_NOARGS, + .ml_doc = PyDoc_STR("CPUs the event is to be used with.") + }, + { + .ml_name = "threads", + .ml_meth = (PyCFunction)pyrf_evsel__threads, + .ml_flags = METH_NOARGS, + .ml_doc = PyDoc_STR("threads the event is to be used with.") + }, + { + .ml_name = "read", + .ml_meth = (PyCFunction)pyrf_evsel__read, + .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("read counters") + }, + { .ml_name = NULL, } +}; + +static PyObject *pyrf_evsel__get_tracking(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + if (pevsel->evsel->tracking) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +static int pyrf_evsel__set_tracking(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + int is_true; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + is_true = PyObject_IsTrue(val); + if (is_true < 0) + return -1; + + pevsel->evsel->tracking = is_true; + return 0; +} + +static int pyrf_evsel__set_attr_config(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + pevsel->evsel->core.attr.config = PyLong_AsUnsignedLongLong(val); + return PyErr_Occurred() ? -1 : 0; +} + +static PyObject *pyrf_evsel__get_attr_config(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.config); +} + +static int pyrf_evsel__set_attr_read_format(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + pevsel->evsel->core.attr.read_format = PyLong_AsUnsignedLongLong(val); + return PyErr_Occurred() ? -1 : 0; +} + +static PyObject *pyrf_evsel__get_attr_read_format(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.read_format); +} + +static int pyrf_evsel__set_attr_sample_period(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); + + pevsel->evsel->core.attr.sample_period = PyLong_AsUnsignedLongLong(val); + return PyErr_Occurred() ? -1 : 0; +} + +static PyObject *pyrf_evsel__get_attr_sample_period(PyObject *self, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_period); } -static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel) +static int pyrf_evsel__set_attr_sample_type(PyObject *self, PyObject *val, void *closure __maybe_unused) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_evsel *pevsel = (void *)self; - if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel.core.cpus); + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); - return (PyObject *)pcpu_map; + pevsel->evsel->core.attr.sample_type = PyLong_AsUnsignedLongLong(val); + return PyErr_Occurred() ? -1 : 0; } -static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel) +static PyObject *pyrf_evsel__get_attr_sample_type(PyObject *self, void *closure __maybe_unused) { - struct pyrf_thread_map *pthread_map = - PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type); + struct pyrf_evsel *pevsel = (void *)self; - if (pthread_map) - pthread_map->threads = perf_thread_map__get(pevsel->evsel.core.threads); + CHECK_INITIALIZED(pevsel->evsel, "evsel"); - return (PyObject *)pthread_map; + return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_type); } -/* - * Ensure evsel's counts and prev_raw_counts are allocated, the latter - * used by tool PMUs to compute the cumulative count as expected by - * stat's process_counter_values. - */ -static int evsel__ensure_counts(struct evsel *evsel) +static PyObject *pyrf_evsel__get_attr_size(PyObject *self, void *closure __maybe_unused) { - int nthreads, ncpus; + struct pyrf_evsel *pevsel = (void *)self; - if (evsel->counts != NULL) - return 0; + CHECK_INITIALIZED(pevsel->evsel, "evsel"); - nthreads = perf_thread_map__nr(evsel->core.threads); - ncpus = perf_cpu_map__nr(evsel->core.cpus); + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.size); +} - evsel->counts = perf_counts__new(ncpus, nthreads); - if (evsel->counts == NULL) - return -ENOMEM; +static int pyrf_evsel__set_attr_type(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; - evsel->prev_raw_counts = perf_counts__new(ncpus, nthreads); - if (evsel->prev_raw_counts == NULL) - return -ENOMEM; + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); - return 0; + pevsel->evsel->core.attr.type = PyLong_AsUnsignedLong(val); + return PyErr_Occurred() ? -1 : 0; } -static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel, - PyObject *args, PyObject *kwargs) +static PyObject *pyrf_evsel__get_attr_type(PyObject *self, void *closure __maybe_unused) { - struct evsel *evsel = &pevsel->evsel; - int cpu = 0, cpu_idx, thread = 0, thread_idx; - struct perf_counts_values *old_count, *new_count; - struct pyrf_counts_values *count_values = PyObject_New(struct pyrf_counts_values, - &pyrf_counts_values__type); + struct pyrf_evsel *pevsel = (void *)self; - if (!count_values) - return NULL; + CHECK_INITIALIZED(pevsel->evsel, "evsel"); - if (!PyArg_ParseTuple(args, "ii", &cpu, &thread)) - return NULL; + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.type); +} - cpu_idx = perf_cpu_map__idx(evsel->core.cpus, (struct perf_cpu){.cpu = cpu}); - if (cpu_idx < 0) { - PyErr_Format(PyExc_TypeError, "CPU %d is not part of evsel's CPUs", cpu); - return NULL; - } - thread_idx = perf_thread_map__idx(evsel->core.threads, thread); - if (thread_idx < 0) { - PyErr_Format(PyExc_TypeError, "Thread %d is not part of evsel's threads", - thread); - return NULL; - } +static int pyrf_evsel__set_attr_wakeup_events(PyObject *self, PyObject *val, void *closure __maybe_unused) +{ + struct pyrf_evsel *pevsel = (void *)self; - if (evsel__ensure_counts(evsel)) - return PyErr_NoMemory(); + CHECK_INITIALIZED_INT(pevsel->evsel, "evsel"); - /* Set up pointers to the old and newly read counter values. */ - old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx); - new_count = perf_counts(evsel->counts, cpu_idx, thread_idx); - /* Update the value in evsel->counts. */ - evsel__read_counter(evsel, cpu_idx, thread_idx); - /* Copy the value and turn it into the delta from old_count. */ - count_values->values = *new_count; - count_values->values.val -= old_count->val; - count_values->values.ena -= old_count->ena; - count_values->values.run -= old_count->run; - /* Save the new count over the old_count for the next read. */ - *old_count = *new_count; - return (PyObject *)count_values; + pevsel->evsel->core.attr.wakeup_events = PyLong_AsUnsignedLong(val); + return PyErr_Occurred() ? -1 : 0; } -static PyObject *pyrf_evsel__str(PyObject *self) +static PyObject *pyrf_evsel__get_attr_wakeup_events(PyObject *self, void *closure __maybe_unused) { struct pyrf_evsel *pevsel = (void *)self; - struct evsel *evsel = &pevsel->evsel; - return PyUnicode_FromFormat("evsel(%s/%s/)", evsel__pmu_name(evsel), evsel__name(evsel)); + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.wakeup_events); } -static PyMethodDef pyrf_evsel__methods[] = { +static PyObject *pyrf_evsel__get_ids(struct pyrf_evsel *pevsel, void *closure __maybe_unused) +{ + struct evsel *evsel; + PyObject *list; + + CHECK_INITIALIZED(pevsel->evsel, "evsel"); + + evsel = pevsel->evsel; + list = PyList_New(0); + + if (!list) + return NULL; + + for (u32 i = 0; i < evsel->core.ids; i++) { + PyObject *id = PyLong_FromUnsignedLongLong(evsel->core.id[i]); + int ret; + + if (!id) { + Py_DECREF(list); + return NULL; + } + ret = PyList_Append(list, id); + Py_DECREF(id); + if (ret < 0) { + Py_DECREF(list); + return NULL; + } + } + + return list; +} + +static PyGetSetDef pyrf_evsel__getset[] = { { - .ml_name = "open", - .ml_meth = (PyCFunction)pyrf_evsel__open, - .ml_flags = METH_VARARGS | METH_KEYWORDS, - .ml_doc = PyDoc_STR("open the event selector file descriptor table.") + .name = "ids", + .get = (getter)pyrf_evsel__get_ids, + .set = NULL, + .doc = "event IDs.", }, { - .ml_name = "cpus", - .ml_meth = (PyCFunction)pyrf_evsel__cpus, - .ml_flags = METH_NOARGS, - .ml_doc = PyDoc_STR("CPUs the event is to be used with.") + .name = "tracking", + .get = pyrf_evsel__get_tracking, + .set = pyrf_evsel__set_tracking, + .doc = "tracking event.", }, { - .ml_name = "threads", - .ml_meth = (PyCFunction)pyrf_evsel__threads, - .ml_flags = METH_NOARGS, - .ml_doc = PyDoc_STR("threads the event is to be used with.") + .name = "config", + .get = pyrf_evsel__get_attr_config, + .set = pyrf_evsel__set_attr_config, + .doc = "attribute config.", }, { - .ml_name = "read", - .ml_meth = (PyCFunction)pyrf_evsel__read, - .ml_flags = METH_VARARGS | METH_KEYWORDS, - .ml_doc = PyDoc_STR("read counters") + .name = "read_format", + .get = pyrf_evsel__get_attr_read_format, + .set = pyrf_evsel__set_attr_read_format, + .doc = "attribute read_format.", }, - { .ml_name = NULL, } + { + .name = "sample_period", + .get = pyrf_evsel__get_attr_sample_period, + .set = pyrf_evsel__set_attr_sample_period, + .doc = "attribute sample_period.", + }, + { + .name = "sample_type", + .get = pyrf_evsel__get_attr_sample_type, + .set = pyrf_evsel__set_attr_sample_type, + .doc = "attribute sample_type.", + }, + { + .name = "size", + .get = pyrf_evsel__get_attr_size, + .doc = "attribute size.", + }, + { + .name = "type", + .get = pyrf_evsel__get_attr_type, + .set = pyrf_evsel__set_attr_type, + .doc = "attribute type.", + }, + { + .name = "wakeup_events", + .get = pyrf_evsel__get_attr_wakeup_events, + .set = pyrf_evsel__set_attr_wakeup_events, + .doc = "attribute wakeup_events.", + }, + { .name = NULL}, }; -#define evsel_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.member), \ - 0, help } - -#define evsel_attr_member_def(member, ptype, help) \ - { #member, ptype, \ - offsetof(struct pyrf_evsel, evsel.core.attr.member), \ - 0, help } +static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object."); -static PyMemberDef pyrf_evsel__members[] = { - evsel_member_def(tracking, T_BOOL, "tracking event."), - evsel_attr_member_def(type, T_UINT, "attribute type."), - evsel_attr_member_def(size, T_UINT, "attribute size."), - evsel_attr_member_def(config, T_ULONGLONG, "attribute config."), - evsel_attr_member_def(sample_period, T_ULONGLONG, "attribute sample_period."), - evsel_attr_member_def(sample_type, T_ULONGLONG, "attribute sample_type."), - evsel_attr_member_def(read_format, T_ULONGLONG, "attribute read_format."), - evsel_attr_member_def(wakeup_events, T_UINT, "attribute wakeup_events."), - { .name = NULL, }, -}; +static PyObject *pyrf_evsel__getattro(struct pyrf_evsel *pevsel, PyObject *attr_name) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevsel, attr_name); +} -static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object."); +static int pyrf_evsel__setattro(struct pyrf_evsel *pevsel, PyObject *attr_name, PyObject *value) +{ + if (!pevsel->evsel) { + PyErr_SetString(PyExc_ValueError, "evsel not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevsel, attr_name, value); +} static PyTypeObject pyrf_evsel__type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -1253,23 +2516,35 @@ static PyTypeObject pyrf_evsel__type = { .tp_dealloc = (destructor)pyrf_evsel__delete, .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, .tp_doc = pyrf_evsel__doc, - .tp_members = pyrf_evsel__members, + .tp_getset = pyrf_evsel__getset, .tp_methods = pyrf_evsel__methods, .tp_init = (initproc)pyrf_evsel__init, .tp_str = pyrf_evsel__str, .tp_repr = pyrf_evsel__str, + .tp_getattro = (getattrofunc) pyrf_evsel__getattro, + .tp_setattro = (setattrofunc) pyrf_evsel__setattro, }; +static PyObject *pyrf_evsel__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evsel *pevsel; + + pevsel = (struct pyrf_evsel *)PyType_GenericNew(type, args, kwargs); + if (pevsel) + pevsel->evsel = NULL; + return (PyObject *)pevsel; +} + static int pyrf_evsel__setup_types(void) { - pyrf_evsel__type.tp_new = PyType_GenericNew; + pyrf_evsel__type.tp_new = pyrf_evsel__new; return PyType_Ready(&pyrf_evsel__type); } struct pyrf_evlist { PyObject_HEAD - struct evlist evlist; + struct evlist *evlist; }; static int pyrf_evlist__init(struct pyrf_evlist *pevlist, @@ -1279,40 +2554,55 @@ static int pyrf_evlist__init(struct pyrf_evlist *pevlist, struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "OO", &pcpus, &pthreads)) + if (!PyArg_ParseTuple(args, "O!O!", + &pyrf_cpu_map__type, &pcpus, + &pyrf_thread_map__type, &pthreads)) return -1; + evlist__put(pevlist->evlist); + pevlist->evlist = evlist__new(); + if (!pevlist->evlist) { + PyErr_NoMemory(); + return -1; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; - evlist__init(&pevlist->evlist, cpus, threads); + perf_evlist__set_maps(evlist__core(pevlist->evlist), cpus, threads); + return 0; } static void pyrf_evlist__delete(struct pyrf_evlist *pevlist) { - evlist__exit(&pevlist->evlist); + evlist__put(pevlist->evlist); Py_TYPE(pevlist)->tp_free((PyObject*)pevlist); } static PyObject *pyrf_evlist__all_cpus(struct pyrf_evlist *pevlist) { - struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); + struct pyrf_cpu_map *pcpu_map; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type); if (pcpu_map) - pcpu_map->cpus = perf_cpu_map__get(pevlist->evlist.core.all_cpus); + pcpu_map->cpus = perf_cpu_map__get(evlist__core(pevlist->evlist)->all_cpus); return (PyObject *)pcpu_map; } static PyObject *pyrf_evlist__metrics(struct pyrf_evlist *pevlist) { - PyObject *list = PyList_New(/*len=*/0); + PyObject *list; struct rb_node *node; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + list = PyList_New(/*len=*/0); if (!list) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); struct list_head *pos; @@ -1415,10 +2705,12 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, double result = 0; struct evsel *metric_evsel = NULL; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + if (!PyArg_ParseTuple(args, "sii", &metric, &cpu, &thread)) return NULL; - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); + for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); mexp == NULL && node; node = rb_next(node)) { struct metric_event *me = container_of(node, struct metric_event, nd); @@ -1434,7 +2726,7 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, if (e->metric_events[0] == NULL) continue; - evlist__for_each_entry(&pevlist->evlist, pos2) { + evlist__for_each_entry(pevlist->evlist, pos2) { if (pos2->metric_leader != e->metric_events[0]) continue; cpu_idx = perf_cpu_map__idx(pos2->core.cpus, @@ -1479,15 +2771,18 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "pages", "overwrite", NULL }; int pages = 128, overwrite = false; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ii", kwlist, &pages, &overwrite)) return NULL; - if (evlist__mmap(evlist, pages) < 0) { + if (evlist__do_mmap(evlist, pages) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -1499,10 +2794,13 @@ static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__poll(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; static char *kwlist[] = { "timeout", NULL }; int timeout = -1, n; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &timeout)) return NULL; @@ -1519,13 +2817,18 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist, PyObject *args __maybe_unused, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; - PyObject *list = PyList_New(0); + struct evlist *evlist; + PyObject *list; int i; - for (i = 0; i < evlist->core.pollfd.nr; ++i) { + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + list = PyList_New(0); + + for (i = 0; i < evlist__core(evlist)->pollfd.nr; ++i) { PyObject *file; - file = PyFile_FromFd(evlist->core.pollfd.entries[i].fd, "perf", "r", -1, + file = PyFile_FromFd(evlist__core(evlist)->pollfd.entries[i].fd, "perf", "r", -1, NULL, NULL, NULL, 0); if (file == NULL) goto free_list; @@ -1548,27 +2851,33 @@ static PyObject *pyrf_evlist__add(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs __maybe_unused) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; PyObject *pevsel; struct evsel *evsel; - if (!PyArg_ParseTuple(args, "O", &pevsel)) + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; + if (!PyArg_ParseTuple(args, "O!", &pyrf_evsel__type, &pevsel)) return NULL; - Py_INCREF(pevsel); - evsel = &((struct pyrf_evsel *)pevsel)->evsel; - evsel->core.idx = evlist->core.nr_entries; - evlist__add(evlist, evsel); + CHECK_INITIALIZED(((struct pyrf_evsel *)pevsel)->evsel, "evsel"); + + evsel = ((struct pyrf_evsel *)pevsel)->evsel; + CHECK_INITIALIZED(evsel, "evsel"); - return Py_BuildValue("i", evlist->core.nr_entries); + evsel->core.idx = evlist__nr_entries(evlist); + evlist__add(evlist, evsel__get(evsel)); + + return Py_BuildValue("i", evlist__nr_entries(evlist)); } static struct mmap *get_md(struct evlist *evlist, int cpu) { int i; - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *md = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *md = &evlist__mmap(evlist)[i]; if (md->core.cpu.cpu == cpu) return md; @@ -1580,65 +2889,60 @@ static struct mmap *get_md(struct evlist *evlist, int cpu) static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; union perf_event *event; + struct evsel *evsel; int sample_id_all = 1, cpu; static char *kwlist[] = { "cpu", "sample_id_all", NULL }; struct mmap *md; + PyObject *pyevent; int err; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|i", kwlist, &cpu, &sample_id_all)) return NULL; md = get_md(evlist, cpu); - if (!md) { - PyErr_Format(PyExc_TypeError, "Unknown CPU '%d'", cpu); - return NULL; + if (!md) + return PyErr_Format(PyExc_TypeError, "Unknown CPU '%d'", cpu); + + err = perf_mmap__read_init(&md->core); + if (err < 0) { + if (err == -EAGAIN) + Py_RETURN_NONE; + return PyErr_Format(PyExc_OSError, + "perf: error mmap read init, err=%d", err); } - if (perf_mmap__read_init(&md->core) < 0) - goto end; - event = perf_mmap__read_event(&md->core); - if (event != NULL) { - PyObject *pyevent = pyrf_event__new(event); - struct pyrf_event *pevent = (struct pyrf_event *)pyevent; - struct evsel *evsel; - - if (pyevent == NULL) - return PyErr_NoMemory(); - - evsel = evlist__event2evsel(evlist, event); - if (!evsel) { - Py_DECREF(pyevent); - Py_INCREF(Py_None); - return Py_None; - } - - pevent->evsel = evsel; + if (event == NULL) + Py_RETURN_NONE; + evsel = evlist__event2evsel(evlist, event); + if (!evsel) { + /* Unknown evsel. */ perf_mmap__consume(&md->core); - - err = evsel__parse_sample(evsel, &pevent->event, &pevent->sample); - if (err) { - Py_DECREF(pyevent); - return PyErr_Format(PyExc_OSError, - "perf: can't parse sample, err=%d", err); - } - - return pyevent; + Py_RETURN_NONE; } -end: - Py_INCREF(Py_None); - return Py_None; + pyevent = pyrf_event__new(event, evsel, evlist__session(evlist), /*machine=*/NULL); + perf_mmap__consume(&md->core); + if (pyevent == NULL) + return PyErr_Occurred() ? NULL : PyErr_NoMemory(); + + return pyevent; } static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, PyObject *args, PyObject *kwargs) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; if (evlist__open(evlist) < 0) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -1650,8 +2954,11 @@ static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist, static PyObject *pyrf_evlist__close(struct pyrf_evlist *pevlist) { - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist = pevlist->evlist; evlist__close(evlist); Py_INCREF(Py_None); @@ -1676,8 +2983,11 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) .no_buffering = true, .no_inherit = true, }; - struct evlist *evlist = &pevlist->evlist; + struct evlist *evlist; + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + + evlist = pevlist->evlist; evlist__config(evlist, &opts, &callchain_param); Py_INCREF(Py_None); return Py_None; @@ -1685,14 +2995,16 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist) static PyObject *pyrf_evlist__disable(struct pyrf_evlist *pevlist) { - evlist__disable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__disable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } static PyObject *pyrf_evlist__enable(struct pyrf_evlist *pevlist) { - evlist__enable(&pevlist->evlist); + CHECK_INITIALIZED(pevlist->evlist, "evlist"); + evlist__enable(pevlist->evlist); Py_INCREF(Py_None); return Py_None; } @@ -1783,7 +3095,21 @@ static Py_ssize_t pyrf_evlist__length(PyObject *obj) { struct pyrf_evlist *pevlist = (void *)obj; - return pevlist->evlist.core.nr_entries; + if (!pevlist->evlist) + return 0; + + return evlist__nr_entries(pevlist->evlist); +} + +static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) +{ + struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); + + if (!pevsel) + return NULL; + + pevsel->evsel = evsel__get(evsel); + return (PyObject *)pevsel; } static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) @@ -1791,17 +3117,16 @@ static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i) struct pyrf_evlist *pevlist = (void *)obj; struct evsel *pos; - if (i >= pevlist->evlist.core.nr_entries) { + if (!pevlist->evlist || i >= evlist__nr_entries(pevlist->evlist)) { PyErr_SetString(PyExc_IndexError, "Index out of range"); return NULL; } - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (i-- == 0) break; } - - return Py_BuildValue("O", container_of(pos, struct pyrf_evsel, evsel)); + return pyrf_evsel__from_evsel(pos); } static PyObject *pyrf_evlist__str(PyObject *self) @@ -1812,14 +3137,14 @@ static PyObject *pyrf_evlist__str(PyObject *self) bool first = true; PyObject *result; + if (!pevlist->evlist) + return PyUnicode_FromString("evlist(uninitialized)"); + strbuf_addstr(&sb, "evlist(["); - evlist__for_each_entry(&pevlist->evlist, pos) { + evlist__for_each_entry(pevlist->evlist, pos) { if (!first) strbuf_addch(&sb, ','); - if (!pos->pmu) - strbuf_addstr(&sb, evsel__name(pos)); - else - strbuf_addf(&sb, "%s/%s/", pos->pmu->name, evsel__name(pos)); + strbuf_addstr(&sb, evsel__name(pos)); first = false; } strbuf_addstr(&sb, "])"); @@ -1835,6 +3160,24 @@ static PySequenceMethods pyrf_evlist__sequence_methods = { static const char pyrf_evlist__doc[] = PyDoc_STR("perf event selector list object."); +static PyObject *pyrf_evlist__getattro(struct pyrf_evlist *pevlist, PyObject *attr_name) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) pevlist, attr_name); +} + +static int pyrf_evlist__setattro(struct pyrf_evlist *pevlist, PyObject *attr_name, PyObject *value) +{ + if (!pevlist->evlist) { + PyErr_SetString(PyExc_ValueError, "evlist not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) pevlist, attr_name, value); +} + static PyTypeObject pyrf_evlist__type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "perf.evlist", @@ -1847,11 +3190,23 @@ static PyTypeObject pyrf_evlist__type = { .tp_init = (initproc)pyrf_evlist__init, .tp_repr = pyrf_evlist__str, .tp_str = pyrf_evlist__str, + .tp_getattro = (getattrofunc) pyrf_evlist__getattro, + .tp_setattro = (setattrofunc) pyrf_evlist__setattro, }; +static PyObject *pyrf_evlist__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_evlist *pevlist; + + pevlist = (struct pyrf_evlist *)PyType_GenericNew(type, args, kwargs); + if (pevlist) + pevlist->evlist = NULL; + return (PyObject *)pevlist; +} + static int pyrf_evlist__setup_types(void) { - pyrf_evlist__type.tp_new = PyType_GenericNew; + pyrf_evlist__type.tp_new = pyrf_evlist__new; return PyType_Ready(&pyrf_evlist__type); } @@ -1871,6 +3226,7 @@ static const struct perf_constant perf__constants[] = { PERF_CONST(TYPE_BREAKPOINT), PERF_CONST(COUNT_HW_CPU_CYCLES), + PERF_CONST(COUNT_HW_REF_CPU_CYCLES), PERF_CONST(COUNT_HW_INSTRUCTIONS), PERF_CONST(COUNT_HW_CACHE_REFERENCES), PERF_CONST(COUNT_HW_CACHE_MISSES), @@ -1935,6 +3291,8 @@ static const struct perf_constant perf__constants[] = { PERF_CONST(RECORD_LOST_SAMPLES), PERF_CONST(RECORD_SWITCH), PERF_CONST(RECORD_SWITCH_CPU_WIDE), + PERF_CONST(RECORD_STAT), + PERF_CONST(RECORD_STAT_ROUND), PERF_CONST(RECORD_MISC_SWITCH_OUT), { .name = NULL, }, @@ -1954,157 +3312,107 @@ static PyObject *pyrf__tracepoint(struct pyrf_evsel *pevsel, return PyLong_FromLong(tp_pmu__id(sys, name)); } -static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel) -{ - struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type); - - if (!pevsel) - return NULL; - - memset(&pevsel->evsel, 0, sizeof(pevsel->evsel)); - evsel__init(&pevsel->evsel, &evsel->core.attr, evsel->core.idx); - - evsel__clone(&pevsel->evsel, evsel); - if (evsel__is_group_leader(evsel)) - evsel__set_leader(&pevsel->evsel, &pevsel->evsel); - return (PyObject *)pevsel; -} - -static int evlist__pos(struct evlist *evlist, struct evsel *evsel) -{ - struct evsel *pos; - int idx = 0; - - evlist__for_each_entry(evlist, pos) { - if (evsel == pos) - return idx; - idx++; - } - return -1; -} - -static struct evsel *evlist__at(struct evlist *evlist, int idx) -{ - struct evsel *pos; - int idx2 = 0; - - evlist__for_each_entry(evlist, pos) { - if (idx == idx2) - return pos; - idx2++; - } - return NULL; -} - static PyObject *pyrf_evlist__from_evlist(struct evlist *evlist) { struct pyrf_evlist *pevlist = PyObject_New(struct pyrf_evlist, &pyrf_evlist__type); - struct evsel *pos; - struct rb_node *node; if (!pevlist) return NULL; - memset(&pevlist->evlist, 0, sizeof(pevlist->evlist)); - evlist__init(&pevlist->evlist, evlist->core.all_cpus, evlist->core.threads); - evlist__for_each_entry(evlist, pos) { - struct pyrf_evsel *pevsel = (void *)pyrf_evsel__from_evsel(pos); - - evlist__add(&pevlist->evlist, &pevsel->evsel); - } - evlist__for_each_entry(&pevlist->evlist, pos) { - struct evsel *leader = evsel__leader(pos); - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - evsel__set_leader(pos, evlist__at(&pevlist->evlist, idx)); - else if (leader == NULL) - evsel__set_leader(pos, pos); - } - - leader = pos->metric_leader; - - if (pos != leader) { - int idx = evlist__pos(evlist, leader); - - if (idx >= 0) - pos->metric_leader = evlist__at(&pevlist->evlist, idx); - else if (leader == NULL) - pos->metric_leader = pos; - } - } - metricgroup__copy_metric_events(&pevlist->evlist, /*cgrp=*/NULL, - &pevlist->evlist.metric_events, - &evlist->metric_events); - for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node; - node = rb_next(node)) { - struct metric_event *me = container_of(node, struct metric_event, nd); - struct list_head *mpos; - int idx = evlist__pos(evlist, me->evsel); - - if (idx >= 0) - me->evsel = evlist__at(&pevlist->evlist, idx); - list_for_each(mpos, &me->head) { - struct metric_expr *e = container_of(mpos, struct metric_expr, nd); - - for (int j = 0; e->metric_events[j]; j++) { - idx = evlist__pos(evlist, e->metric_events[j]); - if (idx >= 0) - e->metric_events[j] = evlist__at(&pevlist->evlist, idx); - } - } - } + pevlist->evlist = evlist__get(evlist); return (PyObject *)pevlist; } static PyObject *pyrf__parse_events(PyObject *self, PyObject *args) { const char *input; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); struct parse_events_error err; PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; - if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) { + evlist__put(evlist); + return NULL; + } + + if (pthreads && pthreads != Py_None && + !PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) { + PyErr_SetString(PyExc_TypeError, "threads must be a perf.thread_map or None"); + evlist__put(evlist); + return NULL; + } + + if (pcpus && pcpus != Py_None && + !PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) { + PyErr_SetString(PyExc_TypeError, "cpus must be a perf.cpu_map or None"); + evlist__put(evlist); return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; parse_events_error__init(&err); - evlist__init(&evlist, cpus, threads); - if (parse_events(&evlist, input, &err)) { + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + if (parse_events(evlist, input, &err)) { parse_events_error__print(&err, input); PyErr_SetFromErrno(PyExc_OSError); + evlist__put(evlist); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) { const char *input, *pmu = NULL; - struct evlist evlist = {}; + struct evlist *evlist = evlist__new(); PyObject *result; PyObject *pcpus = NULL, *pthreads = NULL; struct perf_cpu_map *cpus; struct perf_thread_map *threads; int ret; - if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) + if (!evlist) + return PyErr_NoMemory(); + + if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) { + evlist__put(evlist); + return NULL; + } + + if (pthreads && pthreads != Py_None && + !PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) { + PyErr_SetString(PyExc_TypeError, "threads must be a perf.thread_map or None"); + evlist__put(evlist); return NULL; + } + + if (pcpus && pcpus != Py_None && + !PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) { + PyErr_SetString(PyExc_TypeError, "cpus must be a perf.cpu_map or None"); + evlist__put(evlist); + return NULL; + } - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; - evlist__init(&evlist, cpus, threads); - ret = metricgroup__parse_groups(&evlist, pmu ?: "all", input, + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + ret = metricgroup__parse_groups(evlist, pmu ?: "all", + /*cputype_filter=*/false, input, /*metric_no_group=*/ false, /*metric_no_merge=*/ false, /*metric_no_threshold=*/ true, @@ -2112,12 +3420,13 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) /*system_wide=*/true, /*hardware_aware_grouping=*/ false); if (ret) { + evlist__put(evlist); errno = -ret; PyErr_SetFromErrno(PyExc_OSError); return NULL; } - result = pyrf_evlist__from_evlist(&evlist); - evlist__exit(&evlist); + result = pyrf_evlist__from_evlist(evlist); + evlist__put(evlist); return result; } @@ -2200,7 +3509,544 @@ static PyObject *pyrf__metrics(PyObject *self, PyObject *args) return list; } +struct pyrf_data { + PyObject_HEAD + + struct perf_data data; +}; + +static int pyrf_data__init(struct pyrf_data *pdata, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = { "path", "fd", NULL }; + char *path = NULL; + int fd = -1; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|si", kwlist, &path, &fd)) + return -1; + + if (pdata->data.open) + perf_data__close(&pdata->data); + free((char *)pdata->data.path); + pdata->data.path = NULL; + + if (fd != -1) { + struct stat st; + + if (fstat(fd, &st) < 0 || !S_ISFIFO(st.st_mode)) { + PyErr_SetString(PyExc_ValueError, + "fd argument is only supported for pipes"); + return -1; + } + if (!path) + path = "-"; + else if (strcmp(path, "-") != 0) { + PyErr_SetString(PyExc_ValueError, + "path must be '-' when fd is provided"); + return -1; + } + fd = dup(fd); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } else if (path && strcmp(path, "-") == 0) { + fd = dup(0); + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } + + if (!path) + path = "perf.data"; + + pdata->data.path = strdup(path); + if (!pdata->data.path) { + if (fd != -1) + close(fd); + PyErr_NoMemory(); + return -1; + } + + pdata->data.mode = PERF_DATA_MODE_READ; + pdata->data.file.fd = fd; + if (perf_data__open(&pdata->data) < 0) { + PyErr_Format(PyExc_IOError, "Failed to open perf data: %s", + pdata->data.path ? pdata->data.path : "perf.data"); + return -1; + } + return 0; +} + +static void pyrf_data__delete(struct pyrf_data *pdata) +{ + perf_data__close(&pdata->data); + free((char *)pdata->data.path); + Py_TYPE(pdata)->tp_free((PyObject *)pdata); +} + +static PyObject *pyrf_data__str(PyObject *self) +{ + const struct pyrf_data *pdata = (const struct pyrf_data *)self; + + if (!pdata->data.path) + return PyUnicode_FromString("[uninitialized]"); + return PyUnicode_FromString(pdata->data.path); +} + +static PyObject *pyrf_data__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_data *pdata; + + pdata = (struct pyrf_data *)PyType_GenericNew(type, args, kwargs); + if (pdata) + memset(&pdata->data, 0, sizeof(pdata->data)); + return (PyObject *)pdata; +} + +static const char pyrf_data__doc[] = PyDoc_STR("perf data file object."); + +static PyTypeObject pyrf_data__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.data", + .tp_basicsize = sizeof(struct pyrf_data), + .tp_dealloc = (destructor)pyrf_data__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_doc = pyrf_data__doc, + .tp_init = (initproc)pyrf_data__init, + .tp_repr = pyrf_data__str, + .tp_str = pyrf_data__str, +}; + +static int pyrf_data__setup_types(void) +{ + pyrf_data__type.tp_new = pyrf_data__new; + return PyType_Ready(&pyrf_data__type); +} + +struct pyrf_thread { + PyObject_HEAD + + struct thread *thread; +}; + +static void pyrf_thread__delete(struct pyrf_thread *pthread) +{ + thread__put(pthread->thread); + Py_TYPE(pthread)->tp_free((PyObject *)pthread); +} + +static PyObject *pyrf_thread__comm(PyObject *obj) +{ + struct pyrf_thread *pthread = (void *)obj; + const char *str; + + CHECK_INITIALIZED(pthread->thread, "perf.thread"); + + str = thread__comm_str(pthread->thread); + + if (!str) + Py_RETURN_NONE; + + return PyUnicode_FromString(str); +} + +static PyMethodDef pyrf_thread__methods[] = { + { + .ml_name = "comm", + .ml_meth = (PyCFunction)pyrf_thread__comm, + .ml_flags = METH_NOARGS, + .ml_doc = PyDoc_STR("Comm(and) associated with this thread.") + }, + { .ml_name = NULL, } +}; + +static PyObject *pyrf_thread__get_pid(struct pyrf_thread *pthread, void *closure __maybe_unused) +{ + return PyLong_FromLong(thread__pid(pthread->thread)); +} + +static PyObject *pyrf_thread__get_tid(struct pyrf_thread *pthread, void *closure __maybe_unused) +{ + return PyLong_FromLong(thread__tid(pthread->thread)); +} + +static PyObject *pyrf_thread__get_ppid(struct pyrf_thread *pthread, void *closure __maybe_unused) +{ + return PyLong_FromLong(thread__ppid(pthread->thread)); +} + +static PyObject *pyrf_thread__get_cpu(struct pyrf_thread *pthread, void *closure __maybe_unused) +{ + return PyLong_FromLong(thread__cpu(pthread->thread)); +} + +static PyGetSetDef pyrf_thread__getset[] = { + { .name = "pid", .get = (getter)pyrf_thread__get_pid, .doc = "process ID" }, + { .name = "tid", .get = (getter)pyrf_thread__get_tid, .doc = "thread ID" }, + { .name = "ppid", .get = (getter)pyrf_thread__get_ppid, .doc = "parent process ID" }, + { .name = "cpu", .get = (getter)pyrf_thread__get_cpu, .doc = "cpu number" }, + { .name = NULL } +}; + +static const char pyrf_thread__doc[] = PyDoc_STR("perf thread object."); + +static PyTypeObject pyrf_thread__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.thread", + .tp_basicsize = sizeof(struct pyrf_thread), + .tp_dealloc = (destructor)pyrf_thread__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_methods = pyrf_thread__methods, + .tp_getset = pyrf_thread__getset, + .tp_doc = pyrf_thread__doc, +}; + +static int pyrf_thread__setup_types(void) +{ + return PyType_Ready(&pyrf_thread__type); +} + +static PyObject *pyrf_thread__from_thread(struct thread *thread) +{ + struct pyrf_thread *pthread = PyObject_New(struct pyrf_thread, &pyrf_thread__type); + + if (!pthread) + return NULL; + + pthread->thread = thread__get(thread); + return (PyObject *)pthread; +} + +struct pyrf_session { + PyObject_HEAD + + struct perf_session *session; + struct perf_tool tool; + struct pyrf_data *pdata; + PyObject *sample; + PyObject *stat; +}; + +static int pyrf_session_tool__sample(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); + PyObject *pyevent = pyrf_event__new(event, sample->evsel, psession->session, machine); + PyObject *ret; + + if (pyevent == NULL) + return -ENOMEM; + + ret = PyObject_CallFunction(psession->sample, "O", pyevent); + if (!ret) { + Py_DECREF(pyevent); + return -1; + } + Py_DECREF(ret); + Py_DECREF(pyevent); + return 0; +} + +static int pyrf_session_tool__stat(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); + struct evsel *evsel = evlist__id2evsel(session->evlist, event->stat.id); + PyObject *pyevent = pyrf_event__new(event, /*evsel=*/NULL, psession->session, + /*machine=*/NULL); + const char *name = evsel ? evsel__name(evsel) : "unknown"; + PyObject *ret; + + if (pyevent == NULL) + return -ENOMEM; + + ret = PyObject_CallFunction(psession->stat, "Oz", pyevent, name); + if (!ret) { + Py_DECREF(pyevent); + return -1; + } + Py_DECREF(ret); + Py_DECREF(pyevent); + return 0; +} + +static int pyrf_session_tool__stat_round(const struct perf_tool *tool, + struct perf_session *session __maybe_unused, + union perf_event *event) +{ + struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool); + PyObject *pyevent = pyrf_event__new(event, /*evsel=*/NULL, psession->session, + /*machine=*/NULL); + PyObject *ret; + + if (pyevent == NULL) + return -ENOMEM; + + ret = PyObject_CallFunction(psession->stat, "Oz", pyevent, NULL); + if (!ret) { + Py_DECREF(pyevent); + return -1; + } + Py_DECREF(ret); + Py_DECREF(pyevent); + return 0; +} + +static PyObject *pyrf_session__find_thread(struct pyrf_session *psession, PyObject *args) +{ + struct machine *machine; + struct thread *thread = NULL; + PyObject *result; + int pid; + + CHECK_INITIALIZED(psession->session, "session"); + + if (!PyArg_ParseTuple(args, "i", &pid)) + return NULL; + + machine = &psession->session->machines.host; + thread = machine__find_thread(machine, pid, pid); + + if (!thread) { + machine = perf_session__find_machine(psession->session, pid); + if (machine) + thread = machine__find_thread(machine, pid, pid); + } + + if (!thread) { + PyErr_Format(PyExc_TypeError, "Failed to find thread %d", pid); + return NULL; + } + result = pyrf_thread__from_thread(thread); + thread__put(thread); + return result; +} + +static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + struct pyrf_data *pdata; + PyObject *sample = NULL, *stat = NULL; + static char *kwlist[] = { "data", "sample", "stat", NULL }; + struct pyrf_session *psession; + struct perf_session *session; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!|OO", kwlist, &pyrf_data__type, &pdata, + &sample, &stat)) + return NULL; + + psession = PyObject_New(struct pyrf_session, type); + if (!psession) + return NULL; + + psession->session = NULL; + psession->sample = NULL; + psession->stat = NULL; + psession->pdata = NULL; + + Py_INCREF(pdata); + psession->pdata = pdata; + + perf_tool__init(&psession->tool, /*ordered_events=*/true); + psession->tool.ordering_requires_timestamps = true; + + #define ADD_TOOL(name) \ + do { \ + if (name) { \ + if (!PyCallable_Check(name)) { \ + PyErr_SetString(PyExc_TypeError, #name " must be callable"); \ + goto err_out; \ + } \ + psession->tool.name = pyrf_session_tool__##name; \ + Py_INCREF(name); \ + psession->name = name; \ + } \ + } while (0) + + ADD_TOOL(sample); + ADD_TOOL(stat); + #undef ADD_TOOL + + if (stat) + psession->tool.stat_round = pyrf_session_tool__stat_round; + + + psession->tool.comm = perf_event__process_comm; + psession->tool.mmap = perf_event__process_mmap; + psession->tool.mmap2 = perf_event__process_mmap2; + psession->tool.namespaces = perf_event__process_namespaces; + psession->tool.cgroup = perf_event__process_cgroup; + psession->tool.exit = perf_event__process_exit; + psession->tool.fork = perf_event__process_fork; + psession->tool.ksymbol = perf_event__process_ksymbol; + psession->tool.text_poke = perf_event__process_text_poke; + psession->tool.build_id = perf_event__process_build_id; + psession->tool.attr = perf_event__process_attr; + psession->tool.feature = perf_event__process_feature; + + session = perf_session__new(&pdata->data, &psession->tool); + if (IS_ERR(session)) { + PyErr_Format(PyExc_IOError, "failed to create session: %ld", PTR_ERR(session)); + goto err_out; + } + psession->session = session; + + symbol_conf.use_callchain = true; + symbol_conf.show_kernel_path = true; + symbol_conf.inline_name = false; + if (symbol__init(perf_session__env(session)) < 0) { + PyErr_SetString(PyExc_OSError, "perf: symbol__init failed"); + goto err_out; + } + + + + return (PyObject *)psession; +err_out: + Py_DECREF(psession); + return NULL; +} + +static void pyrf_session__delete(struct pyrf_session *psession) +{ + perf_session__delete(psession->session); + Py_XDECREF(psession->pdata); + Py_XDECREF(psession->sample); + Py_XDECREF(psession->stat); + Py_TYPE(psession)->tp_free((PyObject *)psession); +} + +static PyObject *pyrf_session__find_thread_events(struct pyrf_session *psession) +{ + int err; + + CHECK_INITIALIZED(psession->session, "session"); + + err = perf_session__process_events(psession->session); + + if (PyErr_Occurred()) + return NULL; + + if (err < 0) { + PyErr_Format(PyExc_OSError, "Process events failed: %d", err); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyMethodDef pyrf_session__methods[] = { + { + .ml_name = "process_events", + .ml_meth = (PyCFunction)pyrf_session__find_thread_events, + .ml_flags = METH_NOARGS, + .ml_doc = PyDoc_STR("Iterate and process events.") + }, + { + .ml_name = "find_thread", + .ml_meth = (PyCFunction)pyrf_session__find_thread, + .ml_flags = METH_VARARGS, + .ml_doc = PyDoc_STR("Returns the thread associated with a pid.") + }, + { .ml_name = NULL, } +}; + +static const char pyrf_session__doc[] = PyDoc_STR("perf session object."); + +static PyObject *pyrf_session__getattro(struct pyrf_session *psession, PyObject *attr_name) +{ + if (!psession->session) { + PyErr_SetString(PyExc_ValueError, "session not initialized"); + return NULL; + } + return PyObject_GenericGetAttr((PyObject *) psession, attr_name); +} + +static int pyrf_session__setattro(struct pyrf_session *psession, PyObject *attr_name, + PyObject *value) +{ + if (!psession->session) { + PyErr_SetString(PyExc_ValueError, "session not initialized"); + return -1; + } + return PyObject_GenericSetAttr((PyObject *) psession, attr_name, value); +} + +static PyTypeObject pyrf_session__type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "perf.session", + .tp_basicsize = sizeof(struct pyrf_session), + .tp_dealloc = (destructor)pyrf_session__delete, + .tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + .tp_methods = pyrf_session__methods, + .tp_doc = pyrf_session__doc, + .tp_new = pyrf_session__new, + .tp_getattro = (getattrofunc) pyrf_session__getattro, + .tp_setattro = (setattrofunc) pyrf_session__setattro, +}; + +static int pyrf_session__setup_types(void) +{ + return PyType_Ready(&pyrf_session__type); +} + +static PyObject *pyrf__syscall_name(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *name; + int id; + int elf_machine = EM_HOST; + static char *kwlist[] = { "id", "elf_machine", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|$i", kwlist, &id, &elf_machine)) + return NULL; + + name = syscalltbl__name(elf_machine, id); + if (!name) + Py_RETURN_NONE; + return PyUnicode_FromString(name); +} + +static PyObject *pyrf__syscall_id(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *name; + int id; + int elf_machine = EM_HOST; + static char *kwlist[] = { "name", "elf_machine", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|$i", kwlist, &name, &elf_machine)) + return NULL; + + id = syscalltbl__id(elf_machine, name); + if (id < 0) { + PyErr_Format(PyExc_ValueError, "Failed to find syscall %s", name); + return NULL; + } + return PyLong_FromLong(id); +} + +static PyObject *pyrf__config_get(PyObject *self, PyObject *args) +{ + const char *config_name, *val; + + if (!PyArg_ParseTuple(args, "s", &config_name)) + return NULL; + + val = perf_config_get(config_name); + if (!val) + Py_RETURN_NONE; + return PyUnicode_FromString(val); +} + static PyMethodDef perf__methods[] = { + { + .ml_name = "config_get", + .ml_meth = (PyCFunction) pyrf__config_get, + .ml_flags = METH_VARARGS, + .ml_doc = PyDoc_STR("Get a perf config value.") + }, { .ml_name = "metrics", .ml_meth = (PyCFunction) pyrf__metrics, @@ -2233,6 +4079,18 @@ static PyMethodDef perf__methods[] = { .ml_flags = METH_NOARGS, .ml_doc = PyDoc_STR("Returns a sequence of pmus.") }, + { + .ml_name = "syscall_name", + .ml_meth = (PyCFunction) pyrf__syscall_name, + .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("Turns a syscall number to a string.") + }, + { + .ml_name = "syscall_id", + .ml_meth = (PyCFunction) pyrf__syscall_id, + .ml_flags = METH_VARARGS | METH_KEYWORDS, + .ml_doc = PyDoc_STR("Turns a syscall name to a number.") + }, { .ml_name = NULL, } }; @@ -2254,29 +4112,48 @@ PyMODINIT_FUNC PyInit_perf(void) }; PyObject *module = PyModule_Create(&moduledef); - if (module == NULL || - pyrf_event__setup_types() < 0 || + if (module == NULL) + return NULL; + + if (pyrf_event__setup_types() < 0 || pyrf_evlist__setup_types() < 0 || pyrf_evsel__setup_types() < 0 || pyrf_thread_map__setup_types() < 0 || pyrf_cpu_map__setup_types() < 0 || pyrf_pmu_iterator__setup_types() < 0 || pyrf_pmu__setup_types() < 0 || - pyrf_counts_values__setup_types() < 0) - return module; + pyrf_counts_values__setup_types() < 0 || + pyrf_data__setup_types() < 0 || + pyrf_session__setup_types() < 0 || + pyrf_thread__setup_types() < 0) { + Py_DECREF(module); + return NULL; + } /* The page_size is placed in util object. */ page_size = sysconf(_SC_PAGE_SIZE); Py_INCREF(&pyrf_evlist__type); - PyModule_AddObject(module, "evlist", (PyObject*)&pyrf_evlist__type); + PyModule_AddObject(module, "evlist", (PyObject *)&pyrf_evlist__type); Py_INCREF(&pyrf_evsel__type); - PyModule_AddObject(module, "evsel", (PyObject*)&pyrf_evsel__type); + PyModule_AddObject(module, "evsel", (PyObject *)&pyrf_evsel__type); + + Py_INCREF(&pyrf_thread__type); + PyModule_AddObject(module, "thread", (PyObject *)&pyrf_thread__type); + + Py_INCREF(&pyrf_callchain__type); + PyModule_AddObject(module, "callchain", (PyObject *)&pyrf_callchain__type); + + Py_INCREF(&pyrf_callchain_node__type); + PyModule_AddObject(module, "callchain_node", (PyObject *)&pyrf_callchain_node__type); Py_INCREF(&pyrf_mmap_event__type); PyModule_AddObject(module, "mmap_event", (PyObject *)&pyrf_mmap_event__type); + Py_INCREF(&pyrf_mmap2_event__type); + PyModule_AddObject(module, "mmap2_event", (PyObject *)&pyrf_mmap2_event__type); + Py_INCREF(&pyrf_lost_event__type); PyModule_AddObject(module, "lost_event", (PyObject *)&pyrf_lost_event__type); @@ -2301,6 +4178,12 @@ PyMODINIT_FUNC PyInit_perf(void) Py_INCREF(&pyrf_context_switch_event__type); PyModule_AddObject(module, "switch_event", (PyObject *)&pyrf_context_switch_event__type); + Py_INCREF(&pyrf_stat_event__type); + PyModule_AddObject(module, "stat_event", (PyObject *)&pyrf_stat_event__type); + + Py_INCREF(&pyrf_stat_round_event__type); + PyModule_AddObject(module, "stat_round_event", (PyObject *)&pyrf_stat_round_event__type); + Py_INCREF(&pyrf_thread_map__type); PyModule_AddObject(module, "thread_map", (PyObject*)&pyrf_thread_map__type); @@ -2310,6 +4193,24 @@ PyMODINIT_FUNC PyInit_perf(void) Py_INCREF(&pyrf_counts_values__type); PyModule_AddObject(module, "counts_values", (PyObject *)&pyrf_counts_values__type); + Py_INCREF(&pyrf_data__type); + PyModule_AddObject(module, "data", (PyObject *)&pyrf_data__type); + + Py_INCREF(&pyrf_session__type); + PyModule_AddObject(module, "session", (PyObject *)&pyrf_session__type); + + Py_INCREF(&pyrf_branch_entry__type); + if (PyModule_AddObject(module, "branch_entry", (PyObject *)&pyrf_branch_entry__type) < 0) { + Py_DECREF(&pyrf_branch_entry__type); + goto error; + } + + Py_INCREF(&pyrf_branch_stack__type); + if (PyModule_AddObject(module, "branch_stack", (PyObject *)&pyrf_branch_stack__type) < 0) { + Py_DECREF(&pyrf_branch_stack__type); + goto error; + } + dict = PyModule_GetDict(module); if (dict == NULL) goto error; @@ -2323,7 +4224,9 @@ PyMODINIT_FUNC PyInit_perf(void) } error: - if (PyErr_Occurred()) - PyErr_SetString(PyExc_ImportError, "perf: Init failed!"); + if (PyErr_Occurred()) { + Py_XDECREF(module); + return NULL; + } return module; } diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c index e867de8ddaaa..38e8aee3106b 100644 --- a/tools/perf/util/record.c +++ b/tools/perf/util/record.c @@ -99,7 +99,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call bool use_comm_exec; bool sample_id = opts->sample_id; - if (perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0).cpu < 0) + if (perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0).cpu < 0) opts->no_inherit = true; use_comm_exec = perf_can_comm_exec(); @@ -122,7 +122,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call */ use_sample_identifier = perf_can_sample_identifier(); sample_id = true; - } else if (evlist->core.nr_entries > 1) { + } else if (evlist__nr_entries(evlist) > 1) { struct evsel *first = evlist__first(evlist); evlist__for_each_entry(evlist, evsel) { @@ -237,7 +237,8 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) evsel = evlist__last(temp_evlist); - if (!evlist || perf_cpu_map__is_any_cpu_or_is_empty(evlist->core.user_requested_cpus)) { + if (!evlist || + perf_cpu_map__is_any_cpu_or_is_empty(evlist__core(evlist)->user_requested_cpus)) { struct perf_cpu_map *cpus = perf_cpu_map__new_online_cpus(); if (cpus) @@ -245,7 +246,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) perf_cpu_map__put(cpus); } else { - cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0); + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0); } while (1) { @@ -264,7 +265,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str) ret = true; out_delete: - evlist__delete(temp_evlist); + evlist__put(temp_evlist); return ret; } diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c index c6ae0ae8d86a..aaf8838b2cb2 100644 --- a/tools/perf/util/s390-sample-raw.c +++ b/tools/perf/util/s390-sample-raw.c @@ -12,25 +12,26 @@ * sample was taken from. */ -#include +#include #include #include -#include -#include +#include #include #include -#include +#include +#include +#include "color.h" #include "debug.h" -#include "session.h" #include "evlist.h" -#include "color.h" #include "hashmap.h" -#include "sample-raw.h" +#include "pmu.h" +#include "pmus.h" #include "s390-cpumcf-kernel.h" -#include "util/pmu.h" -#include "util/sample.h" +#include "sample-raw.h" +#include "sample.h" +#include "session.h" static size_t ctrset_size(struct cf_ctrset_entry *set) { @@ -222,8 +223,9 @@ static char *get_counter_name(int set, int nr, struct perf_pmu *pmu) return result; } -static void s390_cpumcfdg_dump(struct perf_pmu *pmu, struct perf_sample *sample) +static void s390_cpumcfdg_dump(struct perf_sample *sample) { + struct perf_pmu *pmu = sample->evsel->pmu; size_t i, len = sample->raw_size, offset = 0; unsigned char *buf = sample->raw_data; const char *color = PERF_COLOR_BLUE; @@ -284,8 +286,9 @@ static bool s390_pai_all_test(struct perf_sample *sample) return true; } -static void s390_pai_all_dump(struct evsel *evsel, struct perf_sample *sample) +static void s390_pai_all_dump(struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; size_t len = sample->raw_size, offset = 0; unsigned char *p = sample->raw_data; const char *color = PERF_COLOR_BLUE; @@ -332,31 +335,33 @@ void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event, struct perf_sample *sample) { const char *pai_name; - struct evsel *evsel; if (event->header.type != PERF_RECORD_SAMPLE) return; - evsel = evlist__event2evsel(evlist, event); - if (!evsel) - return; + if (!sample->evsel) { + sample->evsel = evlist__event2evsel(evlist, event); + if (!sample->evsel) + return; + evsel__get(sample->evsel); + } /* Check for raw data in sample */ if (!sample->raw_size || !sample->raw_data) return; /* Display raw data on screen */ - if (evsel->core.attr.config == PERF_EVENT_CPUM_CF_DIAG) { - if (!evsel->pmu) - evsel->pmu = perf_pmus__find("cpum_cf"); + if (sample->evsel->core.attr.config == PERF_EVENT_CPUM_CF_DIAG) { + if (!sample->evsel->pmu) + sample->evsel->pmu = perf_pmus__find("cpum_cf"); if (!s390_cpumcfdg_testctr(sample)) pr_err("Invalid counter set data encountered\n"); else - s390_cpumcfdg_dump(evsel->pmu, sample); + s390_cpumcfdg_dump(sample); return; } - switch (evsel->core.attr.config) { + switch (sample->evsel->core.attr.config) { case PERF_EVENT_PAI_NNPA_ALL: pai_name = "NNPA_ALL"; break; @@ -370,8 +375,8 @@ void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event, if (!s390_pai_all_test(sample)) { pr_err("Invalid %s raw data encountered\n", pai_name); } else { - if (!evsel->pmu) - evsel->pmu = perf_pmus__find_by_type(evsel->core.attr.type); - s390_pai_all_dump(evsel, sample); + if (!sample->evsel->pmu) + sample->evsel->pmu = perf_pmus__find_by_type(sample->evsel->core.attr.type); + s390_pai_all_dump(sample); } } diff --git a/tools/perf/util/sample-raw.c b/tools/perf/util/sample-raw.c index bcf442574d6e..f5ae9f468983 100644 --- a/tools/perf/util/sample-raw.c +++ b/tools/perf/util/sample-raw.c @@ -1,11 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include "sample-raw.h" -#include +#include #include -#include "evlist.h" + #include "env.h" +#include "evlist.h" #include "header.h" -#include "sample-raw.h" #include "session.h" /* @@ -14,14 +15,14 @@ */ void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env) { - const char *arch_pf = perf_env__arch(env); - const char *cpuid = perf_env__cpuid(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + + if (e_machine == EM_S390) { + evlist__set_trace_event_sample_raw(evlist, evlist__s390_sample_raw); + } else if (e_machine == EM_X86_64 || e_machine == EM_386) { + const char *cpuid = perf_env__cpuid(env); - if (arch_pf && !strcmp("s390", arch_pf)) - evlist->trace_event_sample_raw = evlist__s390_sample_raw; - else if (arch_pf && !strcmp("x86", arch_pf) && - cpuid && strstarts(cpuid, "AuthenticAMD") && - evlist__has_amd_ibs(evlist)) { - evlist->trace_event_sample_raw = evlist__amd_sample_raw; + if (cpuid && strstarts(cpuid, "AuthenticAMD") && evlist__has_amd_ibs(evlist)) + evlist__set_trace_event_sample_raw(evlist, evlist__amd_sample_raw); } } diff --git a/tools/perf/util/sample-raw.h b/tools/perf/util/sample-raw.h index 896e9a87e373..c8d38c841c8c 100644 --- a/tools/perf/util/sample-raw.h +++ b/tools/perf/util/sample-raw.h @@ -2,7 +2,10 @@ #ifndef __SAMPLE_RAW_H #define __SAMPLE_RAW_H 1 +#include + struct evlist; +struct perf_env; union perf_event; struct perf_sample; @@ -12,4 +15,5 @@ bool evlist__has_amd_ibs(struct evlist *evlist); void evlist__amd_sample_raw(struct evlist *evlist, union perf_event *event, struct perf_sample *sample); void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env); -#endif /* __PERF_EVLIST_H */ + +#endif /* __SAMPLE_RAW_H */ diff --git a/tools/perf/util/sample.c b/tools/perf/util/sample.c index cf73329326d7..bccc19e2aaf2 100644 --- a/tools/perf/util/sample.c +++ b/tools/perf/util/sample.c @@ -1,18 +1,23 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include "sample.h" + +#include +#include + +#include +#include + +#include "../../arch/x86/include/asm/insn.h" #include "debug.h" +#include "evsel.h" #include "thread.h" -#include + #ifndef EM_CSKY #define EM_CSKY 252 #endif #ifndef EM_LOONGARCH #define EM_LOONGARCH 258 #endif -#include -#include -#include -#include "../../arch/x86/include/asm/insn.h" void perf_sample__init(struct perf_sample *sample, bool all) { @@ -29,6 +34,8 @@ void perf_sample__init(struct perf_sample *sample, bool all) void perf_sample__exit(struct perf_sample *sample) { + evsel__put(sample->evsel); + sample->evsel = NULL; zfree(&sample->user_regs); zfree(&sample->intr_regs); if (sample->merged_callchain) { diff --git a/tools/perf/util/sample.h b/tools/perf/util/sample.h index e556c9b656ea..c4eae8b2fd06 100644 --- a/tools/perf/util/sample.h +++ b/tools/perf/util/sample.h @@ -158,6 +158,8 @@ struct perf_sample { u64 code_page_size; /** @cgroup: The sample event PERF_SAMPLE_CGROUP value. */ u64 cgroup; + /** @file_offset: Byte offset of this event in the perf.data file. */ + u64 file_offset; /** @flags: Extra flag data from auxiliary events like intel-pt. */ u32 flags; /** @machine_pid: The guest machine pid derived from the sample id. */ diff --git a/tools/perf/util/scripting-engines/trace-event-perl.c b/tools/perf/util/scripting-engines/trace-event-perl.c index e261a57b87d4..410dc4cd0600 100644 --- a/tools/perf/util/scripting-engines/trace-event-perl.c +++ b/tools/perf/util/scripting-engines/trace-event-perl.c @@ -257,7 +257,6 @@ static void define_event_symbols(struct tep_event *event, } static SV *perl_process_callchain(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { struct callchain_cursor *cursor; @@ -272,7 +271,7 @@ static SV *perl_process_callchain(struct perf_sample *sample, cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack) != 0) { pr_err("Failed to resolve callchain. Skipping\n"); goto exit; @@ -304,7 +303,7 @@ static SV *perl_process_callchain(struct perf_sample *sample, } if (!hv_stores(sym, "start", newSVuv(node->ms.sym->start)) || !hv_stores(sym, "end", newSVuv(node->ms.sym->end)) || - !hv_stores(sym, "binding", newSVuv(node->ms.sym->binding)) || + !hv_stores(sym, "binding", newSVuv(symbol__binding(node->ms.sym))) || !hv_stores(sym, "name", newSVpvn(node->ms.sym->name, node->ms.sym->namelen)) || !hv_stores(elem, "sym", newRV_noinc((SV*)sym))) { @@ -340,7 +339,6 @@ static SV *perl_process_callchain(struct perf_sample *sample, } static void perl_process_tracepoint(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { struct thread *thread = al->thread; @@ -355,6 +353,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, unsigned long long nsecs = sample->time; const char *comm = thread__comm_str(thread); DECLARE_BITMAP(events_defined, TRACE_EVENT_TYPE_MAX); + struct evsel *evsel = sample->evsel; bitmap_zero(events_defined, TRACE_EVENT_TYPE_MAX); dSP; @@ -389,7 +388,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, XPUSHs(sv_2mortal(newSVuv(ns))); XPUSHs(sv_2mortal(newSViv(pid))); XPUSHs(sv_2mortal(newSVpv(comm, 0))); - XPUSHs(sv_2mortal(perl_process_callchain(sample, evsel, al))); + XPUSHs(sv_2mortal(perl_process_callchain(sample, al))); /* common fields other than pid can be accessed via xsub fns */ @@ -426,7 +425,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, XPUSHs(sv_2mortal(newSVuv(nsecs))); XPUSHs(sv_2mortal(newSViv(pid))); XPUSHs(sv_2mortal(newSVpv(comm, 0))); - XPUSHs(sv_2mortal(perl_process_callchain(sample, evsel, al))); + XPUSHs(sv_2mortal(perl_process_callchain(sample, al))); call_pv("main::trace_unhandled", G_SCALAR); } SPAGAIN; @@ -435,9 +434,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, LEAVE; } -static void perl_process_event_generic(union perf_event *event, - struct perf_sample *sample, - struct evsel *evsel) +static void perl_process_event_generic(union perf_event *event, struct perf_sample *sample) { dSP; @@ -448,7 +445,8 @@ static void perl_process_event_generic(union perf_event *event, SAVETMPS; PUSHMARK(SP); XPUSHs(sv_2mortal(newSVpvn((const char *)event, event->header.size))); - XPUSHs(sv_2mortal(newSVpvn((const char *)&evsel->core.attr, sizeof(evsel->core.attr)))); + XPUSHs(sv_2mortal(newSVpvn((const char *)&sample->evsel->core.attr, + sizeof(sample->evsel->core.attr)))); XPUSHs(sv_2mortal(newSVpvn((const char *)sample, sizeof(*sample)))); XPUSHs(sv_2mortal(newSVpvn((const char *)sample->raw_data, sample->raw_size))); PUTBACK; @@ -461,13 +459,12 @@ static void perl_process_event_generic(union perf_event *event, static void perl_process_event(union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { - scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); - perl_process_tracepoint(sample, evsel, al); - perl_process_event_generic(event, sample, evsel); + scripting_context__update(scripting_context, event, sample, al, addr_al); + perl_process_tracepoint(sample, al); + perl_process_event_generic(event, sample); } static void run_start_sub(void) diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index 5a30caaec73e..8f832ae316ca 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -390,7 +390,6 @@ static unsigned long get_offset(struct symbol *sym, struct addr_location *al) } static PyObject *python_process_callchain(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { PyObject *pylist; @@ -404,7 +403,7 @@ static PyObject *python_process_callchain(struct perf_sample *sample, goto exit; cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack) != 0) { pr_err("Failed to resolve callchain. Skipping\n"); @@ -437,7 +436,7 @@ static PyObject *python_process_callchain(struct perf_sample *sample, pydict_set_item_string_decref(pysym, "end", PyLong_FromUnsignedLongLong(node->ms.sym->end)); pydict_set_item_string_decref(pysym, "binding", - _PyLong_FromLong(node->ms.sym->binding)); + _PyLong_FromLong(symbol__binding(node->ms.sym))); pydict_set_item_string_decref(pysym, "name", _PyUnicode_FromStringAndSize(node->ms.sym->name, node->ms.sym->namelen)); @@ -651,11 +650,9 @@ static PyObject *get_sample_value_as_tuple(struct sample_read_value *value, return t; } -static void set_sample_read_in_dict(PyObject *dict_sample, - struct perf_sample *sample, - struct evsel *evsel) +static void set_sample_read_in_dict(PyObject *dict_sample, struct perf_sample *sample) { - u64 read_format = evsel->core.attr.read_format; + u64 read_format = sample->evsel->core.attr.read_format; PyObject *values; unsigned int i; @@ -741,11 +738,10 @@ static void regs_map(struct regs_dump *regs, uint64_t mask, uint16_t e_machine, static int set_regs_in_dict(PyObject *dict, struct perf_sample *sample, - struct evsel *evsel, uint16_t e_machine, uint32_t e_flags) { - struct perf_event_attr *attr = &evsel->core.attr; + struct perf_event_attr *attr = &sample->evsel->core.attr; int size = (__sw_hweight64(attr->sample_regs_intr) * MAX_REG_SIZE) + 1; char *bf = NULL; @@ -831,7 +827,6 @@ static void python_process_sample_flags(struct perf_sample *sample, PyObject *di } static PyObject *get_perf_sample_dict(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al, PyObject *callchain) @@ -839,6 +834,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyObject *dict, *dict_sample, *brstack, *brstacksym; uint16_t e_machine = EM_HOST; uint32_t e_flags = EF_HOST; + struct evsel *evsel = sample->evsel; dict = PyDict_New(); if (!dict) @@ -871,7 +867,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->phys_addr)); pydict_set_item_string_decref(dict_sample, "addr", PyLong_FromUnsignedLongLong(sample->addr)); - set_sample_read_in_dict(dict_sample, sample, evsel); + set_sample_read_in_dict(dict_sample, sample); pydict_set_item_string_decref(dict_sample, "weight", PyLong_FromUnsignedLongLong(sample->weight)); pydict_set_item_string_decref(dict_sample, "ins_lat", @@ -928,7 +924,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, if (al->thread) e_machine = thread__e_machine(al->thread, /*machine=*/NULL, &e_flags); - if (set_regs_in_dict(dict, sample, evsel, e_machine, e_flags)) + if (set_regs_in_dict(dict, sample, e_machine, e_flags)) Py_FatalError("Failed to setting regs in dict"); return dict; @@ -936,7 +932,6 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, #ifdef HAVE_LIBTRACEEVENT static void python_process_tracepoint(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { @@ -954,6 +949,7 @@ static void python_process_tracepoint(struct perf_sample *sample, const char *comm = thread__comm_str(al->thread); const char *default_handler_name = "trace_unhandled"; DECLARE_BITMAP(events_defined, TRACE_EVENT_TYPE_MAX); + struct evsel *evsel = sample->evsel; bitmap_zero(events_defined, TRACE_EVENT_TYPE_MAX); @@ -995,7 +991,7 @@ static void python_process_tracepoint(struct perf_sample *sample, PyTuple_SetItem(t, n++, context); /* ip unwinding */ - callchain = python_process_callchain(sample, evsel, al); + callchain = python_process_callchain(sample, al); /* Need an additional reference for the perf_sample dict */ Py_INCREF(callchain); @@ -1051,7 +1047,7 @@ static void python_process_tracepoint(struct perf_sample *sample, PyTuple_SetItem(t, n++, dict); if (get_argument_count(handler) == (int) n + 1) { - all_entries_dict = get_perf_sample_dict(sample, evsel, al, addr_al, + all_entries_dict = get_perf_sample_dict(sample, al, addr_al, callchain); PyTuple_SetItem(t, n++, all_entries_dict); } else { @@ -1070,7 +1066,6 @@ static void python_process_tracepoint(struct perf_sample *sample, } #else static void python_process_tracepoint(struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct addr_location *al __maybe_unused, struct addr_location *addr_al __maybe_unused) { @@ -1275,7 +1270,7 @@ static int python_export_symbol(struct db_export *dbe, struct symbol *sym, tuple_set_d64(t, 1, dso__db_id(dso)); tuple_set_d64(t, 2, sym->start); tuple_set_d64(t, 3, sym->end); - tuple_set_s32(t, 4, sym->binding); + tuple_set_s32(t, 4, symbol__binding(sym)); tuple_set_string(t, 5, sym->name); call_object(tables->symbol_handler, t, "symbol_table"); @@ -1312,7 +1307,7 @@ static void python_export_sample_table(struct db_export *dbe, t = tuple_new(28); tuple_set_d64(t, 0, es->db_id); - tuple_set_d64(t, 1, es->evsel->db_id); + tuple_set_d64(t, 1, es->sample->evsel->db_id); tuple_set_d64(t, 2, maps__machine(thread__maps(es->al->thread))->db_id); tuple_set_d64(t, 3, thread__db_id(es->al->thread)); tuple_set_d64(t, 4, es->comm_db_id); @@ -1353,7 +1348,7 @@ static void python_export_synth(struct db_export *dbe, struct export_sample *es) t = tuple_new(3); tuple_set_d64(t, 0, es->db_id); - tuple_set_d64(t, 1, es->evsel->core.attr.config); + tuple_set_d64(t, 1, es->sample->evsel->core.attr.config); tuple_set_bytes(t, 2, es->sample->raw_data, es->sample->raw_size); call_object(tables->synth_handler, t, "synth_data"); @@ -1368,7 +1363,7 @@ static int python_export_sample(struct db_export *dbe, python_export_sample_table(dbe, es); - if (es->evsel->core.attr.type == PERF_TYPE_SYNTH && tables->synth_handler) + if (es->sample->evsel->core.attr.type == PERF_TYPE_SYNTH && tables->synth_handler) python_export_synth(dbe, es); return 0; @@ -1465,7 +1460,6 @@ static int python_process_call_return(struct call_return *cr, u64 *parent_db_id, } static void python_process_general_event(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { @@ -1488,8 +1482,8 @@ static void python_process_general_event(struct perf_sample *sample, Py_FatalError("couldn't create Python tuple"); /* ip unwinding */ - callchain = python_process_callchain(sample, evsel, al); - dict = get_perf_sample_dict(sample, evsel, al, addr_al, callchain); + callchain = python_process_callchain(sample, al); + dict = get_perf_sample_dict(sample, al, addr_al, callchain); PyTuple_SetItem(t, n++, dict); if (_PyTuple_Resize(&t, n) == -1) @@ -1502,24 +1496,23 @@ static void python_process_general_event(struct perf_sample *sample, static void python_process_event(union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { struct tables *tables = &tables_global; - scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); + scripting_context__update(scripting_context, event, sample, al, addr_al); - switch (evsel->core.attr.type) { + switch (sample->evsel->core.attr.type) { case PERF_TYPE_TRACEPOINT: - python_process_tracepoint(sample, evsel, al, addr_al); + python_process_tracepoint(sample, al, addr_al); break; /* Reserve for future process_hw/sw/raw APIs */ default: if (tables->db_export_mode) - db_export__sample(&tables->dbe, event, sample, evsel, al, addr_al); + db_export__sample(&tables->dbe, event, sample, al, addr_al); else - python_process_general_event(sample, evsel, al, addr_al); + python_process_general_event(sample, al, addr_al); } } @@ -1614,6 +1607,9 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u const char *handler_name = "auxtrace_error"; unsigned long long tm = e->time; const char *msg = e->msg; + s32 machine_pid = 0, vcpu = 0; + char msg_buf[MAX_AUXTRACE_ERROR_MSG + 1]; + int msg_max; PyObject *handler, *t; handler = get_handler(handler_name); @@ -1625,6 +1621,25 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u msg = (const char *)&e->time; } + /* Bound msg to the bytes within the event, ensure NUL-termination */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max <= 0) { + msg_buf[0] = '\0'; + } else { + if (msg_max > (int)sizeof(msg_buf) - 1) + msg_max = sizeof(msg_buf) - 1; + memcpy(msg_buf, msg, msg_max); + msg_buf[msg_max] = '\0'; + } + + /* Only access fmt >= 2 fields if the event is large enough */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu)) { + machine_pid = e->machine_pid; + vcpu = e->vcpu; + } + t = tuple_new(11); tuple_set_u32(t, 0, e->type); @@ -1634,10 +1649,10 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u tuple_set_s32(t, 4, e->tid); tuple_set_u64(t, 5, e->ip); tuple_set_u64(t, 6, tm); - tuple_set_string(t, 7, msg); + tuple_set_string(t, 7, msg_buf); tuple_set_u32(t, 8, cpumode); - tuple_set_s32(t, 9, e->machine_pid); - tuple_set_s32(t, 10, e->vcpu); + tuple_set_s32(t, 9, machine_pid); + tuple_set_s32(t, 10, vcpu); call_object(handler, t, handler_name); diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index fe0de2a0277f..3237870a1a34 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -160,11 +160,12 @@ struct perf_session *__perf_session__new(struct perf_data *data, session->decomp_data.zstd_decomp = &session->zstd_data; session->active_decomp = &session->decomp_data; INIT_LIST_HEAD(&session->auxtrace_index); - machines__init(&session->machines); + perf_env__init(&session->header.env); + if (machines__init(&session->machines)) + goto out_delete; + ordered_events__init(&session->ordered_events, ordered_events__deliver_event, NULL); - - perf_env__init(&session->header.env); if (data) { ret = perf_data__open(data); if (ret < 0) @@ -204,7 +205,7 @@ struct perf_session *__perf_session__new(struct perf_data *data, session->machines.host.env = host_env; } if (session->evlist) - session->evlist->session = session; + evlist__set_session(session->evlist, session); session->machines.host.single_address_space = perf_env__single_address_space(session->machines.host.env); @@ -264,7 +265,7 @@ void perf_session__delete(struct perf_session *session) machines__exit(&session->machines); if (session->data) { if (perf_data__is_read(session->data)) - evlist__delete(session->evlist); + evlist__put(session->evlist); perf_data__close(session->data); } #ifdef HAVE_LIBTRACEEVENT @@ -276,34 +277,58 @@ void perf_session__delete(struct perf_session *session) static void swap_sample_id_all(union perf_event *event, void *data) { void *end = (void *) event + event->header.size; - int size = end - data; + int size; - BUG_ON(size % sizeof(u64)); - mem_bswap_64(data, size); + if (data >= end) + return; + + size = end - data; + if (size % sizeof(u64)) { + pr_warning("swap_sample_id_all: unaligned sample_id_all remainder (%d), skipping swap\n", size); + return; + } + if (size > 0) + mem_bswap_64(data, size); } -static void perf_event__all64_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__all64_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { struct perf_event_header *hdr = &event->header; - mem_bswap_64(hdr + 1, event->header.size - sizeof(*hdr)); + size_t size = event->header.size - sizeof(*hdr); + + /* mem_bswap_64 rounds up to 8-byte chunks — unaligned size overruns the buffer */ + if (size % sizeof(u64)) + return -1; + mem_bswap_64(hdr + 1, size); + return 0; } -static void perf_event__comm_swap(union perf_event *event, bool sample_id_all) +static int perf_event__comm_swap(union perf_event *event, bool sample_id_all) { event->comm.pid = bswap_32(event->comm.pid); event->comm.tid = bswap_32(event->comm.tid); if (sample_id_all) { void *data = &event->comm.comm; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* + * No NUL within the event boundary — can't locate where + * sample_id_all starts. Reject so the event is skipped + * rather than swapping garbage. + */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__mmap_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__mmap_swap(union perf_event *event, + bool sample_id_all) { event->mmap.pid = bswap_32(event->mmap.pid); event->mmap.tid = bswap_32(event->mmap.tid); @@ -313,13 +338,19 @@ static void perf_event__mmap_swap(union perf_event *event, if (sample_id_all) { void *data = &event->mmap.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__mmap2_swap(union perf_event *event, +static int perf_event__mmap2_swap(union perf_event *event, bool sample_id_all) { event->mmap2.pid = bswap_32(event->mmap2.pid); @@ -337,12 +368,19 @@ static void perf_event__mmap2_swap(union perf_event *event, if (sample_id_all) { void *data = &event->mmap2.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__task_swap(union perf_event *event, bool sample_id_all) + +static int perf_event__task_swap(union perf_event *event, bool sample_id_all) { event->fork.pid = bswap_32(event->fork.pid); event->fork.tid = bswap_32(event->fork.tid); @@ -352,22 +390,31 @@ static void perf_event__task_swap(union perf_event *event, bool sample_id_all) if (sample_id_all) swap_sample_id_all(event, &event->fork + 1); + return 0; } -static void perf_event__read_swap(union perf_event *event, bool sample_id_all) +static int perf_event__read_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { + size_t tail; + event->read.pid = bswap_32(event->read.pid); event->read.tid = bswap_32(event->read.tid); - event->read.value = bswap_64(event->read.value); - event->read.time_enabled = bswap_64(event->read.time_enabled); - event->read.time_running = bswap_64(event->read.time_running); - event->read.id = bswap_64(event->read.id); - - if (sample_id_all) - swap_sample_id_all(event, &event->read + 1); + /* + * Everything after pid/tid is u64: the read values (variable + * set determined by attr.read_format, which we don't have + * here) optionally followed by sample_id_all fields. + * Since all are u64, swap the entire remaining tail at once. + */ + tail = event->header.size - offsetof(struct perf_record_read, value); + /* mem_bswap_64 rounds up to 8-byte chunks — unaligned tail overruns the buffer */ + if (tail % sizeof(u64)) + return -1; + mem_bswap_64(&event->read.value, tail); + return 0; } -static void perf_event__aux_swap(union perf_event *event, bool sample_id_all) +static int perf_event__aux_swap(union perf_event *event, bool sample_id_all) { event->aux.aux_offset = bswap_64(event->aux.aux_offset); event->aux.aux_size = bswap_64(event->aux.aux_size); @@ -375,19 +422,21 @@ static void perf_event__aux_swap(union perf_event *event, bool sample_id_all) if (sample_id_all) swap_sample_id_all(event, &event->aux + 1); + return 0; } -static void perf_event__itrace_start_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__itrace_start_swap(union perf_event *event, + bool sample_id_all) { event->itrace_start.pid = bswap_32(event->itrace_start.pid); event->itrace_start.tid = bswap_32(event->itrace_start.tid); if (sample_id_all) swap_sample_id_all(event, &event->itrace_start + 1); + return 0; } -static void perf_event__switch_swap(union perf_event *event, bool sample_id_all) +static int perf_event__switch_swap(union perf_event *event, bool sample_id_all) { if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) { event->context_switch.next_prev_pid = @@ -396,30 +445,45 @@ static void perf_event__switch_swap(union perf_event *event, bool sample_id_all) bswap_32(event->context_switch.next_prev_tid); } - if (sample_id_all) - swap_sample_id_all(event, &event->context_switch + 1); + if (sample_id_all) { + /* + * PERF_RECORD_SWITCH has no fields beyond the header; + * SWITCH_CPU_WIDE adds pid/tid. Use the right offset + * so sample_id starts at the correct position. + */ + if (event->header.type == PERF_RECORD_SWITCH) + swap_sample_id_all(event, (void *)event + sizeof(event->header)); + else + swap_sample_id_all(event, &event->context_switch + 1); + } + return 0; } -static void perf_event__text_poke_swap(union perf_event *event, bool sample_id_all) +static int perf_event__text_poke_swap(union perf_event *event, bool sample_id_all) { event->text_poke.addr = bswap_64(event->text_poke.addr); event->text_poke.old_len = bswap_16(event->text_poke.old_len); event->text_poke.new_len = bswap_16(event->text_poke.new_len); if (sample_id_all) { + void *data = &event->text_poke.old_len; + void *end = (void *)event + event->header.size; size_t len = sizeof(event->text_poke.old_len) + sizeof(event->text_poke.new_len) + event->text_poke.old_len + event->text_poke.new_len; - void *data = &event->text_poke.old_len; + /* old_len + new_len exceeds event — can't find sample_id_all */ + if (data + len > end) + return -1; data += PERF_ALIGN(len, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__throttle_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__throttle_swap(union perf_event *event, + bool sample_id_all) { event->throttle.time = bswap_64(event->throttle.time); event->throttle.id = bswap_64(event->throttle.id); @@ -427,18 +491,41 @@ static void perf_event__throttle_swap(union perf_event *event, if (sample_id_all) swap_sample_id_all(event, &event->throttle + 1); + return 0; } -static void perf_event__namespaces_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__namespaces_swap(union perf_event *event, + bool sample_id_all) { - u64 i; + u64 i, nr, max_nr; event->namespaces.pid = bswap_32(event->namespaces.pid); event->namespaces.tid = bswap_32(event->namespaces.tid); event->namespaces.nr_namespaces = bswap_64(event->namespaces.nr_namespaces); - for (i = 0; i < event->namespaces.nr_namespaces; i++) { + nr = event->namespaces.nr_namespaces; + /* + * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof. + * When sample_id_all is present max_nr slightly overestimates the + * array space because header.size includes the trailing sample_id. + * Harmless: both the per-element bswap_64 loop and swap_sample_id_all() + * perform the same u64 byte swap, so the result is correct regardless + * of where the boundary between array and sample_id falls. + */ + max_nr = (event->header.size - sizeof(event->namespaces)) / + sizeof(event->namespaces.link_info[0]); + /* + * Safe to clamp: each namespace entry is indexed by type; + * missing entries just won't be resolved. + */ + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n", + nr, max_nr); + nr = max_nr; + event->namespaces.nr_namespaces = nr; + } + + for (i = 0; i < nr; i++) { struct perf_ns_link_info *ns = &event->namespaces.link_info[i]; ns->dev = bswap_64(ns->dev); @@ -447,18 +534,25 @@ static void perf_event__namespaces_swap(union perf_event *event, if (sample_id_all) swap_sample_id_all(event, &event->namespaces.link_info[i]); + return 0; } -static void perf_event__cgroup_swap(union perf_event *event, bool sample_id_all) +static int perf_event__cgroup_swap(union perf_event *event, bool sample_id_all) { event->cgroup.id = bswap_64(event->cgroup.id); if (sample_id_all) { void *data = &event->cgroup.path; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } static u8 revbyte(u8 b) @@ -499,9 +593,19 @@ void perf_event__attr_swap(struct perf_event_attr *attr) attr->type = bswap_32(attr->type); attr->size = bswap_32(attr->size); -#define bswap_safe(f, n) \ - (attr->size > (offsetof(struct perf_event_attr, f) + \ - sizeof(attr->f) * (n))) + /* + * ABI0: size == 0 means the producer didn't set it. + * Assume PERF_ATTR_SIZE_VER0 so bswap_safe() below + * correctly swaps the VER0 fields instead of skipping + * everything. Same convention as read_attr(). + */ + if (!attr->size) + attr->size = PERF_ATTR_SIZE_VER0; + +/* Verify the full field extent fits, not just its start offset */ +#define bswap_safe(f, n) \ + (attr->size >= (offsetof(struct perf_event_attr, f) + \ + sizeof(attr->f) * ((n) + 1))) #define bswap_field(f, sz) \ do { \ if (bswap_safe(f, 0)) \ @@ -539,40 +643,189 @@ do { \ #undef bswap_safe } -static void perf_event__hdr_attr_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__hdr_attr_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { + u32 attr_size, payload_size; size_t size; + /* + * Validate attr.size (still foreign-endian) before calling + * perf_event__attr_swap(), which uses it via bswap_safe() + * to decide which fields to swap. A crafted attr.size + * larger than the event payload would swap past the event + * boundary and corrupt adjacent memory. + * + * header.size alignment is already validated by + * perf_session__process_event(). The min_size table + * guarantees header.size >= sizeof(header) + + * PERF_ATTR_SIZE_VER0, so attr.size is safe to access. + */ + attr_size = bswap_32(event->attr.attr.size); + /* + * ABI0: size field not set. This only happens in pipe/inject + * mode where HEADER_ATTR events carry their own attr. For + * regular perf.data files, read_attr() uses f_header.attr_size + * from the file header instead. Assume PERF_ATTR_SIZE_VER0. + */ + if (!attr_size) + attr_size = PERF_ATTR_SIZE_VER0; + payload_size = event->header.size - sizeof(event->header); + + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size % sizeof(u64) || + attr_size > payload_size) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (min: %d, max: %u, 8-byte aligned)\n", + attr_size, PERF_ATTR_SIZE_VER0, payload_size); + return -1; + } + perf_event__attr_swap(&event->attr.attr); size = event->header.size; size -= perf_record_header_attr_id(event) - (void *)event; mem_bswap_64(perf_record_header_attr_id(event), size); + return 0; } -static void perf_event__event_update_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__build_id_swap(union perf_event *event, + bool sample_id_all) { - event->event_update.type = bswap_64(event->event_update.type); - event->event_update.id = bswap_64(event->event_update.id); + event->build_id.pid = bswap_32(event->build_id.pid); + + if (sample_id_all) { + void *data = &event->build_id.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); + + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); + swap_sample_id_all(event, data); + } + return 0; } -static void perf_event__event_type_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__event_update_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + struct perf_record_event_update *ev = &event->event_update; + + ev->type = bswap_64(ev->type); + ev->id = bswap_64(ev->id); + + /* + * Swap variant-specific fields so the processing path + * sees native byte order. + */ + if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) + return -1; + mem_bswap_64(&ev->scale.scale, sizeof(ev->scale.scale)); + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + u32 cpus_payload; + struct perf_record_cpu_map_data *data = &ev->cpus.cpus; + + /* CPUS fields start at the same offset as scale (union) */ + if (event->header.size < offsetof(struct perf_record_event_update, cpus) + + sizeof(__u16) + sizeof(struct perf_record_range_cpu_map)) + return -1; + cpus_payload = event->header.size - offsetof(struct perf_record_event_update, cpus); + data->type = bswap_16(data->type); + /* + * Full swap including array elements — same logic as + * perf_event__cpu_map_swap() but scoped to the + * embedded cpu_map_data within EVENT_UPDATE. + */ + switch (data->type) { + case PERF_CPU_MAP__CPUS: { + u16 nr, max_nr; + + data->cpus_data.nr = bswap_16(data->cpus_data.nr); + nr = data->cpus_data.nr; + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + if (nr > max_nr) { + nr = max_nr; + data->cpus_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]); + break; + } + case PERF_CPU_MAP__MASK: + data->mask32_data.long_size = bswap_16(data->mask32_data.long_size); + switch (data->mask32_data.long_size) { + case 4: { + u16 nr, max_nr; + + data->mask32_data.nr = bswap_16(data->mask32_data.nr); + nr = data->mask32_data.nr; + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + if (nr > max_nr) { + nr = max_nr; + data->mask32_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]); + break; + } + case 8: { + u16 nr, max_nr; + + data->mask64_data.nr = bswap_16(data->mask64_data.nr); + nr = data->mask64_data.nr; + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + data->mask64_data.nr = 0; + break; + } + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (nr > max_nr) { + nr = max_nr; + data->mask64_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]); + break; + } + default: + break; + } + break; + case PERF_CPU_MAP__RANGE_CPUS: + data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu); + data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu); + break; + default: + break; + } + } + return 0; +} + +static int perf_event__event_type_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->event_type.event_type.event_id = bswap_64(event->event_type.event_type.event_id); + return 0; } -static void perf_event__tracing_data_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__tracing_data_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->tracing_data.size = bswap_32(event->tracing_data.size); + return 0; } -static void perf_event__auxtrace_info_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_info_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { size_t size; @@ -581,10 +834,11 @@ static void perf_event__auxtrace_info_swap(union perf_event *event, size = event->header.size; size -= (void *)&event->auxtrace_info.priv - (void *)event; mem_bswap_64(event->auxtrace_info.priv, size); + return 0; } -static void perf_event__auxtrace_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->auxtrace.size = bswap_64(event->auxtrace.size); event->auxtrace.offset = bswap_64(event->auxtrace.offset); @@ -592,10 +846,11 @@ static void perf_event__auxtrace_swap(union perf_event *event, event->auxtrace.idx = bswap_32(event->auxtrace.idx); event->auxtrace.tid = bswap_32(event->auxtrace.tid); event->auxtrace.cpu = bswap_32(event->auxtrace.cpu); + return 0; } -static void perf_event__auxtrace_error_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_error_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->auxtrace_error.type = bswap_32(event->auxtrace_error.type); event->auxtrace_error.code = bswap_32(event->auxtrace_error.code); @@ -607,52 +862,128 @@ static void perf_event__auxtrace_error_swap(union perf_event *event, if (event->auxtrace_error.fmt) event->auxtrace_error.time = bswap_64(event->auxtrace_error.time); if (event->auxtrace_error.fmt >= 2) { - event->auxtrace_error.machine_pid = bswap_32(event->auxtrace_error.machine_pid); - event->auxtrace_error.vcpu = bswap_32(event->auxtrace_error.vcpu); + /* + * fmt >= 2 adds machine_pid and vcpu after msg[64]. + * Older files may have fmt >= 2 but an event size + * that doesn't include these fields — downgrade to + * avoid swapping out of bounds. + */ + if (event->header.size < offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu)) { + pr_warning("WARNING: PERF_RECORD_AUXTRACE_ERROR: fmt %u but event too small for machine_pid/vcpu (%u bytes), downgrading fmt\n", + event->auxtrace_error.fmt, + event->header.size); + event->auxtrace_error.fmt = 1; + } else { + event->auxtrace_error.machine_pid = bswap_32(event->auxtrace_error.machine_pid); + event->auxtrace_error.vcpu = bswap_32(event->auxtrace_error.vcpu); + } } + return 0; } -static void perf_event__thread_map_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__thread_map_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { - unsigned i; + unsigned int i; + u64 nr; event->thread_map.nr = bswap_64(event->thread_map.nr); - for (i = 0; i < event->thread_map.nr; i++) + /* + * Reject rather than clamp: unlike namespaces (indexed by type) + * or stat_config (self-describing tags), a truncated thread map + * is structurally broken — downstream would get a wrong map. + */ + /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */ + nr = event->thread_map.nr; + if (nr > (event->header.size - sizeof(event->thread_map)) / + sizeof(event->thread_map.entries[0])) + return -1; + + for (i = 0; i < nr; i++) event->thread_map.entries[i].pid = bswap_64(event->thread_map.entries[i].pid); + return 0; } -static void perf_event__cpu_map_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__cpu_map_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { struct perf_record_cpu_map_data *data = &event->cpu_map.data; + u32 payload = event->header.size - sizeof(event->header); data->type = bswap_16(data->type); + /* + * Safe to clamp: a shorter CPU map just means some CPUs + * are absent; tools process the CPUs that are present. + */ switch (data->type) { - case PERF_CPU_MAP__CPUS: - data->cpus_data.nr = bswap_16(data->cpus_data.nr); + case PERF_CPU_MAP__CPUS: { + u16 nr, max_nr; - for (unsigned i = 0; i < data->cpus_data.nr; i++) + data->cpus_data.nr = bswap_16(data->cpus_data.nr); + nr = data->cpus_data.nr; + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->cpus_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]); break; + } case PERF_CPU_MAP__MASK: data->mask32_data.long_size = bswap_16(data->mask32_data.long_size); switch (data->mask32_data.long_size) { - case 4: + case 4: { + u16 nr, max_nr; + data->mask32_data.nr = bswap_16(data->mask32_data.nr); - for (unsigned i = 0; i < data->mask32_data.nr; i++) + nr = data->mask32_data.nr; + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->mask32_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]); break; - case 8: + } + case 8: { + u16 nr, max_nr; + data->mask64_data.nr = bswap_16(data->mask64_data.nr); - for (unsigned i = 0; i < data->mask64_data.nr; i++) + nr = data->mask64_data.nr; + if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + data->mask64_data.nr = 0; + break; + } + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->mask64_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]); break; + } default: - pr_err("cpu_map swap: unsupported long size\n"); + pr_err("cpu_map swap: unsupported long size %u\n", + data->mask32_data.long_size); } break; case PERF_CPU_MAP__RANGE_CPUS: @@ -662,20 +993,38 @@ static void perf_event__cpu_map_swap(union perf_event *event, default: break; } + return 0; } -static void perf_event__stat_config_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_config_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { - u64 size; + u64 nr, max_nr, size; - size = bswap_64(event->stat_config.nr) * sizeof(event->stat_config.data[0]); - size += 1; /* nr item itself */ + nr = bswap_64(event->stat_config.nr); + /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */ + max_nr = (event->header.size - sizeof(event->stat_config)) / + sizeof(event->stat_config.data[0]); + /* + * Safe to clamp: each config entry is self-describing + * via its tag; missing entries keep their defaults. + */ + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n", + nr, max_nr); + nr = max_nr; + } + size = nr * sizeof(event->stat_config.data[0]); + /* The swap starts at &nr, so add its size to cover the full range */ + size += sizeof(event->stat_config.nr); mem_bswap_64(&event->stat_config.nr, size); + /* Persist the clamped value in native byte order */ + event->stat_config.nr = nr; + return 0; } -static void perf_event__stat_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->stat.id = bswap_64(event->stat.id); event->stat.thread = bswap_32(event->stat.thread); @@ -683,44 +1032,140 @@ static void perf_event__stat_swap(union perf_event *event, event->stat.val = bswap_64(event->stat.val); event->stat.ena = bswap_64(event->stat.ena); event->stat.run = bswap_64(event->stat.run); + return 0; } -static void perf_event__stat_round_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_round_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->stat_round.type = bswap_64(event->stat_round.type); event->stat_round.time = bswap_64(event->stat_round.time); + return 0; } -static void perf_event__time_conv_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__time_conv_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->time_conv.time_shift = bswap_64(event->time_conv.time_shift); event->time_conv.time_mult = bswap_64(event->time_conv.time_mult); event->time_conv.time_zero = bswap_64(event->time_conv.time_zero); - if (event_contains(event->time_conv, time_cycles)) { + if (event_contains(event->time_conv, time_cycles)) event->time_conv.time_cycles = bswap_64(event->time_conv.time_cycles); + if (event_contains(event->time_conv, time_mask)) event->time_conv.time_mask = bswap_64(event->time_conv.time_mask); - } + return 0; } -static void +static int perf_event__compressed2_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + /* Only data_size needs swapping — compressed payload is a raw byte stream */ + event->pack2.data_size = bswap_64(event->pack2.data_size); + return 0; +} + +static int perf_event__bpf_metadata_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + u64 i, nr, max_nr; + + /* Fixed header must fit before accessing nr_entries or prog_name */ + if (event->header.size < sizeof(event->bpf_metadata)) + return -1; + + event->bpf_metadata.nr_entries = bswap_64(event->bpf_metadata.nr_entries); + + /* + * Ensure NUL-termination on the cross-endian path where the + * mapping is writable (MAP_PRIVATE + PROT_WRITE). Fixing + * the string in place is preferred over rejecting because it + * preserves the event for downstream processing — only the + * last byte is lost. + * + * The native-endian path (MAP_SHARED + PROT_READ) cannot + * write, so it validates and skips unterminated events in + * perf_session__process_user_event() instead. The two + * strategies produce different outcomes for the same + * malformed input (fix vs skip), which is inherent in the + * writable-vs-read-only mapping model. + */ + event->bpf_metadata.prog_name[BPF_PROG_NAME_LEN - 1] = '\0'; + + nr = event->bpf_metadata.nr_entries; + max_nr = (event->header.size - sizeof(event->bpf_metadata)) / + sizeof(event->bpf_metadata.entries[0]); + if (nr > max_nr) { + /* Persist clamped value so the native path processes entries, not skips */ + nr = max_nr; + event->bpf_metadata.nr_entries = nr; + } + + for (i = 0; i < nr; i++) { + event->bpf_metadata.entries[i].key[BPF_METADATA_KEY_LEN - 1] = '\0'; + event->bpf_metadata.entries[i].value[BPF_METADATA_VALUE_LEN - 1] = '\0'; + } + return 0; +} +static int perf_event__schedstat_cpu_swap(union perf_event *event __maybe_unused, bool sample_id_all __maybe_unused) { /* FIXME */ + return 0; } -static void +static int perf_event__schedstat_domain_swap(union perf_event *event __maybe_unused, bool sample_id_all __maybe_unused) { /* FIXME */ + return 0; } -typedef void (*perf_event__swap_op)(union perf_event *event, - bool sample_id_all); +static int perf_event__ksymbol_swap(union perf_event *event, + bool sample_id_all) +{ + event->ksymbol.addr = bswap_64(event->ksymbol.addr); + event->ksymbol.len = bswap_32(event->ksymbol.len); + event->ksymbol.ksym_type = bswap_16(event->ksymbol.ksym_type); + event->ksymbol.flags = bswap_16(event->ksymbol.flags); + + if (sample_id_all) { + void *data = &event->ksymbol.name; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); + + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); + swap_sample_id_all(event, data); + } + return 0; +} + +static int perf_event__bpf_event_swap(union perf_event *event, + bool sample_id_all) +{ + event->bpf.type = bswap_16(event->bpf.type); + event->bpf.flags = bswap_16(event->bpf.flags); + event->bpf.id = bswap_32(event->bpf.id); + + if (sample_id_all) + swap_sample_id_all(event, &event->bpf + 1); + return 0; +} + +static int perf_event__header_feature_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + event->feat.feat_id = bswap_64(event->feat.feat_id); + return 0; +} + +typedef int (*perf_event__swap_op)(union perf_event *event, + bool sample_id_all); static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_MMAP] = perf_event__mmap_swap, @@ -740,13 +1185,16 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_SWITCH_CPU_WIDE] = perf_event__switch_swap, [PERF_RECORD_NAMESPACES] = perf_event__namespaces_swap, [PERF_RECORD_CGROUP] = perf_event__cgroup_swap, + [PERF_RECORD_KSYMBOL] = perf_event__ksymbol_swap, + [PERF_RECORD_BPF_EVENT] = perf_event__bpf_event_swap, [PERF_RECORD_TEXT_POKE] = perf_event__text_poke_swap, [PERF_RECORD_AUX_OUTPUT_HW_ID] = perf_event__all64_swap, [PERF_RECORD_CALLCHAIN_DEFERRED] = perf_event__all64_swap, [PERF_RECORD_HEADER_ATTR] = perf_event__hdr_attr_swap, [PERF_RECORD_HEADER_EVENT_TYPE] = perf_event__event_type_swap, [PERF_RECORD_HEADER_TRACING_DATA] = perf_event__tracing_data_swap, - [PERF_RECORD_HEADER_BUILD_ID] = NULL, + [PERF_RECORD_HEADER_BUILD_ID] = perf_event__build_id_swap, + [PERF_RECORD_HEADER_FEATURE] = perf_event__header_feature_swap, [PERF_RECORD_ID_INDEX] = perf_event__all64_swap, [PERF_RECORD_AUXTRACE_INFO] = perf_event__auxtrace_info_swap, [PERF_RECORD_AUXTRACE] = perf_event__auxtrace_swap, @@ -758,6 +1206,8 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_STAT_ROUND] = perf_event__stat_round_swap, [PERF_RECORD_EVENT_UPDATE] = perf_event__event_update_swap, [PERF_RECORD_TIME_CONV] = perf_event__time_conv_swap, + [PERF_RECORD_COMPRESSED2] = perf_event__compressed2_swap, + [PERF_RECORD_BPF_METADATA] = perf_event__bpf_metadata_swap, [PERF_RECORD_SCHEDSTAT_CPU] = perf_event__schedstat_cpu_swap, [PERF_RECORD_SCHEDSTAT_DOMAIN] = perf_event__schedstat_domain_swap, [PERF_RECORD_HEADER_MAX] = NULL, @@ -1099,8 +1549,8 @@ static void dump_event(struct evlist *evlist, union perf_event *event, file_offset, file_path, event->header.size, event->header.type); trace_event(event); - if (event->header.type == PERF_RECORD_SAMPLE && evlist->trace_event_sample_raw) - evlist->trace_event_sample_raw(evlist, event, sample); + if (event->header.type == PERF_RECORD_SAMPLE && evlist__trace_event_sample_raw(evlist)) + evlist__trace_event_sample_raw(evlist)(evlist, event, sample); if (sample) evlist__print_tstamp(evlist, event, sample); @@ -1117,9 +1567,10 @@ char *get_page_size_name(u64 size, char *str) return str; } -static void dump_sample(struct machine *machine, struct evsel *evsel, union perf_event *event, +static void dump_sample(struct machine *machine, union perf_event *event, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; u64 sample_type; char str[PAGE_SIZE_NAME_LEN]; uint16_t e_machine = EM_NONE; @@ -1183,9 +1634,10 @@ static void dump_sample(struct machine *machine, struct evsel *evsel, union perf sample_read__printf(sample, evsel->core.attr.read_format); } -static void dump_deferred_callchain(struct evsel *evsel, union perf_event *event, - struct perf_sample *sample) +static void dump_deferred_callchain(union perf_event *event, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; + if (!dump_trace) return; @@ -1198,8 +1650,9 @@ static void dump_deferred_callchain(struct evsel *evsel, union perf_event *event static void dump_read(struct evsel *evsel, union perf_event *event) { - struct perf_record_read *read_event = &event->read; u64 read_format; + __u64 *array; + void *end; if (!dump_trace) return; @@ -1211,18 +1664,37 @@ static void dump_read(struct evsel *evsel, union perf_event *event) return; read_format = evsel->core.attr.read_format; + /* + * The kernel packs only the enabled read_format fields + * after value, with no gaps. Walk the packed array + * instead of using fixed struct offsets. + */ + array = &event->read.value + 1; + end = (void *)event + event->header.size; - if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) - printf("... time enabled : %" PRI_lu64 "\n", read_event->time_enabled); + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + if ((void *)(array + 1) > end) + return; + printf("... time enabled : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) - printf("... time running : %" PRI_lu64 "\n", read_event->time_running); + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + if ((void *)(array + 1) > end) + return; + printf("... time running : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_ID) - printf("... id : %" PRI_lu64 "\n", read_event->id); + if (read_format & PERF_FORMAT_ID) { + if ((void *)(array + 1) > end) + return; + printf("... id : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_LOST) - printf("... lost : %" PRI_lu64 "\n", read_event->lost); + if (read_format & PERF_FORMAT_LOST) { + if ((void *)(array + 1) > end) + return; + printf("... lost : %" PRI_lu64 "\n", *array++); + } } static struct machine *machines__find_for_cpumode(struct machines *machines, @@ -1279,7 +1751,7 @@ static int deliver_sample_value(struct evlist *evlist, } if (!storage || sid->evsel == NULL) { - ++evlist->stats.nr_unknown_id; + ++evlist__stats(evlist)->nr_unknown_id; return 0; } @@ -1291,7 +1763,7 @@ static int deliver_sample_value(struct evlist *evlist, return 0; sample->evsel = container_of(sid->evsel, struct evsel, core); - ret = tool->sample(tool, event, sample, sample->evsel, machine); + ret = tool->sample(tool, event, sample, machine); sample->evsel = saved_evsel; return ret; } @@ -1323,8 +1795,9 @@ static int deliver_sample_group(struct evlist *evlist, static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) + struct machine *machine) { + struct evsel *evsel = sample->evsel; /* We know evsel != NULL. */ u64 sample_type = evsel->core.attr.sample_type; u64 read_format = evsel->core.attr.read_format; @@ -1332,7 +1805,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool /* Standard sample delivery. */ if (!(sample_type & PERF_SAMPLE_READ)) - return tool->sample(tool, event, sample, evsel, machine); + return tool->sample(tool, event, sample, machine); /* For PERF_SAMPLE_READ we have either single or group mode. */ if (read_format & PERF_FORMAT_GROUP) @@ -1352,6 +1825,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool struct deferred_event { struct list_head list; union perf_event *event; + u64 file_offset; }; /* @@ -1371,14 +1845,17 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, struct evsel *saved_evsel = sample->evsel; sample->evsel = evlist__id2evsel(evlist, sample->id); - ret = tool->callchain_deferred(tool, event, sample, - sample->evsel, machine); + if (sample->evsel) + sample->evsel = evsel__get(sample->evsel); + ret = tool->callchain_deferred(tool, event, sample, machine); + evsel__put(sample->evsel); sample->evsel = saved_evsel; return ret; } - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample orig_sample; + struct evsel *new_evsel; perf_sample__init(&orig_sample, /*all=*/false); ret = evlist__parse_sample(evlist, de->event, &orig_sample); @@ -1387,6 +1864,7 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, perf_sample__exit(&orig_sample); break; } + orig_sample.file_offset = de->file_offset; if (sample->tid != orig_sample.tid) { perf_sample__exit(&orig_sample); @@ -1398,9 +1876,13 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, else orig_sample.deferred_callchain = false; - orig_sample.evsel = evlist__id2evsel(evlist, orig_sample.id); + new_evsel = evlist__id2evsel(evlist, orig_sample.id); + if (new_evsel != orig_sample.evsel) { + evsel__put(orig_sample.evsel); + orig_sample.evsel = evsel__get(new_evsel); + } ret = evlist__deliver_sample(evlist, tool, de->event, - &orig_sample, orig_sample.evsel, machine); + &orig_sample, machine); perf_sample__exit(&orig_sample); list_del(&de->list); @@ -1425,8 +1907,9 @@ static int session__flush_deferred_samples(struct perf_session *session, struct deferred_event *de, *tmp; int ret = 0; - list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) { + list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) { struct perf_sample sample; + struct evsel *new_evsel; perf_sample__init(&sample, /*all=*/false); ret = evlist__parse_sample(evlist, de->event, &sample); @@ -1435,10 +1918,15 @@ static int session__flush_deferred_samples(struct perf_session *session, perf_sample__exit(&sample); break; } + sample.file_offset = de->file_offset; - sample.evsel = evlist__id2evsel(evlist, sample.id); + new_evsel = evlist__id2evsel(evlist, sample.id); + if (new_evsel != sample.evsel) { + evsel__put(sample.evsel); + sample.evsel = evsel__get(new_evsel); + } ret = evlist__deliver_sample(evlist, tool, de->event, - &sample, sample.evsel, machine); + &sample, machine); perf_sample__exit(&sample); list_del(&de->list); @@ -1451,6 +1939,26 @@ static int session__flush_deferred_samples(struct perf_session *session, return ret; } +/* + * Return true if the string field is properly null-terminated + * within the event boundary. Native-endian files are mapped + * read-only (MAP_SHARED + PROT_READ) so we cannot write a + * null byte in place; skip the event instead. + */ +static bool perf_event__check_nul(const char *str, const void *end, + const char *event_name, u64 file_offset) +{ + size_t max_len = (const char *)end - str; + + if (max_len == 0 || strnlen(str, max_len) == max_len) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_%s: string not null-terminated, skipping event\n", + file_offset, event_name); + return false; + } + + return true; +} + static int machines__deliver_event(struct machines *machines, struct evlist *evlist, union perf_event *event, @@ -1458,31 +1966,31 @@ static int machines__deliver_event(struct machines *machines, const struct perf_tool *tool, u64 file_offset, const char *file_path) { - struct evsel *evsel; struct machine *machine; dump_event(evlist, event, file_offset, sample, file_path); - if (!sample->evsel) + if (!sample->evsel) { sample->evsel = evlist__id2evsel(evlist, sample->id); + if (sample->evsel) + sample->evsel = evsel__get(sample->evsel); + } else assert(sample->evsel == evlist__id2evsel(evlist, sample->id)); - - evsel = sample->evsel; machine = machines__find_for_cpumode(machines, event, sample); switch (event->header.type) { case PERF_RECORD_SAMPLE: - if (evsel == NULL) { - ++evlist->stats.nr_unknown_id; + if (sample->evsel == NULL) { + ++evlist__stats(evlist)->nr_unknown_id; return 0; } if (machine == NULL) { - ++evlist->stats.nr_unprocessable_samples; - dump_sample(machine, evsel, event, sample); + ++evlist__stats(evlist)->nr_unprocessable_samples; + dump_sample(machine, event, sample); return 0; } - dump_sample(machine, evsel, event, sample); + dump_sample(machine, event, sample); if (sample->deferred_callchain && tool->merge_deferred_callchains) { struct deferred_event *de = malloc(sizeof(*de)); size_t sz = event->header.size; @@ -1496,21 +2004,57 @@ static int machines__deliver_event(struct machines *machines, return -ENOMEM; } memcpy(de->event, event, sz); - list_add_tail(&de->list, &evlist->deferred_samples); + de->file_offset = sample->file_offset; + list_add_tail(&de->list, evlist__deferred_samples(evlist)); return 0; } - return evlist__deliver_sample(evlist, tool, event, sample, evsel, machine); + return evlist__deliver_sample(evlist, tool, event, sample, machine); case PERF_RECORD_MMAP: + if (!perf_event__check_nul(event->mmap.filename, + (void *)event + event->header.size, + "MMAP", file_offset)) + return 0; return tool->mmap(tool, event, sample, machine); case PERF_RECORD_MMAP2: if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT) - ++evlist->stats.nr_proc_map_timeout; + ++evlist__stats(evlist)->nr_proc_map_timeout; + if (!perf_event__check_nul(event->mmap2.filename, + (void *)event + event->header.size, + "MMAP2", file_offset)) + return 0; return tool->mmap2(tool, event, sample, machine); case PERF_RECORD_COMM: + if (!perf_event__check_nul(event->comm.comm, + (void *)event + event->header.size, + "COMM", file_offset)) + return 0; return tool->comm(tool, event, sample, machine); - case PERF_RECORD_NAMESPACES: + case PERF_RECORD_NAMESPACES: { + /* + * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof. + * Includes trailing sample_id space when present, but prevents OOB. + */ + u64 max_nr = (event->header.size - sizeof(event->namespaces)) / + sizeof(event->namespaces.link_info[0]); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr in place. Skip the event instead. + * The swap handler already clamps on the writable + * cross-endian path. + */ + if (event->namespaces.nr_namespaces > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n", + file_offset, (u64)event->namespaces.nr_namespaces, max_nr); + return 0; + } return tool->namespaces(tool, event, sample, machine); + } case PERF_RECORD_CGROUP: + if (!perf_event__check_nul(event->cgroup.path, + (void *)event + event->header.size, + "CGROUP", file_offset)) + return 0; return tool->cgroup(tool, event, sample, machine); case PERF_RECORD_FORK: return tool->fork(tool, event, sample, machine); @@ -1518,17 +2062,17 @@ static int machines__deliver_event(struct machines *machines, return tool->exit(tool, event, sample, machine); case PERF_RECORD_LOST: if (tool->lost == perf_event__process_lost) - evlist->stats.total_lost += event->lost.lost; + evlist__stats(evlist)->total_lost += event->lost.lost; return tool->lost(tool, event, sample, machine); case PERF_RECORD_LOST_SAMPLES: if (event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF) - evlist->stats.total_dropped_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_dropped_samples += event->lost_samples.lost; else if (tool->lost_samples == perf_event__process_lost_samples) - evlist->stats.total_lost_samples += event->lost_samples.lost; + evlist__stats(evlist)->total_lost_samples += event->lost_samples.lost; return tool->lost_samples(tool, event, sample, machine); case PERF_RECORD_READ: - dump_read(evsel, event); - return tool->read(tool, event, sample, evsel, machine); + dump_read(sample->evsel, event); + return tool->read(tool, event, sample, machine); case PERF_RECORD_THROTTLE: return tool->throttle(tool, event, sample, machine); case PERF_RECORD_UNTHROTTLE: @@ -1536,11 +2080,11 @@ static int machines__deliver_event(struct machines *machines, case PERF_RECORD_AUX: if (tool->aux == perf_event__process_aux) { if (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) - evlist->stats.total_aux_lost += 1; + evlist__stats(evlist)->total_aux_lost += 1; if (event->aux.flags & PERF_AUX_FLAG_PARTIAL) - evlist->stats.total_aux_partial += 1; + evlist__stats(evlist)->total_aux_partial += 1; if (event->aux.flags & PERF_AUX_FLAG_COLLISION) - evlist->stats.total_aux_collision += 1; + evlist__stats(evlist)->total_aux_collision += 1; } return tool->aux(tool, event, sample, machine); case PERF_RECORD_ITRACE_START: @@ -1549,19 +2093,34 @@ static int machines__deliver_event(struct machines *machines, case PERF_RECORD_SWITCH_CPU_WIDE: return tool->context_switch(tool, event, sample, machine); case PERF_RECORD_KSYMBOL: + if (!perf_event__check_nul(event->ksymbol.name, + (void *)event + event->header.size, + "KSYMBOL", file_offset)) + return 0; return tool->ksymbol(tool, event, sample, machine); case PERF_RECORD_BPF_EVENT: return tool->bpf(tool, event, sample, machine); - case PERF_RECORD_TEXT_POKE: + case PERF_RECORD_TEXT_POKE: { + /* offsetof(bytes), not sizeof — sizeof includes padding past the flexible array */ + size_t text_poke_len = offsetof(struct perf_record_text_poke_event, bytes) + + event->text_poke.old_len + + event->text_poke.new_len; + + if (event->header.size < text_poke_len) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_TEXT_POKE: old_len+new_len exceeds event, skipping\n", + file_offset); + return 0; + } return tool->text_poke(tool, event, sample, machine); + } case PERF_RECORD_AUX_OUTPUT_HW_ID: return tool->aux_output_hw_id(tool, event, sample, machine); case PERF_RECORD_CALLCHAIN_DEFERRED: - dump_deferred_callchain(evsel, event, sample); + dump_deferred_callchain(event, sample); return evlist__deliver_deferred_callchain(evlist, tool, event, sample, machine); default: - ++evlist->stats.nr_unknown_events; + ++evlist__stats(evlist)->nr_unknown_events; return -1; } } @@ -1573,14 +2132,104 @@ static int perf_session__deliver_event(struct perf_session *session, const char *file_path) { struct perf_sample sample; + struct evsel *evsel; int ret; perf_sample__init(&sample, /*all=*/false); - ret = evlist__parse_sample(session->evlist, event, &sample); + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) { + pr_err("ERROR: at offset %#" PRIx64 ": no evsel found for %s (%u) event\n", + file_offset, perf_event__name(event->header.type), + event->header.type); + ret = -EFAULT; + goto out; + } + ret = evsel__parse_sample(evsel, event, &sample); if (ret) { - pr_err("Can't parse sample, err = %d\n", ret); + pr_err("ERROR: at offset %#" PRIx64 ": can't parse %s (%u) sample, err = %d\n", + file_offset, perf_event__name(event->header.type), + event->header.type, ret); goto out; } + sample.file_offset = file_offset; + /* + * evsel__parse_sample() doesn't populate machine_pid/vcpu, + * which are needed by machines__find_for_cpumode() to + * attribute samples to guest VMs. The SID table maps + * sample IDs to the guest that owns the event. + */ + if (perf_guest && sample.id) { + struct perf_sample_id *sid = evlist__id2sid(session->evlist, sample.id); + + if (sid) { + sample.machine_pid = sid->machine_pid; + sample.vcpu = sid->vcpu.cpu; + } + } + + /* + * Validate sample.cpu before any callback can use it as an + * array index (kwork cpus_runtime, timechart cpus_cstate_*, + * sched cpu_last_switched). + * + * When PERF_SAMPLE_CPU is absent, evsel__parse_sample() leaves + * sample.cpu as (u32)-1 — a sentinel that downstream tools + * (script, inject) check to identify events without CPU info. + * Only check when sample.cpu was actually populated from event + * data: PERF_RECORD_SAMPLE always has it when PERF_SAMPLE_CPU + * is set; non-sample events only have it when sample_id_all is + * enabled. Otherwise sample.cpu is the (u32)-1 sentinel from + * evsel__parse_sample() and must not be validated or clamped. + */ + if ((evsel->core.attr.sample_type & PERF_SAMPLE_CPU) && + (event->header.type == PERF_RECORD_SAMPLE || + evsel->core.attr.sample_id_all)) { + int nr_cpus_avail = perf_session__env(session)->nr_cpus_avail; + + /* + * For perf.data files the MAX_NR_CPUS fallback in + * perf_session__read_header() guarantees this is set. + * For pipe mode, HEADER_NRCPUS may arrive late or not + * at all (pre-2017 perf, third-party tools). Fall + * back to MAX_NR_CPUS so the bounds check still works + * against fixed-size downstream arrays. + * + * Do NOT write back to env: this function runs during + * recording (synthesized events) when nr_cpus_avail is + * legitimately 0. Writing MAX_NR_CPUS would cause + * write_cpu_topology() to emit 4096 core_id/socket_id + * pairs instead of the real CPU count, corrupting the + * topology section in the generated perf.data. + */ + if (nr_cpus_avail <= 0) + nr_cpus_avail = MAX_NR_CPUS; + /* + * Cap at MAX_NR_CPUS for the bounds check — downstream + * consumers use fixed-size arrays of that size. Keep + * the true nr_cpus_avail in env for header parsing + * (e.g. process_cpu_topology) which needs the real count. + */ + if (nr_cpus_avail > MAX_NR_CPUS) + nr_cpus_avail = MAX_NR_CPUS; + if (sample.cpu >= (u32)nr_cpus_avail && + sample.cpu != (u32)-1) { + /* + * Warn rather than abort: synthesized events + * (MMAP, COMM) lack sample_id_all data, so + * parse_id_sample reads garbage from the event + * payload. Clamping to 0 protects downstream + * array indexing while keeping the session alive. + * + * Preserve (u32)-1: perf script and perf inject + * use it as a sentinel for "CPU not applicable." + * Downstream array users (timechart, kwork) have + * their own per-callback bounds checks. + */ + pr_warning_once("WARNING: at offset %#" PRIx64 ": sample CPU %u >= nr_cpus_avail %u, clamping to 0\n", + file_offset, sample.cpu, nr_cpus_avail); + sample.cpu = 0; + } + } ret = auxtrace__process_event(session, event, &sample, tool); if (ret < 0) @@ -1607,6 +2256,7 @@ static s64 perf_session__process_user_event(struct perf_session *session, { struct ordered_events *oe = &session->ordered_events; const struct perf_tool *tool = session->tool; + const u32 event_size = READ_ONCE(event->header.size); struct perf_sample sample; int fd = perf_data__fd(session->data); s64 err; @@ -1647,6 +2297,12 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = tool->tracing_data(tool, session, event); break; case PERF_RECORD_HEADER_BUILD_ID: + if (!perf_event__check_nul(event->build_id.filename, + (void *)event + event_size, + "HEADER_BUILD_ID", file_offset)) { + err = 0; + break; + } err = tool->build_id(tool, session, event); break; case PERF_RECORD_FINISHED_ROUND: @@ -1665,22 +2321,119 @@ static s64 perf_session__process_user_event(struct perf_session *session, * place already. */ if (!perf_data__is_pipe(session->data)) - lseek(fd, file_offset + event->header.size, SEEK_SET); + lseek(fd, file_offset + event_size, SEEK_SET); err = tool->auxtrace(tool, session, event); break; case PERF_RECORD_AUXTRACE_ERROR: perf_session__auxtrace_error_inc(session, event); err = tool->auxtrace_error(tool, session, event); break; - case PERF_RECORD_THREAD_MAP: + case PERF_RECORD_THREAD_MAP: { + u64 max_nr; + + if (event_size < sizeof(event->thread_map)) { + pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: header.size (%u) too small\n", + file_offset, event_size); + err = -EINVAL; + break; + } + + max_nr = (event_size - sizeof(event->thread_map)) / + sizeof(event->thread_map.entries[0]); + if (event->thread_map.nr > max_nr) { + pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: nr %" PRIu64 " exceeds max %" PRIu64 "\n", + file_offset, (u64)event->thread_map.nr, max_nr); + err = -EINVAL; + break; + } + err = tool->thread_map(tool, session, event); break; - case PERF_RECORD_CPU_MAP: + } + case PERF_RECORD_CPU_MAP: { + struct perf_record_cpu_map_data *data = &event->cpu_map.data; + u32 payload = event_size - sizeof(event->header); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr fields in place. Skip the event + * if any variant overflows. + */ + switch (data->type) { + case PERF_CPU_MAP__CPUS: { + u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + + if (data->cpus_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->cpus_data.nr, max_nr); + err = 0; + goto out; + } + break; + } + case PERF_CPU_MAP__MASK: + if (data->mask32_data.long_size == 4) { + u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + + if (data->mask32_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->mask32_data.nr, max_nr); + err = 0; + goto out; + } + } else if (data->mask64_data.long_size == 8) { + u16 max_nr; + + if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + err = 0; + goto out; + } + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (data->mask64_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->mask64_data.nr, max_nr); + err = 0; + goto out; + } + } else { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: unsupported long_size %u, skipping\n", + file_offset, data->mask32_data.long_size); + err = 0; + goto out; + } + break; + default: + break; + } + err = tool->cpu_map(tool, session, event); break; - case PERF_RECORD_STAT_CONFIG: + } + case PERF_RECORD_STAT_CONFIG: { + /* Cannot underflow: perf_event__min_size[] guarantees event_size >= sizeof */ + u64 max_nr = (event_size - sizeof(event->stat_config)) / + sizeof(event->stat_config.data[0]); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr in place. Skip the event instead. + */ + if (event->stat_config.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n", + file_offset, (u64)event->stat_config.nr, max_nr); + err = 0; + goto out; + } + err = tool->stat_config(tool, session, event); break; + } case PERF_RECORD_STAT: err = tool->stat(tool, session, event); break; @@ -1688,7 +2441,14 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = tool->stat_round(tool, session, event); break; case PERF_RECORD_TIME_CONV: - session->time_conv = event->time_conv; + /* + * Bounded copy: older kernels emit a shorter struct + * without time_cycles/time_mask/cap_user_time_*. + * Zero the rest so extended fields default to off. + */ + memset(&session->time_conv, 0, sizeof(session->time_conv)); + memcpy(&session->time_conv, &event->time_conv, + min((size_t)event_size, sizeof(session->time_conv))); err = tool->time_conv(tool, session, event); break; case PERF_RECORD_HEADER_FEATURE: @@ -1703,9 +2463,53 @@ static s64 perf_session__process_user_event(struct perf_session *session, case PERF_RECORD_FINISHED_INIT: err = tool->finished_init(tool, session, event); break; - case PERF_RECORD_BPF_METADATA: + case PERF_RECORD_BPF_METADATA: { + u64 nr_entries, max_entries; + + if (event_size < sizeof(event->bpf_metadata)) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: header.size (%u) too small, skipping\n", + file_offset, event_size); + err = 0; + break; + } + + /* + * Native-endian files are mmap'd read-only — validate + * NUL-termination instead of writing. + */ + if (strnlen(event->bpf_metadata.prog_name, + BPF_PROG_NAME_LEN) == BPF_PROG_NAME_LEN) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: prog_name not null-terminated, skipping\n", + file_offset); + err = 0; + break; + } + + nr_entries = READ_ONCE(event->bpf_metadata.nr_entries); + max_entries = (event_size - sizeof(event->bpf_metadata)) / + sizeof(event->bpf_metadata.entries[0]); + if (nr_entries > max_entries) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: nr_entries %" PRIu64 " exceeds max %" PRIu64 ", skipping\n", + file_offset, nr_entries, max_entries); + err = 0; + break; + } + + for (u64 i = 0; i < nr_entries; i++) { + if (strnlen(event->bpf_metadata.entries[i].key, + BPF_METADATA_KEY_LEN) == BPF_METADATA_KEY_LEN || + strnlen(event->bpf_metadata.entries[i].value, + BPF_METADATA_VALUE_LEN) == BPF_METADATA_VALUE_LEN) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: entry %" PRIu64 " key/value not null-terminated, skipping\n", + file_offset, i); + err = 0; + goto out; + } + } + err = tool->bpf_metadata(tool, session, event); break; + } case PERF_RECORD_SCHEDSTAT_CPU: err = tool->schedstat_cpu(tool, session, event); break; @@ -1716,6 +2520,7 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = -EINVAL; break; } +out: perf_sample__exit(&sample); return err; } @@ -1727,7 +2532,7 @@ int perf_session__deliver_synth_event(struct perf_session *session, struct evlist *evlist = session->evlist; const struct perf_tool *tool = session->tool; - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, 0, NULL); @@ -1759,15 +2564,142 @@ int perf_session__deliver_synth_attr_event(struct perf_session *session, return perf_session__deliver_synth_event(session, &ev.ev, NULL); } -static void event_swap(union perf_event *event, bool sample_id_all) +/* Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX */ +static int event_swap(union perf_event *event, bool sample_id_all) { - perf_event__swap_op swap; + perf_event__swap_op swap = perf_event__swap_ops[event->header.type]; - swap = perf_event__swap_ops[event->header.type]; if (swap) - swap(event, sample_id_all); + return swap(event, sample_id_all); + return 0; +} + +/* + * Minimum event sizes indexed by type. Checked before swap and + * processing so that both cross-endian and native-endian paths + * are protected from accessing fields past the event boundary. + * Zero means no minimum beyond the 8-byte header (already + * enforced by the reader). + * + * These values represent the smallest event the kernel has ever + * emitted for each type, so they do not reject legitimate legacy + * perf.data files from older kernels. Variable-length events + * use offsetof() to the first variable field; the variable + * content is validated separately (e.g., perf_event__check_nul). + */ +static const u32 perf_event__min_size[PERF_RECORD_HEADER_MAX] = { + /* + * offsetof() + 1 for types with a trailing variable-length + * string (filename, comm, path, name, msg): the +1 ensures + * room for at least a null terminator. Full null-termination + * within the event boundary is checked separately. + * + * PERF_RECORD_SAMPLE is omitted: all64_swap is bounded by + * header.size, and the internal layout varies by sample_type + * so a fixed minimum is not meaningful. + */ + [PERF_RECORD_MMAP] = offsetof(struct perf_record_mmap, filename) + 1, + [PERF_RECORD_LOST] = sizeof(struct perf_record_lost), + [PERF_RECORD_COMM] = offsetof(struct perf_record_comm, comm) + 1, + [PERF_RECORD_EXIT] = sizeof(struct perf_record_fork), + [PERF_RECORD_THROTTLE] = sizeof(struct perf_record_throttle), + [PERF_RECORD_UNTHROTTLE] = sizeof(struct perf_record_throttle), + [PERF_RECORD_FORK] = sizeof(struct perf_record_fork), + /* + * The kernel dynamically sizes PERF_RECORD_READ based on + * attr.read_format — only the enabled fields are emitted, + * packed with no gaps. The minimum valid event has just + * pid + tid + one u64 value (no optional fields). + */ + [PERF_RECORD_READ] = offsetof(struct perf_record_read, time_enabled), + [PERF_RECORD_MMAP2] = offsetof(struct perf_record_mmap2, filename) + 1, + [PERF_RECORD_LOST_SAMPLES] = sizeof(struct perf_record_lost_samples), + [PERF_RECORD_AUX] = sizeof(struct perf_record_aux), + [PERF_RECORD_ITRACE_START] = sizeof(struct perf_record_itrace_start), + [PERF_RECORD_SWITCH] = sizeof(struct perf_event_header), + [PERF_RECORD_SWITCH_CPU_WIDE] = sizeof(struct perf_record_switch), + [PERF_RECORD_NAMESPACES] = sizeof(struct perf_record_namespaces), + [PERF_RECORD_CGROUP] = offsetof(struct perf_record_cgroup, path) + 1, + [PERF_RECORD_TEXT_POKE] = sizeof(struct perf_record_text_poke_event), + [PERF_RECORD_KSYMBOL] = offsetof(struct perf_record_ksymbol, name) + 1, + [PERF_RECORD_BPF_EVENT] = sizeof(struct perf_record_bpf_event), + [PERF_RECORD_HEADER_ATTR] = sizeof(struct perf_event_header) + PERF_ATTR_SIZE_VER0, + [PERF_RECORD_HEADER_EVENT_TYPE] = sizeof(struct perf_record_header_event_type), + /* Legacy events predate the __u32 pad field, accept 12-byte records */ + [PERF_RECORD_HEADER_TRACING_DATA] = offsetof(struct perf_record_header_tracing_data, pad), + [PERF_RECORD_AUX_OUTPUT_HW_ID] = sizeof(struct perf_record_aux_output_hw_id), + [PERF_RECORD_AUXTRACE_INFO] = sizeof(struct perf_record_auxtrace_info), + [PERF_RECORD_AUXTRACE] = sizeof(struct perf_record_auxtrace), + [PERF_RECORD_AUXTRACE_ERROR] = offsetof(struct perf_record_auxtrace_error, msg) + 1, + [PERF_RECORD_THREAD_MAP] = sizeof(struct perf_record_thread_map), + /* + * sizeof(perf_record_cpu_map) is 20 because the outer struct + * isn't packed and GCC adds 2 bytes of trailing padding. + * The smallest valid variant (RANGE_CPUS) is only 16 bytes: + * header(8) + type(2) + range_cpu_data(6). Per-variant + * bounds are checked in the swap handler via payload. + */ + [PERF_RECORD_CPU_MAP] = sizeof(struct perf_event_header) + + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map), + [PERF_RECORD_STAT_CONFIG] = sizeof(struct perf_record_stat_config), + [PERF_RECORD_STAT] = sizeof(struct perf_record_stat), + [PERF_RECORD_STAT_ROUND] = sizeof(struct perf_record_stat_round), + /* + * EVENT_UPDATE has a union whose largest member (cpus) + * inflates sizeof to 40, but SCALE events are only 32 + * and UNIT/NAME events can be even smaller. Use the + * fixed header fields (header + type + id) as minimum. + */ + [PERF_RECORD_EVENT_UPDATE] = offsetof(struct perf_record_event_update, scale), + [PERF_RECORD_TIME_CONV] = offsetof(struct perf_record_time_conv, time_cycles), + [PERF_RECORD_ID_INDEX] = sizeof(struct perf_record_id_index), + [PERF_RECORD_HEADER_BUILD_ID] = sizeof(struct perf_record_header_build_id), + [PERF_RECORD_HEADER_FEATURE] = sizeof(struct perf_record_header_feature), + [PERF_RECORD_COMPRESSED2] = sizeof(struct perf_record_compressed2), + [PERF_RECORD_BPF_METADATA] = sizeof(struct perf_record_bpf_metadata), + [PERF_RECORD_CALLCHAIN_DEFERRED] = sizeof(struct perf_event_header) + sizeof(__u64), + /* + * SCHEDSTAT events have a version-dependent union after the + * fixed header fields; the minimum is the base (pre-union) + * portion so old and new versions both pass. + */ + [PERF_RECORD_SCHEDSTAT_CPU] = offsetof(struct perf_record_schedstat_cpu, v15), + [PERF_RECORD_SCHEDSTAT_DOMAIN] = offsetof(struct perf_record_schedstat_domain, v15), +}; + +/* + * Return true if the event is too small for its declared type. + * Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX. + * If min is non-NULL, stores the required minimum on failure. + */ +bool perf_event__too_small(const union perf_event *event, u32 *min) +{ + u32 min_sz = perf_event__min_size[event->header.type]; + + if (min_sz && event->header.size < min_sz) { + if (min) + *min = min_sz; + return true; + } + + return false; } +/* + * Read and validate the event at @file_offset. + * + * Returns: + * 0 — success: *event_ptr is set and safe to access. + * -1 — error; check *event_ptr to decide whether to advance or abort: + * *event_ptr set — event header was read but the event is + * malformed (too small for its type, or byte-swap + * failed). header.size is still valid, so the + * caller can advance past the event. + * *event_ptr NULL — fatal: couldn't read the header at all + * (I/O error, offset out of range, pipe mode). + * Caller must abort. + */ int perf_session__peek_event(struct perf_session *session, off_t file_offset, void *buf, size_t buf_sz, union perf_event **event_ptr, @@ -1775,52 +2707,108 @@ int perf_session__peek_event(struct perf_session *session, off_t file_offset, { union perf_event *event; size_t hdr_sz, rest; + u32 min_sz; int fd; + *event_ptr = NULL; + if (session->one_mmap && !session->header.needs_swap) { - event = file_offset - session->one_mmap_offset + - session->one_mmap_addr; - goto out_parse_sample; - } + u64 offset_in_mmap; - if (perf_data__is_pipe(session->data)) - return -1; + /* Validate offset with integer arithmetic to avoid pointer UB */ + if ((u64)file_offset < session->one_mmap_offset) + return -1; - fd = perf_data__fd(session->data); - hdr_sz = sizeof(struct perf_event_header); + offset_in_mmap = (u64)file_offset - session->one_mmap_offset; - if (buf_sz < hdr_sz) - return -1; + /* Use subtraction to avoid addition overflow */ + if (offset_in_mmap >= session->one_mmap_size || + session->one_mmap_size - offset_in_mmap < sizeof(struct perf_event_header)) + return -1; - if (lseek(fd, file_offset, SEEK_SET) == (off_t)-1 || - readn(fd, buf, hdr_sz) != (ssize_t)hdr_sz) - return -1; + event = session->one_mmap_addr + offset_in_mmap; - event = (union perf_event *)buf; + if (event->header.size < sizeof(struct perf_event_header)) + return -1; - if (session->header.needs_swap) - perf_event_header__bswap(&event->header); + /* Ensure full event is within the mmap region */ + if (session->one_mmap_size - offset_in_mmap < event->header.size) + return -1; + } else { + if (perf_data__is_pipe(session->data)) + return -1; - if (event->header.size < hdr_sz || event->header.size > buf_sz) - return -1; + fd = perf_data__fd(session->data); + hdr_sz = sizeof(struct perf_event_header); - buf += hdr_sz; - rest = event->header.size - hdr_sz; + if (buf_sz < hdr_sz) + return -1; - if (readn(fd, buf, rest) != (ssize_t)rest) + if (lseek(fd, file_offset, SEEK_SET) == (off_t)-1 || + readn(fd, buf, hdr_sz) != (ssize_t)hdr_sz) + return -1; + + event = (union perf_event *)buf; + + if (session->header.needs_swap) + perf_event_header__bswap(&event->header); + + if (event->header.size < hdr_sz || event->header.size > buf_sz) + return -1; + + buf += hdr_sz; + rest = event->header.size - hdr_sz; + + if (readn(fd, buf, rest) != (ssize_t)rest) + return -1; + } + + /* Event data is fully loaded — expose so callers can advance */ + *event_ptr = event; + + /* + * Check alignment before type: an unaligned size misaligns the + * stream for all subsequent reads regardless of event type. + * Three legacy user events predate the 8-byte rule — exempt them. + */ + if (event->header.size % sizeof(u64) && + event->header.type != PERF_RECORD_HEADER_TRACING_DATA && + event->header.type != PERF_RECORD_COMPRESSED && + event->header.type != PERF_RECORD_HEADER_FEATURE) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u not aligned to %zu\n", + (u64)file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, sizeof(u64)); return -1; + } - if (session->header.needs_swap) - event_swap(event, evlist__sample_id_all(session->evlist)); + if (event->header.type >= PERF_RECORD_HEADER_MAX) { + pr_warning("WARNING: at offset %#" PRIx64 ": unsupported event type %u, skipping\n", + (u64)file_offset, event->header.type); + return 0; + } + + if (perf_event__too_small(event, &min_sz)) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u)\n", + (u64)file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, min_sz); + return -1; + } -out_parse_sample: + if (session->header.needs_swap && + event_swap(event, evlist__sample_id_all(session->evlist))) { + /* + * The header was already swapped so header.size is + * valid — expose the event so callers can advance + * past this malformed entry instead of aborting. + */ + *event_ptr = event; + return -1; + } if (sample && event->header.type < PERF_RECORD_USER_TYPE_START && evlist__parse_sample(session->evlist, event, sample)) return -1; - *event_ptr = event; - return 0; } @@ -1833,11 +2821,37 @@ int perf_session__peek_events(struct perf_session *session, u64 offset, int err; do { + event = NULL; err = perf_session__peek_event(session, offset, buf, PERF_SAMPLE_MAX_SIZE, &event, NULL); - if (err) - return err; + if (err) { + /* + * Recoverable error: peek_event returns -1 but + * sets event_ptr when the header was read + * successfully but the event is malformed (too + * small or swap failed). Skip past it using + * header.size — don't invoke the callback since + * type-specific fields may be truncated. + * + * Must abort if: event_ptr is NULL (I/O error), + * size is 0 (can't advance), type is AUXTRACE + * (payload extends beyond header.size), or size + * is unaligned (would misalign all subsequent reads). + * + * Direct callers (auxtrace, cs-etm) treat any + * non-zero return as fatal — only this loop skips. + */ + if (event && event->header.size && + event->header.type != PERF_RECORD_AUXTRACE && + event->header.size % sizeof(u64) == 0) { + offset += event->header.size; + err = 0; + } else { + return err; + } + continue; + } err = cb(session, event, offset, data); if (err) @@ -1858,24 +2872,77 @@ static s64 perf_session__process_event(struct perf_session *session, { struct evlist *evlist = session->evlist; const struct perf_tool *tool = session->tool; + u32 min_sz; int ret; - if (session->header.needs_swap) - event_swap(event, evlist__sample_id_all(evlist)); + /* + * The kernel aligns all event sizes to sizeof(u64) — see + * perf_event_comm_event() (ALIGN), perf_event_mmap_event(), + * perf_event_cgroup(), perf_event_ksymbol() (IS_ALIGNED loops), + * and perf_event_text_poke() (ALIGN) in kernel/events/core.c. + * + * An unaligned size means the file is corrupted or crafted. + * Abort: there is no point continuing to read unaligned records + * because the caller advances rd->head by event->header.size, + * so every subsequent read would start at a misaligned offset, + * producing garbage headers for the rest of the file. + * + * Exempt three legacy user events that predate the alignment rule: + * + * TRACING_DATA (66): struct tracing_data_event was 12 bytes before + * b39c915a4f36 ("libperf event: Ensure tracing data is multiple + * of 8 sized") added __u32 pad; old perf.data files still contain + * 12-byte records. + * TODO: introduce HEADER_TRACING_DATA2 with guaranteed alignment. + * + * COMPRESSED (81): raw ZSTD output, arbitrary length. Already + * superseded by COMPRESSED2 (83) with PERF_ALIGN. + * + * HEADER_FEATURE (80): do_write_string() uses a 4-byte length + * prefix with no padding to 8-byte total. + * TODO: introduce HEADER_FEATURE2 with guaranteed alignment. + */ + if (event->header.size % sizeof(u64) && + event->header.type != PERF_RECORD_HEADER_TRACING_DATA && + event->header.type != PERF_RECORD_COMPRESSED && + event->header.type != PERF_RECORD_HEADER_FEATURE) { + pr_err("ERROR: at offset %#" PRIx64 ": %s (%u) event size %u is not 8-byte aligned, aborting\n", + file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size); + return -EINVAL; + } if (event->header.type >= PERF_RECORD_HEADER_MAX) { - /* perf should not support unaligned event, stop here. */ - if (event->header.size % sizeof(u64)) - return -EINVAL; - /* This perf is outdated and does not support the latest event type. */ ui__warning("Unsupported header type %u, please consider updating perf.\n", event->header.type); - /* Skip unsupported event by returning its size. */ - return event->header.size; + /* + * Return 0 to skip: the caller (reader__read_event) + * already advances by event->header.size. + */ + return 0; + } + + /* + * Skip rather than abort: a too-small-but-aligned event + * can be safely stepped over without misaligning the stream. + */ + if (perf_event__too_small(event, &min_sz)) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u), skipping\n", + file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, min_sz); + return 0; + } + + if (session->header.needs_swap && + event_swap(event, evlist__sample_id_all(evlist))) { + pr_warning("WARNING: at offset %#" PRIx64 ": swap failed for %s (%u) event, skipping\n", + file_offset, perf_event__name(event->header.type), + event->header.type); + return 0; } - events_stats__inc(&evlist->stats, event->header.type); + events_stats__inc(evlist__stats(evlist), event->header.type); if (event->header.type >= PERF_RECORD_USER_TYPE_START) return perf_session__process_user_event(session, event, file_offset, file_path); @@ -1936,7 +3003,7 @@ perf_session__warn_order(const struct perf_session *session) static void perf_session__warn_about_errors(const struct perf_session *session) { - const struct events_stats *stats = &session->evlist->stats; + const struct events_stats *stats = evlist__stats(session->evlist); if (session->tool->lost == perf_event__process_lost && stats->nr_events[PERF_RECORD_LOST] != 0) { @@ -2342,6 +3409,17 @@ reader__mmap(struct reader *rd, struct perf_session *session) char *buf, **mmaps = rd->mmaps; u64 page_offset; + /* + * Native-endian: MAP_SHARED + PROT_READ — the kernel + * guarantees page-level coherence but a concurrent writer + * could modify the file between validation and use. This + * is a theoretical TOCTOU that affects the entire perf.data + * processing pipeline; fixing it would require copying each + * event to a private buffer before processing. + * + * Cross-endian: MAP_PRIVATE + PROT_WRITE — swap handlers + * get a copy-on-write snapshot immune to concurrent writes. + */ mmap_prot = PROT_READ; mmap_flags = MAP_SHARED; @@ -2373,6 +3451,14 @@ reader__mmap(struct reader *rd, struct perf_session *session) if (session->one_mmap) { session->one_mmap_addr = buf; session->one_mmap_offset = rd->file_offset; + /* + * mmap_size was set to the full file extent (data_offset + + * data_size) but file_offset was shifted forward by + * page_offset for page alignment. Reduce by page_offset + * so the bounds check reflects the file-backed portion + * of the mapping — pages beyond the file cause SIGBUS. + */ + session->one_mmap_size = rd->mmap_size - page_offset; } return 0; @@ -2750,7 +3836,7 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp) ret = fprintf(fp, "\nAggregated stats:%s\n", msg); - ret += events_stats__fprintf(&session->evlist->stats, fp); + ret += events_stats__fprintf(evlist__stats(session->evlist), fp); return ret; } @@ -3023,14 +4109,19 @@ uint16_t perf_session__e_machine(struct perf_session *session, uint32_t *e_flags return EM_HOST; } + /* + * Is the env caching an e_machine? If not we want to compute from the + * more accurate threads. + */ env = perf_session__env(session); - if (env && env->e_machine != EM_NONE) { - if (e_flags) - *e_flags = env->e_flags; - - return env->e_machine; - } + if (env && env->e_machine != EM_NONE) + return perf_env__e_machine(env, e_flags); + /* + * Compute from threads, note this is more accurate than + * perf_env__e_machine that falls back on EM_HOST and doesn't consider + * mixed 32-bit and 64-bit threads. + */ machines__for_each_thread(&session->machines, perf_session__e_machine_cb, &args); @@ -3048,10 +4139,9 @@ uint16_t perf_session__e_machine(struct perf_session *session, uint32_t *e_flags /* * Couldn't determine from the perf_env or current set of - * threads. Default to the host. + * threads. Potentially use logic that uses the arch string otherwise + * default to the host. Don't cache in the perf_env in case later + * threads indicate a better ELF machine type. */ - if (e_flags) - *e_flags = EF_HOST; - - return EM_HOST; + return perf_env__e_machine_nocache(env, e_flags); } diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h index f05f0d4a6c23..ac5803d5fb4e 100644 --- a/tools/perf/util/session.h +++ b/tools/perf/util/session.h @@ -71,6 +71,8 @@ struct perf_session { void *one_mmap_addr; /** @one_mmap_offset: File offset in perf.data file when mapped. */ u64 one_mmap_offset; + /** @one_mmap_size: Size of the single mmap in bytes. */ + u64 one_mmap_size; /** @ordered_events: Used to turn unordered events into ordered ones. */ struct ordered_events ordered_events; /** @data: Optional perf data file being read from. */ @@ -120,6 +122,8 @@ void perf_session__delete(struct perf_session *session); void perf_event_header__bswap(struct perf_event_header *hdr); +bool perf_event__too_small(const union perf_event *event, u32 *min); + int perf_session__peek_event(struct perf_session *session, off_t file_offset, void *buf, size_t buf_sz, union perf_event **event_ptr, diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py index b65b1792ca05..cb14554454a2 100644 --- a/tools/perf/util/setup.py +++ b/tools/perf/util/setup.py @@ -2,6 +2,7 @@ from subprocess import Popen, PIPE from re import sub import shlex +import shutil cc = getenv("CC") assert cc, "Environment variable CC not set" @@ -73,19 +74,22 @@ def finalize_options(self): _install_lib.finalize_options(self) self.build_dir = build_lib + def run(self): + _install_lib.run(self) + shutil.copy2(f'{src_perf}/python/perf.pyi', self.install_dir) + -cflags = getenv('CFLAGS', '').split() # switch off several checks (need to be at the end of cflags list) -cflags += ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ] +extra_cflags = ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ] if cc_is_clang: - cflags += ["-Wno-unused-command-line-argument" ] + extra_cflags += ["-Wno-unused-command-line-argument" ] if clang_has_option("-Wno-cast-function-type-mismatch"): - cflags += ["-Wno-cast-function-type-mismatch" ] + extra_cflags += ["-Wno-cast-function-type-mismatch" ] else: - cflags += ['-Wno-cast-function-type' ] + extra_cflags += ['-Wno-cast-function-type' ] # The python headers have mixed code with declarations (decls after asserts, for instance) -cflags += [ "-Wno-declaration-after-statement" ] +extra_cflags += [ "-Wno-declaration-after-statement" ] src_perf = f'{srctree}/tools/perf' build_lib = getenv('PYTHON_EXTBUILD_LIB') @@ -94,7 +98,7 @@ def finalize_options(self): perf = Extension('perf', sources = [ src_perf + '/util/python.c' ], include_dirs = ['util/include'], - extra_compile_args = cflags, + extra_compile_args = extra_cflags, ) setup(name='perf', diff --git a/tools/perf/util/sideband_evlist.c b/tools/perf/util/sideband_evlist.c index 388846f17bc1..c07dacf3c54c 100644 --- a/tools/perf/util/sideband_evlist.c +++ b/tools/perf/util/sideband_evlist.c @@ -22,7 +22,7 @@ int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, attr->sample_id_all = 1; } - evsel = evsel__new_idx(attr, evlist->core.nr_entries); + evsel = evsel__new_idx(attr, evlist__nr_entries(evlist)); if (!evsel) return -1; @@ -49,14 +49,14 @@ static void *perf_evlist__poll_thread(void *arg) while (!done) { bool got_data = false; - if (evlist->thread.done) + if (evlist__sb_thread_done(evlist)) draining = true; if (!draining) evlist__poll(evlist, 1000); - for (i = 0; i < evlist->core.nr_mmaps; i++) { - struct mmap *map = &evlist->mmap[i]; + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct mmap *map = &evlist__mmap(evlist)[i]; union perf_event *event; if (perf_mmap__read_init(&map->core)) @@ -102,9 +102,9 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) return 0; if (evlist__create_maps(evlist, target)) - goto out_delete_evlist; + goto out_put_evlist; - if (evlist->core.nr_entries > 1) { + if (evlist__nr_entries(evlist) > 1) { bool can_sample_identifier = perf_can_sample_identifier(); evlist__for_each_entry(evlist, counter) @@ -114,27 +114,27 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target) } evlist__for_each_entry(evlist, counter) { - if (evsel__open(counter, evlist->core.user_requested_cpus, - evlist->core.threads) < 0) - goto out_delete_evlist; + if (evsel__open(counter, evlist__core(evlist)->user_requested_cpus, + evlist__core(evlist)->threads) < 0) + goto out_put_evlist; } - if (evlist__mmap(evlist, UINT_MAX)) - goto out_delete_evlist; + if (evlist__do_mmap(evlist, UINT_MAX)) + goto out_put_evlist; evlist__for_each_entry(evlist, counter) { if (evsel__enable(counter)) - goto out_delete_evlist; + goto out_put_evlist; } - evlist->thread.done = 0; - if (pthread_create(&evlist->thread.th, NULL, perf_evlist__poll_thread, evlist)) - goto out_delete_evlist; + evlist__set_sb_thread_done(evlist, 0); + if (pthread_create(evlist__sb_thread_th(evlist), NULL, perf_evlist__poll_thread, evlist)) + goto out_put_evlist; return 0; -out_delete_evlist: - evlist__delete(evlist); +out_put_evlist: + evlist__put(evlist); evlist = NULL; return -1; } @@ -143,7 +143,7 @@ void evlist__stop_sb_thread(struct evlist *evlist) { if (!evlist) return; - evlist->thread.done = 1; - pthread_join(evlist->thread.th, NULL); - evlist__delete(evlist); + evlist__set_sb_thread_done(evlist, 1); + pthread_join(*evlist__sb_thread_th(evlist), NULL); + evlist__put(evlist); } diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 0020089cb13c..dcf9189786f8 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -1,40 +1,45 @@ // SPDX-License-Identifier: GPL-2.0 +#include "sort.h" + #include #include #include -#include #include + +#include +#include #include +#include #include + +#include + +#include "annotate-data.h" +#include "annotate.h" +#include "branch.h" +#include "cacheline.h" +#include "cgroup.h" +#include "comm.h" #include "debug.h" #include "dso.h" -#include "sort.h" +#include "event.h" +#include "evlist.h" +#include "evsel.h" #include "hist.h" -#include "cacheline.h" -#include "comm.h" +#include "machine.h" #include "map.h" -#include "maps.h" -#include "symbol.h" #include "map_symbol.h" -#include "branch.h" -#include "thread.h" -#include "evsel.h" -#include "evlist.h" -#include "srcline.h" -#include "strlist.h" -#include "strbuf.h" +#include "maps.h" #include "mem-events.h" #include "mem-info.h" -#include "annotate.h" -#include "annotate-data.h" -#include "event.h" -#include "time-utils.h" -#include "cgroup.h" -#include "machine.h" #include "session.h" +#include "srcline.h" +#include "strbuf.h" +#include "strlist.h" +#include "symbol.h" +#include "thread.h" +#include "time-utils.h" #include "trace-event.h" -#include -#include #ifdef HAVE_LIBTRACEEVENT #include @@ -464,7 +469,7 @@ int64_t _sort__sym_cmp(struct symbol *sym_l, struct symbol *sym_r) if (sym_l == sym_r) return 0; - if (sym_l->inlined || sym_r->inlined) { + if (symbol__inlined(sym_l) || symbol__inlined(sym_r)) { int ret = strcmp(sym_l->name, sym_r->name); if (ret) @@ -531,7 +536,7 @@ static int _hist_entry__sym_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", level); if (sym && map) { - if (sym->type == STT_OBJECT) { + if (symbol__type(sym) == STT_OBJECT) { ret += repsep_snprintf(bf + ret, size - ret, "%s", sym->name); ret += repsep_snprintf(bf + ret, size - ret, "+0x%llx", ip - map__unmap_ip(map, sym->start)); @@ -539,7 +544,7 @@ static int _hist_entry__sym_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "%.*s", width - ret, sym->name); - if (sym->inlined) + if (symbol__inlined(sym)) ret += repsep_snprintf(bf + ret, size - ret, " (inlined)"); } @@ -1478,7 +1483,7 @@ static int _hist_entry__addr_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", level); if (sym && map) { - if (sym->type == STT_OBJECT) { + if (symbol__type(sym) == STT_OBJECT) { ret += repsep_snprintf(bf + ret, size - ret, "%s", sym->name); ret += repsep_snprintf(bf + ret, size - ret, "+0x%llx", ip - map__unmap_ip(map, sym->start)); @@ -2673,9 +2678,10 @@ struct sort_dimension { static int arch_support_sort_key(const char *sort_key, struct perf_env *env) { - const char *arch = perf_env__arch(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL); - if (!strcmp("x86", arch) || !strcmp("powerpc", arch)) { + if (e_machine == EM_X86_64 || e_machine == EM_386 || e_machine == EM_PPC64 || + e_machine == EM_PPC) { if (!strcmp(sort_key, "p_stage_cyc")) return 1; if (!strcmp(sort_key, "local_p_stage_cyc")) @@ -2686,14 +2692,14 @@ static int arch_support_sort_key(const char *sort_key, struct perf_env *env) static const char *arch_perf_header_entry(const char *se_header, struct perf_env *env) { - const char *arch = perf_env__arch(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL); - if (!strcmp("x86", arch)) { + if (e_machine == EM_X86_64 || e_machine == EM_386) { if (!strcmp(se_header, "Local Pipeline Stage Cycle")) return "Local Retire Latency"; else if (!strcmp(se_header, "Pipeline Stage Cycle")) return "Retire Latency"; - } else if (!strcmp("powerpc", arch)) { + } else if (e_machine == EM_PPC64 || e_machine == EM_PPC) { if (!strcmp(se_header, "Local INSTR Latency")) return "Finish Cyc"; else if (!strcmp(se_header, "INSTR Latency")) @@ -3481,7 +3487,7 @@ static struct evsel *find_evsel(struct evlist *evlist, char *event_name) if (event_name[0] == '%') { int nr = strtol(event_name+1, NULL, 0); - if (nr > evlist->core.nr_entries) + if (nr > evlist__nr_entries(evlist)) return NULL; evsel = evlist__first(evlist); diff --git a/tools/perf/util/srcline.c b/tools/perf/util/srcline.c index db164d258163..b082178c279b 100644 --- a/tools/perf/util/srcline.c +++ b/tools/perf/util/srcline.c @@ -113,16 +113,16 @@ struct symbol *new_inline_sym(struct dso *dso, /* ensure that we don't alias an inlined symbol, which could * lead to double frees in inline_node__delete */ - assert(!base_sym->inlined); + assert(!symbol__inlined(base_sym)); } else { /* create a fake symbol for the inline frame */ inline_sym = symbol__new(base_sym ? base_sym->start : 0, base_sym ? (base_sym->end - base_sym->start) : 0, - base_sym ? base_sym->binding : 0, - base_sym ? base_sym->type : 0, + base_sym ? symbol__binding(base_sym) : 0, + base_sym ? symbol__type(base_sym) : 0, funcname); if (inline_sym) - inline_sym->inlined = 1; + symbol__set_inlined(inline_sym, true); } free(demangled); @@ -429,19 +429,26 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, return addr2inlines(dso_name, addr, dso, sym); } -void inline_node__delete(struct inline_node *node) +void inline_node__clear_frames(struct inline_node *node) { struct inline_list *ilist, *tmp; + if (node == NULL) + return; + list_for_each_entry_safe(ilist, tmp, &node->val, list) { list_del_init(&ilist->list); zfree_srcline(&ilist->srcline); /* only the inlined symbols are owned by the list */ - if (ilist->symbol && ilist->symbol->inlined) + if (ilist->symbol && symbol__inlined(ilist->symbol)) symbol__delete(ilist->symbol); free(ilist); } +} +void inline_node__delete(struct inline_node *node) +{ + inline_node__clear_frames(node); free(node); } diff --git a/tools/perf/util/srcline.h b/tools/perf/util/srcline.h index 7c37b3bf9ce7..1018cbc886d6 100644 --- a/tools/perf/util/srcline.h +++ b/tools/perf/util/srcline.h @@ -47,6 +47,7 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, struct symbol *sym); /* free resources associated to the inline node list */ void inline_node__delete(struct inline_node *node); +void inline_node__clear_frames(struct inline_node *node); /* insert the inline node list into the DSO, which will take ownership */ void inlines__tree_insert(struct rb_root_cached *tree, diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 993f4c4b8f44..f94f1324d24a 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -580,16 +580,13 @@ static void print_metricgroup_header_std(struct perf_stat_config *config, const char *metricgroup_name) { struct outstate *os = ctx; - int n; if (!metricgroup_name) { __new_line_std(config, os); return; } - n = fprintf(config->output, " %*s", EVNAME_LEN, metricgroup_name); - - fprintf(config->output, "%*s", MGROUP_LEN + config->unit_width + 2 - n, ""); + fprintf(config->output, " %*s", config->metric_only_len, metricgroup_name); } static void print_metric_only(struct perf_stat_config *config, @@ -599,19 +596,20 @@ static void print_metric_only(struct perf_stat_config *config, struct outstate *os = ctx; FILE *out = os->fh; char str[1024]; - unsigned mlen = config->metric_only_len; + unsigned mlen; const char *color = metric_threshold_classify__color(thresh); + int olen; - if (!unit) - unit = ""; - if (mlen < strlen(unit)) - mlen = strlen(unit) + 1; + if (!unit) { + os->first = false; + return; + } - if (color) - mlen += strlen(color) + sizeof(PERF_COLOR_RESET) - 1; + mlen = max_t(unsigned, strlen(unit), config->metric_only_len); + olen = snprintf(str, sizeof(str), fmt ?: "", val); color_snprintf(str, sizeof(str), color ?: "", fmt ?: "", val); - fprintf(out, "%*s ", mlen, str); + fprintf(out, "%*s%s", max_t(int, mlen - olen, 1), "", str); os->first = false; } @@ -669,7 +667,7 @@ static void print_metric_header(struct perf_stat_config *config, /* In case of iostat, print metric header for first root port only */ if (config->iostat_run && - os->evsel->priv != os->evsel->evlist->selected->priv) + os->evsel->priv != evlist__selected(os->evsel->evlist)->priv) return; if (os->evsel->cgrp != os->cgrp) @@ -823,9 +821,9 @@ static void printout(struct perf_stat_config *config, struct outstate *os, ok = false; if (counter->supported) { - if (!evlist__has_hybrid_pmus(counter->evlist)) { + if (!evlist__has_hybrid_pmus(counter->evlist) && + counter->pmu && counter->pmu->is_core) config->print_free_counters_hint = 1; - } } } @@ -1128,7 +1126,7 @@ static void print_no_aggr_metric(struct perf_stat_config *config, unsigned int all_idx; struct perf_cpu cpu; - perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.user_requested_cpus) { + perf_cpu_map__for_each_cpu(cpu, all_idx, evlist__core(evlist)->user_requested_cpus) { struct evsel *counter; bool first = true; @@ -1545,7 +1543,7 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf evlist__uniquify_evsel_names(evlist, config); if (config->iostat_run) - evlist->selected = evlist__first(evlist); + evlist__set_selected(evlist, evlist__first(evlist)); if (config->interval) prepare_timestamp(config, &os, ts); diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index bc2d44df7baf..525a3fe4a46e 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -2,20 +2,24 @@ #include #include #include -#include "evsel.h" -#include "stat.h" + +#include + +#include "cgroup.h" #include "color.h" #include "debug.h" -#include "pmu.h" -#include "rblist.h" #include "evlist.h" +#include "evsel.h" #include "expr.h" -#include "metricgroup.h" -#include "cgroup.h" -#include "units.h" +#include "hashmap.h" #include "iostat.h" -#include "util/hashmap.h" +#include "metricgroup.h" +#include "pmu.h" +#include "pmus.h" +#include "rblist.h" +#include "stat.h" #include "tool_pmu.h" +#include "units.h" static bool tool_pmu__is_time_event(const struct perf_stat_config *config, const struct evsel *evsel, int *tool_aggr_idx) @@ -53,6 +57,7 @@ static int prepare_metric(struct perf_stat_config *config, for (i = 0; metric_events[i]; i++) { int source_count = 0, tool_aggr_idx; + int aggr_nr = 1; bool is_tool_time = tool_pmu__is_time_event(config, metric_events[i], &tool_aggr_idx); struct perf_stat_evsel *ps = metric_events[i]->stats; @@ -89,6 +94,7 @@ static int prepare_metric(struct perf_stat_config *config, */ val = NAN; source_count = 0; + aggr_nr = 0; } else { struct perf_stat_aggr *aggr = &ps->aggr[is_tool_time ? tool_aggr_idx : aggr_idx]; @@ -96,6 +102,7 @@ static int prepare_metric(struct perf_stat_config *config, if (aggr->counts.run == 0) { val = NAN; source_count = 0; + aggr_nr = 0; } else { val = aggr->counts.val; if (is_tool_time) { @@ -104,13 +111,14 @@ static int prepare_metric(struct perf_stat_config *config, } if (!source_count) source_count = evsel__source_count(metric_events[i]); + aggr_nr = aggr->nr ?: 1; } } n = strdup(evsel__metric_id(metric_events[i])); if (!n) return -ENOMEM; - expr__add_id_val_source_count(pctx, n, val, source_count); + expr__add_id_val_source_count_aggr_nr(pctx, n, val, source_count, aggr_nr); } for (int j = 0; metric_refs && metric_refs[j].metric_name; j++) { @@ -279,7 +287,7 @@ void *perf_stat__print_shadow_stats_metricgroup(struct perf_stat_config *config, void *ctxp = out->ctx; bool header_printed = false; const char *name = NULL; - struct rblist *metric_events = &evsel->evlist->metric_events; + struct rblist *metric_events = evlist__metric_events(evsel->evlist); me = metricgroup__lookup(metric_events, evsel, false); if (me == NULL) @@ -347,5 +355,5 @@ bool perf_stat__skip_metric_event(struct evsel *evsel) if (!evsel->default_metricgroup) return false; - return !metricgroup__lookup(&evsel->evlist->metric_events, evsel, false); + return !metricgroup__lookup(evlist__metric_events(evsel->evlist), evsel, false); } diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 14d169e22e8f..25f31a174368 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -1,21 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 +#include "stat.h" + #include -#include #include #include #include + +#include +#include + #include "counts.h" #include "cpumap.h" #include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "hashmap.h" #include "header.h" -#include "stat.h" +#include "pmu.h" #include "session.h" #include "target.h" -#include "evlist.h" -#include "evsel.h" #include "thread_map.h" -#include "util/hashmap.h" -#include void update_stats(struct stats *stats, u64 val) { @@ -543,8 +547,8 @@ static void evsel__merge_aliases(struct evsel *evsel) struct evlist *evlist = evsel->evlist; struct evsel *alias; - alias = list_prepare_entry(evsel, &(evlist->core.entries), core.node); - list_for_each_entry_continue(alias, &evlist->core.entries, core.node) { + alias = list_prepare_entry(evsel, &(evlist__core(evlist)->entries), core.node); + list_for_each_entry_continue(alias, &evlist__core(evlist)->entries, core.node) { if (alias->first_wildcard_match == evsel) { /* Merge the same events on different PMUs. */ evsel__merge_aggr_counters(evsel, alias); diff --git a/tools/perf/util/stream.c b/tools/perf/util/stream.c index 3de4a6130853..7bccd2378344 100644 --- a/tools/perf/util/stream.c +++ b/tools/perf/util/stream.c @@ -131,7 +131,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evsel *pos; int i = 0; - BUG_ON(els->nr_evsel < evlist->core.nr_entries); + BUG_ON(els->nr_evsel < evlist__nr_entries(evlist)); evlist__for_each_entry(evlist, pos) { struct hists *hists = evsel__hists(pos); @@ -148,7 +148,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, struct evlist_streams *evlist__create_streams(struct evlist *evlist, int nr_streams_max) { - int nr_evsel = evlist->core.nr_entries, ret = -1; + int nr_evsel = evlist__nr_entries(evlist), ret = -1; struct evlist_streams *els = evlist_streams__new(nr_evsel, nr_streams_max); diff --git a/tools/perf/util/svghelper.c b/tools/perf/util/svghelper.c index e360e7736c7b..826bd2577344 100644 --- a/tools/perf/util/svghelper.c +++ b/tools/perf/util/svghelper.c @@ -47,13 +47,13 @@ static double cpu2slot(int cpu) } static int *topology_map; +static int topology_map_size; static double cpu2y(int cpu) { - if (topology_map) + if (topology_map && cpu >= 0 && cpu < topology_map_size) return cpu2slot(topology_map[cpu]) * SLOT_MULT; - else - return cpu2slot(cpu) * SLOT_MULT; + return cpu2slot(cpu) * SLOT_MULT; } static double time2pixels(u64 __time) @@ -736,7 +736,8 @@ static int str_to_bitmap(char *s, cpumask_t *b, int nr_cpus) return -1; perf_cpu_map__for_each_cpu(cpu, idx, map) { - if (cpu.cpu >= nr_cpus) { + /* perf_cpu_map__new("") returns cpu.cpu == -1 */ + if (cpu.cpu < 0 || cpu.cpu >= nr_cpus) { ret = -1; break; } @@ -794,6 +795,7 @@ int svg_build_topology_map(struct perf_env *env) fprintf(stderr, "topology: no memory\n"); goto exit; } + topology_map_size = nr_cpus; for (i = 0; i < nr_cpus; i++) topology_map[i] = -1; diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 7afa8a117139..f6f6b54da131 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -217,7 +217,7 @@ bool filename__has_section(const char *filename, const char *sec) GElf_Shdr shdr; bool found = false; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) return false; @@ -350,7 +350,8 @@ static bool get_ifunc_name(Elf *elf, struct dso *dso, GElf_Ehdr *ehdr, sym = dso__find_symbol_nocache(dso, addr); /* Expecting the address to be an IFUNC or IFUNC alias */ - if (!sym || sym->start != addr || (sym->type != STT_GNU_IFUNC && !sym->ifunc_alias)) + if (!sym || sym->start != addr || + (symbol__type(sym) != STT_GNU_IFUNC && !symbol__ifunc_alias(sym))) return false; snprintf(buf, buf_sz, "%s@plt", sym->name); @@ -834,10 +835,24 @@ static int elf_read_build_id(Elf *elf, void *bf, size_t size) ptr = data->d_buf; while (ptr < (data->d_buf + data->d_size)) { GElf_Nhdr *nhdr = ptr; - size_t namesz = NOTE_ALIGN(nhdr->n_namesz), - descsz = NOTE_ALIGN(nhdr->n_descsz); + size_t namesz, descsz, remaining; const char *name; + /* ensure the note header fits within the section */ + if (ptr + sizeof(*nhdr) > data->d_buf + data->d_size) + break; + + namesz = NOTE_ALIGN(nhdr->n_namesz); + descsz = NOTE_ALIGN(nhdr->n_descsz); + + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = data->d_buf + data->d_size - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -871,7 +886,7 @@ static int read_build_id(const char *filename, struct build_id *bid) if (size < BUILD_ID_SIZE) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -919,12 +934,14 @@ int filename__read_build_id(const char *filename, struct build_id *bid) return -1; } close(fd); - filename = path; + /* non-empty path means a temp file was created */ + if (path[0] != '\0') + filename = path; } err = read_build_id(filename, bid); - if (m.comp) + if (m.comp && filename == path) unlink(filename); return err; } @@ -934,7 +951,7 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) size_t size = sizeof(bid->data); int fd, err = -1; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -960,17 +977,28 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) err = 0; break; } - } else if (read(fd, bf, descsz) != (ssize_t)descsz) - break; + } else { + /* descsz from untrusted file — clamp to buffer */ + if (descsz > sizeof(bf)) + break; + if (read(fd, bf, descsz) != (ssize_t)descsz) + break; + } } else { - int n = namesz + descsz; + size_t n; - if (n > (int)sizeof(bf)) { + /* int sum of namesz+descsz can overflow negative, bypassing size check */ + if (namesz > sizeof(bf) || descsz > sizeof(bf) - namesz) { n = sizeof(bf); pr_debug("%s: truncating reading of build id in sysfs file %s: n_namesz=%u, n_descsz=%u.\n", __func__, filename, nhdr.n_namesz, nhdr.n_descsz); + } else { + n = namesz + descsz; } - if (read(fd, bf, n) != n) + /* no valid note has both namesz and descsz zero */ + if (n == 0) + break; + if (read(fd, bf, n) != (ssize_t)n) break; } } @@ -994,7 +1022,7 @@ int filename__read_debuglink(const char *filename, char *debuglink, if (err >= 0) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -1023,7 +1051,14 @@ int filename__read_debuglink(const char *filename, char *debuglink, goto out_elf_end; /* the start of this section is a zero-terminated string */ - strncpy(debuglink, data->d_buf, size); + if (data->d_size > 0) { + size_t len = min(size - 1, data->d_size); + + memcpy(debuglink, data->d_buf, len); + debuglink[len] = '\0'; + } else { + debuglink[0] = '\0'; + } err = 0; @@ -1108,9 +1143,9 @@ static Elf *read_gnu_debugdata(struct dso *dso, Elf *elf, const char *name, int return NULL; } - temp_fd = mkstemp(temp_filename); + temp_fd = mkostemp(temp_filename, O_CLOEXEC); if (temp_fd < 0) { - pr_debug("%s: mkstemp: %m\n", __func__); + pr_debug("%s: mkostemp: %m\n", __func__); *dso__load_errno(dso) = -errno; fclose(wrapped); return NULL; @@ -1152,7 +1187,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, type = dso__symtab_type(dso); } else { - fd = open(name, O_RDONLY); + fd = open(name, O_RDONLY | O_CLOEXEC); if (fd < 0) { *dso__load_errno(dso) = errno; return -1; @@ -1341,6 +1376,24 @@ static u64 ref_reloc(struct kmap *kmap) void __weak arch__sym_update(struct symbol *s __maybe_unused, GElf_Sym *sym __maybe_unused) { } +struct remap_kernel_ctx { + u64 sh_addr; + u64 sh_size; + u64 sh_offset; + struct kmap *kmap; +}; + +static int remap_kernel_cb(struct map *map, void *data) +{ + struct remap_kernel_ctx *ctx = data; + + map__set_start(map, ctx->sh_addr + ref_reloc(ctx->kmap)); + map__set_end(map, map__start(map) + ctx->sh_size); + map__set_pgoff(map, ctx->sh_offset); + map__set_mapping_type(map, MAPPING_TYPE__DSO); + return 0; +} + static int dso__process_kernel_symbol(struct dso *dso, struct map *map, GElf_Sym *sym, GElf_Shdr *shdr, struct maps *kmaps, struct kmap *kmap, @@ -1371,22 +1424,15 @@ static int dso__process_kernel_symbol(struct dso *dso, struct map *map, * map to the kernel dso. */ if (*remap_kernel && dso__kernel(dso) && !kmodule) { + struct remap_kernel_ctx ctx = { + .sh_addr = shdr->sh_addr, + .sh_size = shdr->sh_size, + .sh_offset = shdr->sh_offset, + .kmap = kmap + }; + *remap_kernel = false; - map__set_start(map, shdr->sh_addr + ref_reloc(kmap)); - map__set_end(map, map__start(map) + shdr->sh_size); - map__set_pgoff(map, shdr->sh_offset); - map__set_mapping_type(map, MAPPING_TYPE__DSO); - /* Ensure maps are correctly ordered */ - if (kmaps) { - int err; - struct map *tmp = map__get(map); - - maps__remove(kmaps, map); - err = maps__insert(kmaps, map); - map__put(tmp); - if (err) - return err; - } + maps__mutate_mapping(kmaps, map, remap_kernel_cb, &ctx); } /* @@ -1592,9 +1638,11 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, if (!is_label && !elf_sym__filter(&sym)) continue; - /* Reject ARM ELF "mapping symbols": these aren't unique and + /* + * Reject ARM ELF "mapping symbols": these aren't unique and * don't identify functions, so will confuse the profile - * output: */ + * output: + */ if (ehdr.e_machine == EM_ARM || ehdr.e_machine == EM_AARCH64) { if (elf_name[0] == '$' && strchr("adtx", elf_name[1]) && (elf_name[2] == '\0' || elf_name[2] == '.')) @@ -1607,6 +1655,10 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, continue; } + /* Reject kernel mapping symbols for kernel DSOs only */ + if (dso__kernel(dso) && is_ignored_kernel_symbol(elf_name)) + continue; + if (runtime_ss->opdsec && sym.st_shndx == runtime_ss->opdidx) { u32 offset = sym.st_value - syms_ss->opdshdr.sh_addr; u64 *opd = opddata->d_buf + offset; @@ -1727,7 +1779,7 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, arch__sym_update(f, &sym); - __symbols__insert(dso__symbols(curr_dso), f, dso__kernel(dso)); + __symbols__insert(dso__symbols(curr_dso), f); nr++; } dso__put(curr_dso); @@ -1945,7 +1997,7 @@ static int kcore__open(struct kcore *kcore, const char *filename) { GElf_Ehdr *ehdr; - kcore->fd = open(filename, O_RDONLY); + kcore->fd = open(filename, O_RDONLY | O_CLOEXEC); if (kcore->fd == -1) return -1; @@ -1976,9 +2028,9 @@ static int kcore__init(struct kcore *kcore, char *filename, int elfclass, kcore->elfclass = elfclass; if (temp) - kcore->fd = mkstemp(filename); + kcore->fd = mkostemp(filename, O_CLOEXEC); else - kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0400); + kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0400); if (kcore->fd == -1) return -1; @@ -2164,6 +2216,10 @@ static int kcore_copy__process_kallsyms(void *arg, const char *name, char type, if (!kallsyms__is_function(type)) return 0; + /* Ignore livepatch symbols */ + if (is_livepatch_symbol(name)) + return 0; + if (strchr(name, '[')) { if (!kci->first_module_symbol || start < kci->first_module_symbol) kci->first_module_symbol = start; @@ -2454,11 +2510,11 @@ static int kcore_copy__compare_files(const char *from_filename, { int from, to, err = -1; - from = open(from_filename, O_RDONLY); + from = open(from_filename, O_RDONLY | O_CLOEXEC); if (from < 0) return -1; - to = open(to_filename, O_RDONLY); + to = open(to_filename, O_RDONLY | O_CLOEXEC); if (to < 0) goto out_close_from; @@ -2876,7 +2932,7 @@ int get_sdt_note_list(struct list_head *head, const char *target) Elf *elf; int fd, ret; - fd = open(target, O_RDONLY); + fd = open(target, O_RDONLY | O_CLOEXEC); if (fd < 0) return -EBADF; diff --git a/tools/perf/util/symbol-minimal.c b/tools/perf/util/symbol-minimal.c index 8221dc9868f7..0a71d1463952 100644 --- a/tools/perf/util/symbol-minimal.c +++ b/tools/perf/util/symbol-minimal.c @@ -1,3 +1,4 @@ +#include "debug.h" #include "dso.h" #include "symbol.h" #include "symsrc.h" @@ -44,7 +45,7 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, ptr = note_data; while ((ptr + sizeof(*nhdr)) < (note_data + note_len)) { const char *name; - size_t namesz, descsz; + size_t namesz, descsz, remaining; nhdr = ptr; if (need_swap) { @@ -56,6 +57,14 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, namesz = NOTE_ALIGN(nhdr->n_namesz); descsz = NOTE_ALIGN(nhdr->n_descsz); + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = note_data + note_len - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -166,7 +175,7 @@ int filename__read_build_id(const char *filename, struct build_id *bid) if (elf32) { hdrs.phdr32[i].p_type = bswap_32(hdrs.phdr32[i].p_type); hdrs.phdr32[i].p_offset = bswap_32(hdrs.phdr32[i].p_offset); - hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_offset); + hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_filesz); } else { hdrs.phdr64[i].p_type = bswap_32(hdrs.phdr64[i].p_type); hdrs.phdr64[i].p_offset = bswap_64(hdrs.phdr64[i].p_offset); @@ -177,6 +186,9 @@ int filename__read_build_id(const char *filename, struct build_id *bid) continue; p_filesz = elf32 ? hdrs.phdr32[i].p_filesz : hdrs.phdr64[i].p_filesz; + /* ssize_t can go negative with crafted ELF p_filesz values */ + if (p_filesz <= 0) + continue; if (p_filesz > buf_size) { void *tmp; diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index fcaeeddbbb6b..a562702b4841 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -48,9 +48,15 @@ #include #include +static int map_fixup_cb(struct map *map, void *data __maybe_unused) +{ + map__fixup_start(map); + map__fixup_end(map); + return 0; +} + static int dso__load_kernel_sym(struct dso *dso, struct map *map); static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map); -static bool symbol__is_idle(const char *name); int vmlinux_path__nr_entries; char **vmlinux_path; @@ -163,24 +169,24 @@ static int choose_best_symbol(struct symbol *syma, struct symbol *symb) else if ((a == 0) && (b > 0)) return SYMBOL_B; - if (syma->type != symb->type) { - if (syma->type == STT_NOTYPE) + if (symbol__type(syma) != symbol__type(symb)) { + if (symbol__type(syma) == STT_NOTYPE) return SYMBOL_B; - if (symb->type == STT_NOTYPE) + if (symbol__type(symb) == STT_NOTYPE) return SYMBOL_A; } /* Prefer a non weak symbol over a weak one */ - a = syma->binding == STB_WEAK; - b = symb->binding == STB_WEAK; + a = symbol__binding(syma) == STB_WEAK; + b = symbol__binding(symb) == STB_WEAK; if (b && !a) return SYMBOL_A; if (a && !b) return SYMBOL_B; /* Prefer a global symbol over a non global one */ - a = syma->binding == STB_GLOBAL; - b = symb->binding == STB_GLOBAL; + a = symbol__binding(syma) == STB_GLOBAL; + b = symbol__binding(symb) == STB_GLOBAL; if (a && !b) return SYMBOL_A; if (b && !a) @@ -227,14 +233,14 @@ void symbols__fixup_duplicate(struct rb_root_cached *symbols) continue; if (choose_best_symbol(curr, next) == SYMBOL_A) { - if (next->type == STT_GNU_IFUNC) - curr->ifunc_alias = true; + if (symbol__type(next) == STT_GNU_IFUNC) + symbol__set_ifunc_alias(curr, true); rb_erase_cached(&next->rb_node, symbols); symbol__delete(next); goto again; } else { - if (curr->type == STT_GNU_IFUNC) - next->ifunc_alias = true; + if (symbol__type(curr) == STT_GNU_IFUNC) + symbol__set_ifunc_alias(next, true); nd = rb_next(&curr->rb_node); rb_erase_cached(&curr->rb_node, symbols); symbol__delete(curr); @@ -322,8 +328,8 @@ struct symbol *symbol__new(u64 start, u64 len, u8 binding, u8 type, const char * sym->start = start; sym->end = len ? start + len : start; - sym->type = type; - sym->binding = binding; + atomic_init(&sym->flags, (type << SYMBOL_FLAG_TYPE_SHIFT) | + (binding << SYMBOL_FLAG_BINDING_SHIFT)); sym->namelen = namelen - 1; pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n", @@ -345,6 +351,49 @@ void symbol__delete(struct symbol *sym) free(((void *)sym) - symbol_conf.priv_size); } +void symbol__set_ignore(struct symbol *sym, bool ignore) +{ + if (ignore) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_IGNORE); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_IGNORE); +} + +void symbol__set_annotate2(struct symbol *sym, bool annotate2) +{ + if (annotate2) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_ANNOTATE2); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_ANNOTATE2); +} + +void symbol__set_inlined(struct symbol *sym, bool inlined) +{ + if (inlined) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_INLINED); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_INLINED); +} + +void symbol__set_ifunc_alias(struct symbol *sym, bool ifunc_alias) +{ + if (ifunc_alias) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_IFUNC_ALIAS); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_IFUNC_ALIAS); +} + +static void symbol__set_idle(struct symbol *sym, bool idle) +{ + uint16_t old_flags = atomic_load_explicit(&sym->flags, memory_order_relaxed); + uint16_t new_flags; + uint16_t idle_val = idle ? SYMBOL_IDLE__IDLE : SYMBOL_IDLE__NOT_IDLE; + + do { + new_flags = old_flags & ~SYMBOL_FLAG_IDLE_MASK; + new_flags |= (idle_val << SYMBOL_FLAG_IDLE_SHIFT); + } while (!atomic_compare_exchange_weak(&sym->flags, &old_flags, new_flags)); +} void symbols__delete(struct rb_root_cached *symbols) { struct symbol *pos; @@ -358,8 +407,7 @@ void symbols__delete(struct rb_root_cached *symbols) } } -void __symbols__insert(struct rb_root_cached *symbols, - struct symbol *sym, bool kernel) +void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym) { struct rb_node **p = &symbols->rb_root.rb_node; struct rb_node *parent = NULL; @@ -367,17 +415,6 @@ void __symbols__insert(struct rb_root_cached *symbols, struct symbol *s; bool leftmost = true; - if (kernel) { - const char *name = sym->name; - /* - * ppc64 uses function descriptors and appends a '.' to the - * start of every instruction address. Remove it. - */ - if (name[0] == '.') - name++; - sym->idle = symbol__is_idle(name); - } - while (*p != NULL) { parent = *p; s = rb_entry(parent, struct symbol, rb_node); @@ -394,7 +431,7 @@ void __symbols__insert(struct rb_root_cached *symbols, void symbols__insert(struct rb_root_cached *symbols, struct symbol *sym) { - __symbols__insert(symbols, sym, false); + __symbols__insert(symbols, sym); } static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) @@ -555,7 +592,7 @@ void dso__reset_find_symbol_cache(struct dso *dso) void dso__insert_symbol(struct dso *dso, struct symbol *sym) { - __symbols__insert(dso__symbols(dso), sym, dso__kernel(dso)); + __symbols__insert(dso__symbols(dso), sym); /* update the symbol cache if necessary */ if (dso__last_find_result_addr(dso) >= sym->start && @@ -717,47 +754,120 @@ int modules__parse(const char *filename, void *arg, return err; } + /* * These are symbols in the kernel image, so make sure that * sym is from a kernel DSO. */ -static bool symbol__is_idle(const char *name) +static int sym_name_cmp(const void *a, const void *b) +{ + const char *name = a; + const char *const *sym = b; + + return strcmp(name, *sym); +} + +static bool match_x86_idle_routine(const char *name, const char *base) +{ + if (strstarts(name, base)) { + size_t len = strlen(base); + + if (name[len] == '\0' || name[len] == '.') + return true; + } + return false; +} + +bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env) { - const char * const idle_symbols[] = { + static const char * const idle_symbols[] = { "acpi_idle_do_entry", "acpi_processor_ffh_cstate_enter", "arch_cpu_idle", "cpu_idle", "cpu_startup_entry", - "idle_cpu", - "intel_idle", - "intel_idle_ibrs", "default_idle", - "native_safe_halt", "enter_idle", "exit_idle", - "mwait_idle", - "mwait_idle_with_hints", - "mwait_idle_with_hints.constprop.0", + "idle_cpu", + "native_safe_halt", "poll_idle", - "ppc64_runlatch_off", "pseries_dedicated_idle_sleep", - "psw_idle", - "psw_idle_exit", - NULL }; - int i; - static struct strlist *idle_symbols_list; + const char *name = sym->name; + uint16_t e_machine; + + { + uint16_t flags = atomic_load_explicit(&sym->flags, memory_order_relaxed); + uint16_t idle_val = (flags & SYMBOL_FLAG_IDLE_MASK) >> SYMBOL_FLAG_IDLE_SHIFT; + + if (idle_val != SYMBOL_IDLE__UNKNOWN) + return idle_val == SYMBOL_IDLE__IDLE; + } + + if (!dso || dso__kernel(dso) == DSO_SPACE__USER) { + symbol__set_idle(sym, /*idle=*/false); + return false; + } + + /* + * ppc64 uses function descriptors and appends a '.' to the + * start of every instruction address. Remove it. + */ + if (name[0] == '.') + name++; + + if (bsearch(name, idle_symbols, ARRAY_SIZE(idle_symbols), + sizeof(idle_symbols[0]), sym_name_cmp)) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + + e_machine = (env && env->arch) ? perf_env__e_machine(env, NULL) : EM_NONE; + if (e_machine == EM_NONE && dso) + e_machine = dso__e_machine((struct dso *)dso, NULL, NULL); + if (e_machine == EM_NONE && env) + e_machine = perf_env__e_machine(env, NULL); + + if (e_machine == EM_386 || e_machine == EM_X86_64) { + if (match_x86_idle_routine(name, "intel_idle") || + match_x86_idle_routine(name, "intel_idle_irq") || + match_x86_idle_routine(name, "intel_idle_ibrs") || + match_x86_idle_routine(name, "mwait_idle") || + match_x86_idle_routine(name, "mwait_idle_with_hints")) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + } - if (idle_symbols_list) - return strlist__has_entry(idle_symbols_list, name); + if (e_machine == EM_PPC64 && !strcmp(name, "ppc64_runlatch_off")) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + + if (e_machine == EM_S390 && strstarts(name, "psw_idle")) { + int major = 0, minor = 0; + const char *release = env ? perf_env__os_release(env) : NULL; - idle_symbols_list = strlist__new(NULL, NULL); + /* + * If we can't determine the release (e.g. unpopulated guest traces), + * default to idle. + */ + if (!release) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } - for (i = 0; idle_symbols[i]; i++) - strlist__add(idle_symbols_list, idle_symbols[i]); + /* Before v6.10, s390 used psw_idle. */ + if (sscanf(release, "%d.%d", &major, &minor) == 2 && + (major < 6 || (major == 6 && minor < 10))) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + } - return strlist__has_entry(idle_symbols_list, name); + symbol__set_idle(sym, /*idle=*/false); + return false; } static int map__process_kallsym_symbol(void *arg, const char *name, @@ -770,8 +880,8 @@ static int map__process_kallsym_symbol(void *arg, const char *name, if (!symbol_type__filter(type)) return 0; - /* Ignore local symbols for ARM modules */ - if (name[0] == '$') + /* Ignore mapping and livepatch symbols in kallsyms */ + if (is_ignored_kernel_symbol(name) || is_livepatch_symbol(name)) return 0; /* @@ -786,7 +896,7 @@ static int map__process_kallsym_symbol(void *arg, const char *name, * We will pass the symbols to the filter later, in * map__split_kallsyms, when we have split the maps per module */ - __symbols__insert(root, sym, !strchr(name, '[')); + __symbols__insert(root, sym); return 0; } @@ -851,6 +961,23 @@ static int maps__split_kallsyms_for_kcore(struct maps *kmaps, struct dso *dso) return count; } +static uint16_t machine_or_dso_e_machine(struct machine *machine, struct dso *dso) +{ + uint16_t e_machine = EM_NONE; + /* DSO should be most accurate */ + if (dso) + e_machine = dso__e_machine(dso, machine, /*e_flags=*/NULL); + + if (e_machine != EM_NONE) + return e_machine; + + /* Check the global environment next. */ + if (machine && machine->env && machine->env->e_machine != EM_NONE) + return machine->env->e_machine; + + return perf_env__e_machine(machine ? machine->env : NULL, /*e_flags=*/NULL); +} + /* * Split the symbols into maps, making sure there are no overlaps, i.e. the * kernel range is broken in several maps, named [kernel].N, as we don't have @@ -866,14 +993,13 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, struct rb_root_cached *root = dso__symbols(dso); struct rb_node *next = rb_first_cached(root); int kernel_range = 0; - bool x86_64; + uint16_t e_machine = EM_NONE; if (!kmaps) return -1; machine = maps__machine(kmaps); - - x86_64 = machine__is(machine, "x86_64"); + e_machine = machine_or_dso_e_machine(machine, dso); while (next) { char *module; @@ -925,7 +1051,7 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, */ pos->start = map__map_ip(curr_map, pos->start); pos->end = map__map_ip(curr_map, pos->end); - } else if (x86_64 && is_entry_trampoline(pos->name)) { + } else if (e_machine == EM_X86_64 && is_entry_trampoline(pos->name)) { /* * These symbols are not needed anymore since the * trampoline maps refer to the text section and it's @@ -1428,7 +1554,7 @@ static int dso__load_kcore(struct dso *dso, struct map *map, free(new_node); } - if (machine__is(machine, "x86_64")) { + if (machine_or_dso_e_machine(machine, dso) == EM_X86_64) { u64 addr; /* @@ -1716,7 +1842,7 @@ int dso__load(struct dso *dso, struct map *map) ret = dso__load_guest_kernel_sym(dso, map); machine = maps__machine(map__kmaps(map)); - if (machine__is(machine, "x86_64")) + if (machine && machine_or_dso_e_machine(machine, dso) == EM_X86_64) machine__map_x86_64_entry_trampolines(machine, dso); goto out; } @@ -2121,10 +2247,11 @@ static int dso__load_kernel_sym(struct dso *dso, struct map *map) free(kallsyms_allocated_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__KALLSYMS); dso__set_long_name(dso, DSO__NAME_KALLSYMS, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; @@ -2156,7 +2283,7 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (!kallsyms_filename) return -1; } else { - sprintf(path, "%s/proc/kallsyms", machine->root_dir); + snprintf(path, sizeof(path), "%s/proc/kallsyms", machine->root_dir); kallsyms_filename = path; } @@ -2164,10 +2291,11 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (err > 0) pr_debug("Using %s for symbols\n", kallsyms_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__GUEST_KALLSYMS); dso__set_long_name(dso, machine->mmap_name, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; @@ -2209,7 +2337,7 @@ static int vmlinux_path__init(struct perf_env *env) { struct utsname uts; char bf[PATH_MAX]; - char *kernel_version; + const char *kernel_version; unsigned int i; vmlinux_path = malloc(sizeof(char *) * (ARRAY_SIZE(vmlinux_paths) + @@ -2226,7 +2354,7 @@ static int vmlinux_path__init(struct perf_env *env) return 0; if (env) { - kernel_version = env->os_release; + kernel_version = perf_env__os_release(env); } else { if (uname(&uts) < 0) goto out_fail; @@ -2476,6 +2604,7 @@ void symbol__exit(void) { if (!symbol_conf.initialized) return; + strlist__delete(symbol_conf.bt_stop_list); strlist__delete(symbol_conf.sym_list); strlist__delete(symbol_conf.dso_list); diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h index bd6eb90c8668..d0bac824c79c 100644 --- a/tools/perf/util/symbol.h +++ b/tools/perf/util/symbol.h @@ -6,8 +6,11 @@ #include #include #include +#include #include +#include #include +#include #include #include #include "addr_location.h" @@ -27,6 +30,32 @@ struct map; struct maps; struct option; struct build_id; +struct perf_env; + +/* + * Ignore kernel mapping symbols, matching kernel is_mapping_symbol() logic. + * This checks for '$' prefix (used by ARM, AArch64, RISC-V) and + * x86 local symbol prefixes (.L* and L0*). + * Only use this for kernel symbols (kallsyms, ksymbol events, kernel ELF DSOs). + */ +static inline bool is_ignored_kernel_symbol(const char *str) +{ + if (str[0] == '.' && str[1] == 'L') + return true; + if (str[0] == 'L' && str[1] == '0') + return true; + return str[0] == '$'; +} + +/* + * Livepatch symbols (.klp.sym.*) are relocation placeholders whose resolved + * addresses alias existing kernel symbols. They carry a [module] tag which + * confuses module boundary tracking and symbol table lookups. + */ +static inline bool is_livepatch_symbol(const char *str) +{ + return strstarts(str, KLP_SYM_PREFIX); +} /* * libelf 0.8.x and earlier do not support ELF_C_READ_MMAP; @@ -43,6 +72,22 @@ Elf_Scn *elf_section_by_name(Elf *elf, GElf_Ehdr *ep, GElf_Shdr *shp, const char *name, size_t *idx); #endif +enum symbol_idle_kind { + SYMBOL_IDLE__UNKNOWN = 0, + SYMBOL_IDLE__NOT_IDLE = 1, + SYMBOL_IDLE__IDLE = 2, +}; + +#define SYMBOL_FLAG_TYPE_SHIFT 0 +#define SYMBOL_FLAG_TYPE_MASK (0xF << SYMBOL_FLAG_TYPE_SHIFT) +#define SYMBOL_FLAG_BINDING_SHIFT 4 +#define SYMBOL_FLAG_BINDING_MASK (0xF << SYMBOL_FLAG_BINDING_SHIFT) +#define SYMBOL_FLAG_IDLE_SHIFT 8 +#define SYMBOL_FLAG_IDLE_MASK (0x3 << SYMBOL_FLAG_IDLE_SHIFT) +#define SYMBOL_FLAG_IGNORE (1 << 10) +#define SYMBOL_FLAG_INLINED (1 << 11) +#define SYMBOL_FLAG_ANNOTATE2 (1 << 12) +#define SYMBOL_FLAG_IFUNC_ALIAS (1 << 13) /** * A symtab entry. When allocated this may be preceded by an annotation (see * symbol__annotation) and/or a browser_index (see symbol__browser_index). @@ -54,20 +99,7 @@ struct symbol { u64 end; /** Length of the string name. */ u16 namelen; - /** ELF symbol type as defined for st_info. E.g STT_OBJECT or STT_FUNC. */ - u8 type:4; - /** ELF binding type as defined for st_info. E.g. STB_WEAK or STB_GLOBAL. */ - u8 binding:4; - /** Set true for kernel symbols of idle routines. */ - u8 idle:1; - /** Resolvable but tools ignore it (e.g. idle routines). */ - u8 ignore:1; - /** Symbol for an inlined function. */ - u8 inlined:1; - /** Has symbol__annotate2 been performed. */ - u8 annotate2:1; - /** Symbol is an alias of an STT_GNU_IFUNC */ - u8 ifunc_alias:1; + _Atomic uint16_t flags; /** Architecture specific. Unused except on PPC where it holds st_other. */ u8 arch_sym; /** The name of length namelen associated with the symbol. */ @@ -77,6 +109,49 @@ struct symbol { void symbol__delete(struct symbol *sym); void symbols__delete(struct rb_root_cached *symbols); +static inline u8 symbol__type(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_TYPE_MASK) >> SYMBOL_FLAG_TYPE_SHIFT; +} + +static inline u8 symbol__binding(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_BINDING_MASK) >> SYMBOL_FLAG_BINDING_SHIFT; +} + +static inline bool symbol__ignore(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_IGNORE) != 0; +} + +static inline bool symbol__inlined(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_INLINED) != 0; +} + +static inline bool symbol__is_annotate2(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_ANNOTATE2) != 0; +} + +static inline bool symbol__ifunc_alias(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_IFUNC_ALIAS) != 0; +} + +bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env); + +void symbol__set_ignore(struct symbol *sym, bool ignore); +void symbol__set_annotate2(struct symbol *sym, bool annotate2); +void symbol__set_inlined(struct symbol *sym, bool inlined); +void symbol__set_ifunc_alias(struct symbol *sym, bool ifunc_alias); + /* symbols__for_each_entry - iterate over symbols (rb_root) * * @symbols: the rb_root of symbols @@ -154,7 +229,6 @@ int filename__read_debuglink(const char *filename, char *debuglink, size_t size); bool filename__has_section(const char *filename, const char *sec); -struct perf_env; int symbol__init(struct perf_env *env); void symbol__exit(void); void symbol__elf_init(void); @@ -194,8 +268,7 @@ int dso__synthesize_plt_symbols(struct dso *dso, struct symsrc *ss); char *dso__demangle_sym(struct dso *dso, int kmodule, const char *elf_name); -void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym, - bool kernel); +void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym); void symbols__insert(struct rb_root_cached *symbols, struct symbol *sym); void symbols__fixup_duplicate(struct rb_root_cached *symbols); void symbols__fixup_end(struct rb_root_cached *symbols, bool is_kallsyms); diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h index 6cd454d7c98e..0dee5aa6a534 100644 --- a/tools/perf/util/symbol_conf.h +++ b/tools/perf/util/symbol_conf.h @@ -9,6 +9,15 @@ struct strlist; struct intlist; +enum unwind_style { + UNWIND_STYLE_UNKNOWN = 0, + UNWIND_STYLE_LIBDW, + UNWIND_STYLE_LIBUNWIND, +}; + +#define MAX_UNWIND_STYLE (UNWIND_STYLE_LIBUNWIND + 1) + + enum a2l_style { A2L_STYLE_UNKNOWN = 0, A2L_STYLE_LIBDW, @@ -81,6 +90,7 @@ struct symbol_conf { const char *addr2line_path; enum a2l_style addr2line_style[MAX_A2L_STYLE]; int addr2line_timeout_ms; + enum unwind_style unwind_style[MAX_UNWIND_STYLE]; unsigned long time_quantum; struct strlist *dso_list, *comm_list, diff --git a/tools/perf/util/symbol_fprintf.c b/tools/perf/util/symbol_fprintf.c index 53e1af4ed9ac..4dc8d5761f52 100644 --- a/tools/perf/util/symbol_fprintf.c +++ b/tools/perf/util/symbol_fprintf.c @@ -11,8 +11,8 @@ size_t symbol__fprintf(struct symbol *sym, FILE *fp) { return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n", sym->start, sym->end, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w', + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', sym->name); } diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 85bee747f4cd..b75f9dcf4dbf 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1455,7 +1455,8 @@ int perf_event__synthesize_stat_round(const struct perf_tool *tool, return process(tool, (union perf_event *) &event, NULL, machine); } -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format) +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format, + u64 branch_sample_type) { size_t sz, result = sizeof(struct perf_record_sample); @@ -1515,8 +1516,10 @@ size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); + /* nr */ + sz += sizeof(u64); + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + sz += sizeof(u64); result += sz; } @@ -1605,7 +1608,7 @@ static __u64 *copy_read_group_values(__u64 *array, __u64 read_format, } int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, - const struct perf_sample *sample) + u64 branch_sample_type, const struct perf_sample *sample) { __u64 *array; size_t sz; @@ -1719,9 +1722,17 @@ int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_fo if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); - memcpy(array, sample->branch_stack, sz); + + *array++ = sample->branch_stack->nr; + + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) { + if (sample->no_hw_idx) + *array++ = 0; + else + *array++ = sample->branch_stack->hw_idx; + } + + memcpy(array, perf_sample__branch_entries((struct perf_sample *)sample), sz); array = (void *)array + sz; } @@ -2170,11 +2181,21 @@ int perf_event__synthesize_attr(const struct perf_tool *tool, struct perf_event_ u32 ids, u64 *id, perf_event__handler_t process) { union perf_event *ev; - size_t size; + size_t attr_size, size; int err; - size = sizeof(struct perf_event_attr); - size = PERF_ALIGN(size, sizeof(u64)); + /* + * Use attr->size for the event layout, not the compiled + * sizeof(struct perf_event_attr), so that synthesized events + * match the source perf.data layout. This matters for perf + * inject, which re-synthesizes attrs from a file that may + * have been recorded by a different version of perf. + * perf_record_header_attr_id() locates the ID array at + * attr->size bytes past the attr. + */ + attr_size = attr->size ?: sizeof(struct perf_event_attr); + + size = PERF_ALIGN(attr_size, sizeof(u64)); size += sizeof(struct perf_event_header); size += ids * sizeof(u64); @@ -2183,7 +2204,14 @@ int perf_event__synthesize_attr(const struct perf_tool *tool, struct perf_event_ if (ev == NULL) return -ENOMEM; - ev->attr.attr = *attr; + /* + * Copy only the bytes we understand; zalloc ensures that any + * extra bytes between sizeof(struct perf_event_attr) and + * attr_size are zero when the source file uses a newer, larger + * struct. + */ + memcpy(&ev->attr.attr, attr, min(sizeof(struct perf_event_attr), attr_size)); + ev->attr.attr.size = attr_size; memcpy(perf_record_header_attr_id(ev), id, ids * sizeof(u64)); ev->attr.header.type = PERF_RECORD_HEADER_ATTR; @@ -2219,7 +2247,7 @@ int perf_event__synthesize_tracing_data(const struct perf_tool *tool, int fd, st * - write the tracing data from the temp file * to the pipe */ - tdata = tracing_data_get(&evlist->core.entries, fd, true); + tdata = tracing_data_get(&evlist__core(evlist)->entries, fd, true); if (!tdata) return -1; @@ -2252,17 +2280,25 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, const struct build_id *bid, const char *filename) { union perf_event ev; - size_t len; + size_t len, filename_len = strlen(filename); + u64 sample_type = sample->evsel ? sample->evsel->core.attr.sample_type : 0; + void *array = &ev; + int ret; - len = sizeof(ev.build_id) + strlen(filename) + 1; + if (filename_len >= PATH_MAX) + return -EINVAL; + + len = sizeof(ev.build_id) + filename_len + 1; len = PERF_ALIGN(len, sizeof(u64)); + if (len + MAX_ID_HDR_ENTRIES * sizeof(__u64) > sizeof(ev)) + return -E2BIG; + memset(&ev, 0, len); ev.build_id.size = bid->size; @@ -2275,23 +2311,17 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, ev.build_id.header.size = len; strcpy(ev.build_id.filename, filename); - if (evsel) { - void *array = &ev; - int ret; - - array += ev.header.size; - ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample); - if (ret < 0) - return ret; - - if (ret & 7) { - pr_err("Bad id sample size %d\n", ret); - return -EINVAL; - } + array += ev.header.size; + ret = perf_event__synthesize_id_sample(array, sample_type, sample); + if (ret < 0) + return ret; - ev.header.size += ret; + if (ret & 7) { + pr_err("Bad id sample size %d\n", ret); + return -EINVAL; } + ev.header.size += ret; return process(tool, &ev, sample, machine); } @@ -2299,7 +2329,6 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -2308,13 +2337,21 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, const char *filename) { union perf_event ev; + size_t filename_len = strlen(filename); size_t ev_len; + u64 sample_type = sample->evsel ? sample->evsel->core.attr.sample_type : 0; void *array; int ret; - ev_len = sizeof(ev.mmap2) - sizeof(ev.mmap2.filename) + strlen(filename) + 1; + if (filename_len >= sizeof(ev.mmap2.filename)) + return -EINVAL; + + ev_len = sizeof(ev.mmap2) - sizeof(ev.mmap2.filename) + filename_len + 1; ev_len = PERF_ALIGN(ev_len, sizeof(u64)); + if (ev_len + MAX_ID_HDR_ENTRIES * sizeof(__u64) > sizeof(ev)) + return -E2BIG; + memset(&ev, 0, ev_len); ev.mmap2.header.type = PERF_RECORD_MMAP2; @@ -2339,7 +2376,7 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, array = &ev; array += ev.header.size; - ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample); + ret = perf_event__synthesize_id_sample(array, sample_type, sample); if (ret < 0) return ret; @@ -2367,13 +2404,16 @@ int perf_event__synthesize_stat_events(struct perf_stat_config *config, const st } err = perf_event__synthesize_extra_attr(tool, evlist, process, attrs); - err = perf_event__synthesize_thread_map2(tool, evlist->core.threads, process, NULL); + err = perf_event__synthesize_thread_map2(tool, evlist__core(evlist)->threads, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; } - err = perf_event__synthesize_cpu_map(tool, evlist->core.user_requested_cpus, process, NULL); + err = perf_event__synthesize_cpu_map(tool, + evlist__core(evlist)->user_requested_cpus, + process, /*machine=*/NULL); if (err < 0) { pr_err("Couldn't synthesize thread map.\n"); return err; @@ -2481,7 +2521,7 @@ int perf_event__synthesize_for_pipe(const struct perf_tool *tool, ret += err; #ifdef HAVE_LIBTRACEEVENT - if (have_tracepoints(&evlist->core.entries)) { + if (have_tracepoints(&evlist__core(evlist)->entries)) { int fd = perf_data__fd(data); /* diff --git a/tools/perf/util/synthetic-events.h b/tools/perf/util/synthetic-events.h index b0edad0c3100..243115ea61ae 100644 --- a/tools/perf/util/synthetic-events.h +++ b/tools/perf/util/synthetic-events.h @@ -50,7 +50,6 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, const struct build_id *bid, const char *filename); @@ -58,7 +57,6 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -81,7 +79,8 @@ int perf_event__synthesize_mmap_events(const struct perf_tool *tool, union perf_ int perf_event__synthesize_modules(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_namespaces(const struct perf_tool *tool, union perf_event *event, pid_t pid, pid_t tgid, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_cgroups(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); -int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, const struct perf_sample *sample); +int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, + u64 branch_sample_type, const struct perf_sample *sample); int perf_event__synthesize_stat_config(const struct perf_tool *tool, struct perf_stat_config *config, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_stat_events(struct perf_stat_config *config, const struct perf_tool *tool, struct evlist *evlist, perf_event__handler_t process, bool attrs); int perf_event__synthesize_stat_round(const struct perf_tool *tool, u64 time, u64 type, perf_event__handler_t process, struct machine *machine); @@ -97,7 +96,8 @@ void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, int perf_tool__process_synth_event(const struct perf_tool *tool, union perf_event *event, struct machine *machine, perf_event__handler_t process); -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format); +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, + u64 read_format, u64 branch_sample_type); int __machine__synthesize_threads(struct machine *machine, const struct perf_tool *tool, struct target *target, struct perf_thread_map *threads, diff --git a/tools/perf/util/thread.c b/tools/perf/util/thread.c index 22be77225bb0..e483ffcb5d93 100644 --- a/tools/perf/util/thread.c +++ b/tools/perf/util/thread.c @@ -56,6 +56,7 @@ struct thread *thread__new(pid_t pid, pid_t tid) thread__set_cpu(thread, -1); thread__set_guest_cpu(thread, -1); thread__set_e_machine(thread, EM_NONE); + thread__set_e_is_big_endian(thread, false); thread__set_lbr_stitch_enable(thread, false); INIT_LIST_HEAD(thread__namespaces_list(thread)); INIT_LIST_HEAD(thread__comm_list(thread)); @@ -294,6 +295,11 @@ int thread__set_comm_from_proc(struct thread *thread) if (!(snprintf(path, sizeof(path), "%d/task/%d/comm", thread__pid(thread), thread__tid(thread)) >= (int)sizeof(path)) && procfs__read_str(path, &comm, &sz) == 0) { + /* sz==0: read got nothing, e.g. race during exit teardown */ + if (sz == 0) { + free(comm); + return -1; + } comm[sz - 1] = '\0'; err = thread__set_comm(thread, comm, 0); } @@ -358,41 +364,21 @@ size_t thread__fprintf(struct thread *thread, FILE *fp) int thread__insert_map(struct thread *thread, struct map *map) { int ret; + uint16_t e_machine; - ret = unwind__prepare_access(thread__maps(thread), map, NULL); + ret = maps__fixup_overlap_and_insert(thread__maps(thread), map); if (ret) return ret; - return maps__fixup_overlap_and_insert(thread__maps(thread), map); -} - -struct thread__prepare_access_maps_cb_args { - int err; - struct maps *maps; -}; - -static int thread__prepare_access_maps_cb(struct map *map, void *data) -{ - bool initialized = false; - struct thread__prepare_access_maps_cb_args *args = data; - - args->err = unwind__prepare_access(args->maps, map, &initialized); - - return (args->err || initialized) ? 1 : 0; + e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); + return unwind__prepare_access(thread__maps(thread), e_machine); } static int thread__prepare_access(struct thread *thread) { - struct thread__prepare_access_maps_cb_args args = { - .err = 0, - }; + uint16_t e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); - if (dwarf_callchain_users) { - args.maps = thread__maps(thread); - maps__for_each_map(thread__maps(thread), thread__prepare_access_maps_cb, &args); - } - - return args.err; + return unwind__prepare_access(thread__maps(thread), e_machine); } static int thread__clone_maps(struct thread *thread, struct thread *parent, bool do_maps_clone) @@ -449,7 +435,7 @@ void thread__find_cpumode_addr_location(struct thread *thread, u64 addr, } } -static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags) +static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags, bool *is_big_endian) { char path[6 /* "/proc/" */ + 11 /* max length of pid */ + 5 /* "/exe\0" */]; int fd; @@ -458,7 +444,8 @@ static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags) snprintf(path, sizeof(path), "/proc/%d/exe", pid); fd = open(path, O_RDONLY); if (fd >= 0) { - e_machine = dso__read_e_machine(/*optional_dso=*/NULL, fd, e_flags); + e_machine = dso__read_e_machine_endian(/*optional_dso=*/NULL, fd, e_flags, + is_big_endian); close(fd); } return e_machine; @@ -468,6 +455,7 @@ struct thread__e_machine_callback_args { struct machine *machine; uint32_t e_flags; uint16_t e_machine; + bool is_big_endian; }; static int thread__e_machine_callback(struct map *map, void *_args) @@ -478,24 +466,38 @@ static int thread__e_machine_callback(struct map *map, void *_args) if (!dso) return 0; // No dso, continue search. - args->e_machine = dso__e_machine(dso, args->machine, &args->e_flags); + args->e_machine = + dso__e_machine_endian(dso, args->machine, &args->e_flags, &args->is_big_endian); return args->e_machine != EM_NONE ? 1 /* stop search */ : 0 /* continue search */; } -uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint32_t *e_flags) +uint16_t thread__e_machine_endian(struct thread *thread, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) { pid_t tid, pid; - uint16_t e_machine = RC_CHK_ACCESS(thread)->e_machine; + uint16_t e_machine; uint32_t local_e_flags = 0; - struct thread__e_machine_callback_args args = { - .machine = machine, - .e_flags = 0, - .e_machine = EM_NONE, - }; + struct thread__e_machine_callback_args args; + + if (!thread) { + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); + } + + e_machine = RC_CHK_ACCESS(thread)->e_machine; + args.machine = machine; + args.e_flags = 0; + args.e_machine = EM_NONE; + args.is_big_endian = false; if (e_machine != EM_NONE) { if (e_flags) *e_flags = thread__e_flags(thread); + if (is_big_endian) + *is_big_endian = thread__e_is_big_endian(thread); return e_machine; } @@ -503,6 +505,7 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 struct maps *maps = thread__maps(thread); machine = maps__machine(maps); + args.machine = machine; } tid = thread__tid(thread); pid = thread__pid(thread); @@ -510,7 +513,8 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 struct thread *parent = machine__findnew_thread(machine, pid, pid); if (parent) { - e_machine = thread__e_machine(parent, machine, &local_e_flags); + e_machine = thread__e_machine_endian(parent, machine, &local_e_flags, + &args.is_big_endian); thread__put(parent); goto out; } @@ -535,16 +539,27 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 is_live = !!session->data; } /* Read from /proc/pid/exe if live. */ - if (is_live) - e_machine = read_proc_e_machine_for_pid(pid, &local_e_flags); + if (is_live) { + e_machine = read_proc_e_machine_for_pid(pid, &local_e_flags, + &args.is_big_endian); + } else if (machine && machine->env) { + /* Offline analysis: fallback to environment metadata. */ + e_machine = perf_env__e_machine(machine->env, &local_e_flags); + args.is_big_endian = perf_arch_is_big_endian(perf_env__arch(machine->env)); + } } out: if (e_machine != EM_NONE) { - thread__set_e_machine(thread, e_machine); thread__set_e_flags(thread, local_e_flags); + thread__set_e_is_big_endian(thread, args.is_big_endian); + thread__set_e_machine(thread, e_machine); + if (is_big_endian) + *is_big_endian = args.is_big_endian; } else { e_machine = EM_HOST; local_e_flags = EF_HOST; + if (is_big_endian) + *is_big_endian = (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__); } if (e_flags) *e_flags = local_e_flags; diff --git a/tools/perf/util/thread.h b/tools/perf/util/thread.h index f5792d3e8a16..d82fce8173ae 100644 --- a/tools/perf/util/thread.h +++ b/tools/perf/util/thread.h @@ -69,6 +69,11 @@ DECLARE_RC_STRUCT(thread) { * computed. */ uint16_t e_machine; + /** + * @e_is_big_endian: True if the ELF architecture of the thread is big endian. + * Valid if e_machine != EM_NONE. + */ + bool e_is_big_endian; /* LBR call stack stitch */ bool lbr_stitch_enable; struct lbr_stitch *lbr_stitch; @@ -311,7 +316,13 @@ static inline void thread__set_filter_entry_depth(struct thread *thread, int dep RC_CHK_ACCESS(thread)->filter_entry_depth = depth; } -uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint32_t *e_flags); +uint16_t thread__e_machine_endian(struct thread *thread, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t thread__e_machine(struct thread *thread, struct machine *machine, + uint32_t *e_flags) +{ + return thread__e_machine_endian(thread, machine, e_flags, NULL); +} static inline void thread__set_e_machine(struct thread *thread, uint16_t e_machine) { @@ -328,6 +339,16 @@ static inline void thread__set_e_flags(struct thread *thread, uint32_t e_flags) RC_CHK_ACCESS(thread)->e_flags = e_flags; } +static inline bool thread__e_is_big_endian(const struct thread *thread) +{ + return RC_CHK_ACCESS(thread)->e_is_big_endian; +} + +static inline void thread__set_e_is_big_endian(struct thread *thread, bool is_big_endian) +{ + RC_CHK_ACCESS(thread)->e_is_big_endian = is_big_endian; +} + static inline bool thread__lbr_stitch_enable(const struct thread *thread) { diff --git a/tools/perf/util/time-utils.c b/tools/perf/util/time-utils.c index d43c4577d7eb..5558a5a0fea4 100644 --- a/tools/perf/util/time-utils.c +++ b/tools/perf/util/time-utils.c @@ -473,8 +473,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, return -ENOMEM; if (has_percent || reltime) { - if (session->evlist->first_sample_time == 0 && - session->evlist->last_sample_time == 0) { + if (evlist__first_sample_time(session->evlist) == 0 && + evlist__last_sample_time(session->evlist) == 0) { pr_err("HINT: no first/last sample time found in perf data.\n" "Please use latest perf binary to execute 'perf record'\n" "(if '--buildid-all' is enabled, please set '--timestamp-boundary').\n"); @@ -486,8 +486,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, num = perf_time__percent_parse_str( ptime_range, size, time_str, - session->evlist->first_sample_time, - session->evlist->last_sample_time); + evlist__first_sample_time(session->evlist), + evlist__last_sample_time(session->evlist)); } else { num = perf_time__parse_strs(ptime_range, time_str, size); } @@ -499,8 +499,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str, int i; for (i = 0; i < num; i++) { - ptime_range[i].start += session->evlist->first_sample_time; - ptime_range[i].end += session->evlist->first_sample_time; + ptime_range[i].start += evlist__first_sample_time(session->evlist); + ptime_range[i].end += evlist__first_sample_time(session->evlist); } } diff --git a/tools/perf/util/tool.c b/tools/perf/util/tool.c index 013c7839e2cf..25c9b378aa16 100644 --- a/tools/perf/util/tool.c +++ b/tools/perf/util/tool.c @@ -24,11 +24,32 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ size_t mmap_len, decomp_len = perf_session__env(session)->comp_mmap_len; struct decomp *decomp, *decomp_last = session->active_decomp->decomp_last; + if (!decomp_len) { + pr_err("Compressed events found but HEADER_COMPRESSED not set\n"); + return -1; + } + if (decomp_last) { + /* Prevent u64 underflow in decomp_last_rem */ + if (decomp_last->head > decomp_last->size) + return -1; decomp_last_rem = decomp_last->size - decomp_last->head; + /* + * Check before adding: on 32-bit, size_t += u64 + * silently truncates, bypassing the overflow check + * below and producing an undersized buffer. + */ + if (decomp_last_rem > SIZE_MAX - decomp_len - sizeof(struct decomp)) { + pr_err("Decompression buffer size overflow\n"); + return -1; + } decomp_len += decomp_last_rem; } + if (decomp_len > SIZE_MAX - sizeof(struct decomp)) { + pr_err("Decompression buffer size overflow\n"); + return -1; + } mmap_len = sizeof(struct decomp) + decomp_len; decomp = mmap(NULL, mmap_len, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); @@ -47,14 +68,37 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ decomp->size = decomp_last_rem; } + /* + * Events are read directly from the mmap'd file; fields could + * theoretically change via a FUSE-backed file, but that applies + * to the entire event processing pipeline, not just here. + */ if (event->header.type == PERF_RECORD_COMPRESSED) { + if (event->header.size < sizeof(struct perf_record_compressed)) + goto err_decomp; src = (void *)event + sizeof(struct perf_record_compressed); src_size = event->pack.header.size - sizeof(struct perf_record_compressed); } else if (event->header.type == PERF_RECORD_COMPRESSED2) { + /* + * prefetch_event() only guarantees that the 8-byte + * event header fits; validate that header.size covers + * the data_size field before accessing it, otherwise a + * crafted event reads data_size from adjacent memory. + */ + if (event->header.size < sizeof(struct perf_record_compressed2)) + goto err_decomp; src = (void *)event + sizeof(struct perf_record_compressed2); src_size = event->pack2.data_size; + /* + * data_size is independent of header.size (which + * includes padding); verify it doesn't exceed the + * actual payload to prevent out-of-bounds reads in + * zstd_decompress_stream(). + */ + if (src_size > event->header.size - sizeof(struct perf_record_compressed2)) + goto err_decomp; } else { - return -1; + goto err_decomp; } decomp_size = zstd_decompress_stream(session->active_decomp->zstd_decomp, src, src_size, @@ -77,6 +121,11 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ pr_debug("decomp (B): %zd to %zd\n", src_size, decomp_size); return 0; + +err_decomp: + munmap(decomp, mmap_len); + pr_err("Couldn't decompress data\n"); + return -1; } #endif @@ -110,7 +159,6 @@ static int process_event_synth_event_update_stub(const struct perf_tool *tool __ int process_event_sample_stub(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct machine *machine __maybe_unused) { dump_printf(": unhandled!\n"); @@ -285,6 +333,7 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events) tool->no_warn = false; tool->show_feat_hdr = SHOW_FEAT_NO_HEADER; tool->merge_deferred_callchains = true; + tool->dont_split_sample_group = false; tool->sample = process_event_sample_stub; tool->mmap = process_event_stub; @@ -348,12 +397,11 @@ bool perf_tool__compressed_is_stub(const struct perf_tool *tool) static int delegate_ ## name(const struct perf_tool *tool, \ union perf_event *event, \ struct perf_sample *sample, \ - struct evsel *evsel, \ struct machine *machine) \ { \ struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); \ struct perf_tool *delegate = del_tool->delegate; \ - return delegate->name(delegate, event, sample, evsel, machine); \ + return delegate->name(delegate, event, sample, machine); \ } CREATE_DELEGATE_SAMPLE(read); CREATE_DELEGATE_SAMPLE(sample); @@ -433,6 +481,8 @@ CREATE_DELEGATE_OP2(stat_config); CREATE_DELEGATE_OP2(stat_round); CREATE_DELEGATE_OP2(thread_map); CREATE_DELEGATE_OP2(time_conv); +CREATE_DELEGATE_OP2(schedstat_cpu); +CREATE_DELEGATE_OP2(schedstat_domain); CREATE_DELEGATE_OP2(tracing_data); #define CREATE_DELEGATE_OP3(name) \ @@ -470,6 +520,7 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.no_warn = delegate->no_warn; tool->tool.show_feat_hdr = delegate->show_feat_hdr; tool->tool.merge_deferred_callchains = delegate->merge_deferred_callchains; + tool->tool.dont_split_sample_group = delegate->dont_split_sample_group; tool->tool.sample = delegate_sample; tool->tool.read = delegate_read; @@ -516,4 +567,6 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.bpf_metadata = delegate_bpf_metadata; tool->tool.compressed = delegate_compressed; tool->tool.auxtrace = delegate_auxtrace; + tool->tool.schedstat_cpu = delegate_schedstat_cpu; + tool->tool.schedstat_domain = delegate_schedstat_domain; } diff --git a/tools/perf/util/tool.h b/tools/perf/util/tool.h index 2d9a4b1ca9d0..2a4f124ffd8d 100644 --- a/tools/perf/util/tool.h +++ b/tools/perf/util/tool.h @@ -9,7 +9,6 @@ struct perf_session; union perf_event; struct evlist; -struct evsel; struct perf_sample; struct perf_tool; struct machine; @@ -17,7 +16,7 @@ struct ordered_events; typedef int (*event_sample)(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine); + struct machine *machine); typedef int (*event_op)(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, struct machine *machine); @@ -103,7 +102,6 @@ bool perf_tool__compressed_is_stub(const struct perf_tool *tool); int process_event_sample_stub(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine); struct delegate_tool { diff --git a/tools/perf/util/tool_pmu.c b/tools/perf/util/tool_pmu.c index 6a9df3dc0e07..5c30854b4644 100644 --- a/tools/perf/util/tool_pmu.c +++ b/tools/perf/util/tool_pmu.c @@ -17,6 +17,8 @@ #include #include +#define INVALID_START_TIME ~0ULL + static const char *const tool_pmu__event_names[TOOL_PMU__EVENT_MAX] = { NULL, "duration_time", @@ -205,20 +207,57 @@ int evsel__tool_pmu_prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, int nthreads) { - if ((evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || - evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) && - !evsel->start_times) { - evsel->start_times = xyarray__new(perf_cpu_map__nr(cpus), - nthreads, - sizeof(__u64)); - if (!evsel->start_times) - return -ENOMEM; + enum tool_pmu_event ev = evsel__tool_event(evsel); + + if (ev == TOOL_PMU__EVENT_SYSTEM_TIME || ev == TOOL_PMU__EVENT_USER_TIME) { + if (!evsel->process_time.start_times) { + evsel->process_time.start_times = + xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); + if (!evsel->process_time.start_times) + return -ENOMEM; + } + if (!evsel->process_time.accumulated_times) { + evsel->process_time.accumulated_times = + xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); + if (!evsel->process_time.accumulated_times) + return -ENOMEM; + } } return 0; } #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) +static int tool_pmu__read_stat(struct evsel *evsel, int cpu_map_idx, int thread, __u64 *val) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + bool system = ev == TOOL_PMU__EVENT_SYSTEM_TIME; + int fd = FD(evsel, cpu_map_idx, thread); + int err = 0; + + if (fd < 0) { + *val = 0; + return 0; + } + + lseek(fd, 0, SEEK_SET); + if (evsel->pid_stat) { + if (cpu_map_idx == 0) + err = read_pid_stat_field(fd, system ? 15 : 14, val); + else + *val = 0; + } else { + if (thread == 0) { + struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx); + + err = read_stat_field(fd, cpu, system ? 3 : 1, val); + } else { + *val = 0; + } + } + return err; +} + int evsel__tool_pmu_open(struct evsel *evsel, struct perf_thread_map *threads, int start_cpu_map_idx, int end_cpu_map_idx) @@ -232,7 +271,14 @@ int evsel__tool_pmu_open(struct evsel *evsel, if (ev == TOOL_PMU__EVENT_DURATION_TIME) { if (evsel->core.attr.sample_period) /* no sampling */ return -EINVAL; - evsel->start_time = rdclock(); + evsel->duration_time.accumulated_time = 0; + if (evsel->core.attr.disabled) { + evsel->disabled = true; + evsel->duration_time.start_time = INVALID_START_TIME; + } else { + evsel->disabled = false; + evsel->duration_time.start_time = rdclock(); + } return 0; } @@ -246,8 +292,8 @@ int evsel__tool_pmu_open(struct evsel *evsel, pid = perf_thread_map__pid(threads, thread); if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { - bool system = ev == TOOL_PMU__EVENT_SYSTEM_TIME; __u64 *start_time = NULL; + __u64 *accumulated_time = NULL; int fd; if (evsel->core.attr.sample_period) { @@ -269,21 +315,25 @@ int evsel__tool_pmu_open(struct evsel *evsel, err = -errno; goto out_close; } - start_time = xyarray__entry(evsel->start_times, idx, thread); - if (pid > -1) { - err = read_pid_stat_field(fd, system ? 15 : 14, - start_time); + start_time = xyarray__entry(evsel->process_time.start_times, idx, + thread); + accumulated_time = xyarray__entry( + evsel->process_time.accumulated_times, idx, thread); + *accumulated_time = 0; + + if (evsel->core.attr.disabled) { + evsel->disabled = true; + *start_time = INVALID_START_TIME; } else { - struct perf_cpu cpu; - - cpu = perf_cpu_map__cpu(evsel->core.cpus, idx); - err = read_stat_field(fd, cpu, system ? 3 : 1, - start_time); + evsel->disabled = false; + err = tool_pmu__read_stat(evsel, idx, thread, start_time); + if (err) { + close(fd); + FD(evsel, idx, thread) = -1; + goto out_close; + } } - if (err) - goto out_close; } - } } return 0; @@ -467,10 +517,111 @@ static void perf_counts__update(struct perf_counts_values *count, count->lost = 0; } } +int evsel__tool_pmu_enable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + int thread, nthreads; + + if (!evsel->disabled) + return 0; + + if (ev == TOOL_PMU__EVENT_DURATION_TIME) { + if (cpu_map_idx == 0) + evsel->duration_time.start_time = rdclock(); + return 0; + } + + if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { + nthreads = xyarray__max_y(evsel->process_time.start_times); + for (thread = 0; thread < nthreads; thread++) { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 val; + int err; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &val); + if (!err) + *start_time = val; + else + *start_time = INVALID_START_TIME; + } + } + return 0; +} + +int evsel__tool_pmu_enable(struct evsel *evsel) +{ + unsigned int idx; + int err = 0; + + if (!evsel->disabled) + return 0; + + for (idx = 0; idx < perf_cpu_map__nr(evsel->core.cpus); idx++) { + err = evsel__tool_pmu_enable_cpu(evsel, idx); + if (err) + break; + } + return err; +} + +int evsel__tool_pmu_disable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + int thread, nthreads; + + if (evsel->disabled) + return 0; + + if (ev == TOOL_PMU__EVENT_DURATION_TIME) { + if (cpu_map_idx == 0) { + __u64 delta = rdclock() - evsel->duration_time.start_time; + + evsel->duration_time.accumulated_time += delta; + } + return 0; + } + + if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { + nthreads = xyarray__max_y(evsel->process_time.start_times); + for (thread = 0; thread < nthreads; thread++) { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 *accumulated_time = xyarray__entry( + evsel->process_time.accumulated_times, cpu_map_idx, thread); + __u64 val; + int err; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &val); + if (!err) { + if (*start_time != INVALID_START_TIME && val >= *start_time) + *accumulated_time += (val - *start_time); + } + *start_time = INVALID_START_TIME; + } + } + return 0; +} + +int evsel__tool_pmu_disable(struct evsel *evsel) +{ + unsigned int idx; + int err = 0; + + if (evsel->disabled) + return 0; + + for (idx = 0; idx < perf_cpu_map__nr(evsel->core.cpus); idx++) { + err = evsel__tool_pmu_disable_cpu(evsel, idx); + if (err) + break; + } + return err; +} int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) { - __u64 *start_time, cur_time, delta_start; + __u64 delta_start = 0; int err = 0; struct perf_counts_values *count, *old_count = NULL; bool adjust = false; @@ -507,39 +658,33 @@ int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) return 0; } case TOOL_PMU__EVENT_DURATION_TIME: - /* - * Pretend duration_time is only on the first CPU and thread, or - * else aggregation will scale duration_time by the number of - * CPUs/threads. - */ - start_time = &evsel->start_time; - if (cpu_map_idx == 0 && thread == 0) - cur_time = rdclock(); - else - cur_time = *start_time; + if (cpu_map_idx == 0 && thread == 0) { + delta_start = evsel->duration_time.accumulated_time; + if (!evsel->disabled && + evsel->duration_time.start_time != INVALID_START_TIME) + delta_start += (rdclock() - evsel->duration_time.start_time); + } else { + delta_start = 0; + } break; case TOOL_PMU__EVENT_USER_TIME: case TOOL_PMU__EVENT_SYSTEM_TIME: { - bool system = evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME; - int fd = FD(evsel, cpu_map_idx, thread); - - start_time = xyarray__entry(evsel->start_times, cpu_map_idx, thread); - lseek(fd, SEEK_SET, 0); - if (evsel->pid_stat) { - /* The event exists solely on 1 CPU. */ - if (cpu_map_idx == 0) - err = read_pid_stat_field(fd, system ? 15 : 14, &cur_time); - else - cur_time = 0; - } else { - /* The event is for all threads. */ - if (thread == 0) { - struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, - cpu_map_idx); + __u64 accumulated = *(__u64 *)xyarray__entry(evsel->process_time.accumulated_times, + cpu_map_idx, thread); - err = read_stat_field(fd, cpu, system ? 3 : 1, &cur_time); - } else { - cur_time = 0; + if (evsel->disabled) { + delta_start = accumulated; + } else { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 cur_time; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &cur_time); + if (!err) { + if (*start_time != INVALID_START_TIME && cur_time >= *start_time) + delta_start = accumulated + (cur_time - *start_time); + else + delta_start = accumulated; } } adjust = true; @@ -553,7 +698,6 @@ int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) if (err) return err; - delta_start = cur_time - *start_time; if (adjust) { __u64 ticks_per_sec = sysconf(_SC_CLK_TCK); diff --git a/tools/perf/util/tool_pmu.h b/tools/perf/util/tool_pmu.h index f1714001bc1d..f66d24cf3502 100644 --- a/tools/perf/util/tool_pmu.h +++ b/tools/perf/util/tool_pmu.h @@ -56,6 +56,10 @@ int evsel__tool_pmu_prepare_open(struct evsel *evsel, int evsel__tool_pmu_open(struct evsel *evsel, struct perf_thread_map *threads, int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__tool_pmu_enable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__tool_pmu_enable(struct evsel *evsel); +int evsel__tool_pmu_disable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__tool_pmu_disable(struct evsel *evsel); int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); struct perf_pmu *tool_pmu__new(void); diff --git a/tools/perf/util/top.c b/tools/perf/util/top.c index b06e10a116bb..851a26be6931 100644 --- a/tools/perf/util/top.c +++ b/tools/perf/util/top.c @@ -71,7 +71,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) esamples_percent); } - if (top->evlist->core.nr_entries == 1) { + if (evlist__nr_entries(top->evlist) == 1) { struct evsel *first = evlist__first(top->evlist); ret += SNPRINTF(bf + ret, size - ret, "%" PRIu64 "%s ", (uint64_t)first->core.attr.sample_period, @@ -94,7 +94,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size) else ret += SNPRINTF(bf + ret, size - ret, " (all"); - nr_cpus = perf_cpu_map__nr(top->evlist->core.user_requested_cpus); + nr_cpus = perf_cpu_map__nr(evlist__core(top->evlist)->user_requested_cpus); if (target->cpu_list) ret += SNPRINTF(bf + ret, size - ret, ", CPU%s: %s)", nr_cpus > 1 ? "s" : "", diff --git a/tools/perf/util/trace-event-scripting.c b/tools/perf/util/trace-event-scripting.c index fa850e44cb46..dc584ac316a3 100644 --- a/tools/perf/util/trace-event-scripting.c +++ b/tools/perf/util/trace-event-scripting.c @@ -103,12 +103,11 @@ int script_spec__for_each(int (*cb)(struct scripting_ops *ops, const char *spec) void scripting_context__update(struct scripting_context *c, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { #ifdef HAVE_LIBTRACEEVENT - const struct tep_event *tp_format = evsel__tp_format(evsel); + const struct tep_event *tp_format = evsel__tp_format(sample->evsel); c->pevent = tp_format ? tp_format->tep : NULL; #else @@ -117,7 +116,6 @@ void scripting_context__update(struct scripting_context *c, c->event_data = sample->raw_data; c->event = event; c->sample = sample; - c->evsel = evsel; c->al = al; c->addr_al = addr_al; } @@ -134,7 +132,6 @@ static int stop_script_unsupported(void) static void process_event_unsupported(union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct addr_location *al __maybe_unused, struct addr_location *addr_al __maybe_unused) { diff --git a/tools/perf/util/trace-event.h b/tools/perf/util/trace-event.h index 914d9b69ed62..720121c74f1d 100644 --- a/tools/perf/util/trace-event.h +++ b/tools/perf/util/trace-event.h @@ -94,7 +94,6 @@ struct scripting_ops { int (*stop_script) (void); void (*process_event) (union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al); void (*process_switch)(union perf_event *event, @@ -124,7 +123,6 @@ struct scripting_context { void *event_data; union perf_event *event; struct perf_sample *sample; - struct evsel *evsel; struct addr_location *al; struct addr_location *addr_al; struct perf_session *session; @@ -133,7 +131,6 @@ struct scripting_context { void scripting_context__update(struct scripting_context *scripting_context, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al); diff --git a/tools/perf/util/tsc.c b/tools/perf/util/tsc.c index 511a517ce613..ebf289bf6b9d 100644 --- a/tools/perf/util/tsc.c +++ b/tools/perf/util/tsc.c @@ -127,7 +127,7 @@ size_t perf_event__fprintf_time_conv(union perf_event *event, FILE *fp) * when supported cap_user_time_short, for backward compatibility, * prints the extended fields only if they are contained in the event. */ - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { ret += fprintf(fp, "... Time Cycles %" PRI_lu64 "\n", tc->time_cycles); ret += fprintf(fp, "... Time Mask %#" PRI_lx64 "\n", diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index 05e8e68bd49c..7f35042be567 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -339,7 +339,7 @@ frame_callback(Dwfl_Frame *state, void *arg) DWARF_CB_ABORT : DWARF_CB_OK; } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, @@ -353,10 +353,10 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, static struct unwind_info *ui; Dwfl *dwfl; Dwarf_Word ip; - int err = -EINVAL, i; + int err = -EINVAL, i, entries; if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; + return 0; ui = zalloc(sizeof(*ui) + sizeof(ui->entries[0]) * max_stack); if (!ui) @@ -430,6 +430,18 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, map_symbol__exit(&ui->entries[i].ms); dwfl_ui_ti->ui = NULL; + entries = (int)ui->idx; free(ui); - return 0; + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). If we found frames but hit an error + * (e.g. truncated stack), report success to preserve existing frames. + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. missing debug info, DWARF corruption) to allow fallback unwinder or + * kernel callchain resolution to proceed. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (err) + return (err == -ENOMEM) ? -ENOMEM : (entries > 0 ? 1 : 0); + return entries; } diff --git a/tools/perf/util/unwind-libunwind-local.c b/tools/perf/util/unwind-libunwind-local.c deleted file mode 100644 index 87d496e9dfa6..000000000000 --- a/tools/perf/util/unwind-libunwind-local.c +++ /dev/null @@ -1,831 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Post mortem Dwarf CFI based unwinding on top of regs and stack dumps. - * - * Lots of this code have been borrowed or heavily inspired from parts of - * the libunwind 0.99 code which are (amongst other contributors I may have - * forgotten): - * - * Copyright (C) 2002-2007 Hewlett-Packard Co - * Contributed by David Mosberger-Tang - * - * And the bugs have been added by: - * - * Copyright (C) 2010, Frederic Weisbecker - * Copyright (C) 2012, Jiri Olsa - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef REMOTE_UNWIND_LIBUNWIND -#include -#include -#endif -#include "callchain.h" -#include "thread.h" -#include "session.h" -#include "perf_regs.h" -#include "unwind.h" -#include "map.h" -#include "symbol.h" -#include "debug.h" -#include "asm/bug.h" -#include "dso.h" - -extern int -UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -extern int -UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, - unw_word_t ip, - unw_word_t segbase, - const char *obj_name, unw_word_t start, - unw_word_t end); - -#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) - -#define DW_EH_PE_FORMAT_MASK 0x0f /* format of the encoded value */ -#define DW_EH_PE_APPL_MASK 0x70 /* how the value is to be applied */ - -/* Pointer-encoding formats: */ -#define DW_EH_PE_omit 0xff -#define DW_EH_PE_ptr 0x00 /* pointer-sized unsigned value */ -#define DW_EH_PE_udata4 0x03 /* unsigned 32-bit value */ -#define DW_EH_PE_udata8 0x04 /* unsigned 64-bit value */ -#define DW_EH_PE_sdata4 0x0b /* signed 32-bit value */ -#define DW_EH_PE_sdata8 0x0c /* signed 64-bit value */ - -/* Pointer-encoding application: */ -#define DW_EH_PE_absptr 0x00 /* absolute value */ -#define DW_EH_PE_pcrel 0x10 /* rel. to addr. of encoded value */ - -/* - * The following are not documented by LSB v1.3, yet they are used by - * GCC, presumably they aren't documented by LSB since they aren't - * used on Linux: - */ -#define DW_EH_PE_funcrel 0x40 /* start-of-procedure-relative */ -#define DW_EH_PE_aligned 0x50 /* aligned pointer */ - -/* Flags intentionally not handled, since they're not needed: - * #define DW_EH_PE_indirect 0x80 - * #define DW_EH_PE_uleb128 0x01 - * #define DW_EH_PE_udata2 0x02 - * #define DW_EH_PE_sleb128 0x09 - * #define DW_EH_PE_sdata2 0x0a - * #define DW_EH_PE_textrel 0x20 - * #define DW_EH_PE_datarel 0x30 - */ - -struct unwind_info { - struct perf_sample *sample; - struct machine *machine; - struct thread *thread; - bool best_effort; -}; - -#define dw_read(ptr, type, end) ({ \ - type *__p = (type *) ptr; \ - type __v; \ - if ((__p + 1) > (type *) end) \ - return -EINVAL; \ - __v = *__p++; \ - ptr = (typeof(ptr)) __p; \ - __v; \ - }) - -static int __dw_read_encoded_value(u8 **p, u8 *end, u64 *val, - u8 encoding) -{ - u8 *cur = *p; - *val = 0; - - switch (encoding) { - case DW_EH_PE_omit: - *val = 0; - goto out; - case DW_EH_PE_ptr: - *val = dw_read(cur, unsigned long, end); - goto out; - default: - break; - } - - switch (encoding & DW_EH_PE_APPL_MASK) { - case DW_EH_PE_absptr: - break; - case DW_EH_PE_pcrel: - *val = (unsigned long) cur; - break; - default: - return -EINVAL; - } - - if ((encoding & 0x07) == 0x00) - encoding |= DW_EH_PE_udata4; - - switch (encoding & DW_EH_PE_FORMAT_MASK) { - case DW_EH_PE_sdata4: - *val += dw_read(cur, s32, end); - break; - case DW_EH_PE_udata4: - *val += dw_read(cur, u32, end); - break; - case DW_EH_PE_sdata8: - *val += dw_read(cur, s64, end); - break; - case DW_EH_PE_udata8: - *val += dw_read(cur, u64, end); - break; - default: - return -EINVAL; - } - - out: - *p = cur; - return 0; -} - -#define dw_read_encoded_value(ptr, end, enc) ({ \ - u64 __v; \ - if (__dw_read_encoded_value(&ptr, end, &__v, enc)) { \ - return -EINVAL; \ - } \ - __v; \ - }) - -static int elf_section_address_and_offset(int fd, const char *name, u64 *address, u64 *offset) -{ - Elf *elf; - GElf_Ehdr ehdr; - GElf_Shdr shdr; - int ret = -1; - - elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - if (elf == NULL) - return -1; - - if (gelf_getehdr(elf, &ehdr) == NULL) - goto out_err; - - if (!elf_section_by_name(elf, &ehdr, &shdr, name, NULL)) - goto out_err; - - *address = shdr.sh_addr; - *offset = shdr.sh_offset; - ret = 0; -out_err: - elf_end(elf); - return ret; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static u64 elf_section_offset(int fd, const char *name) -{ - u64 address, offset = 0; - - if (elf_section_address_and_offset(fd, name, &address, &offset)) - return 0; - - return offset; -} -#endif - -static u64 elf_base_address(int fd) -{ - Elf *elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - GElf_Phdr phdr; - u64 retval = 0; - size_t i, phdrnum = 0; - - if (elf == NULL) - return 0; - (void)elf_getphdrnum(elf, &phdrnum); - /* PT_LOAD segments are sorted by p_vaddr, so the first has the minimum p_vaddr. */ - for (i = 0; i < phdrnum; i++) { - if (gelf_getphdr(elf, i, &phdr) && phdr.p_type == PT_LOAD) { - retval = phdr.p_vaddr & -getpagesize(); - break; - } - } - - elf_end(elf); - return retval; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static int elf_is_exec(int fd, const char *name) -{ - Elf *elf; - GElf_Ehdr ehdr; - int retval = 0; - - elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - if (elf == NULL) - return 0; - if (gelf_getehdr(elf, &ehdr) == NULL) - goto out; - - retval = (ehdr.e_type == ET_EXEC); - -out: - elf_end(elf); - pr_debug("unwind: elf_is_exec(%s): %d\n", name, retval); - return retval; -} -#endif - -struct table_entry { - u32 start_ip_offset; - u32 fde_offset; -}; - -struct eh_frame_hdr { - unsigned char version; - unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; - - /* - * The rest of the header is variable-length and consists of the - * following members: - * - * encoded_t eh_frame_ptr; - * encoded_t fde_count; - */ - - /* A single encoded pointer should not be more than 8 bytes. */ - u64 enc[2]; - - /* - * struct { - * encoded_t start_ip; - * encoded_t fde_addr; - * } binary_search_table[fde_count]; - */ - char data[]; -} __packed; - -static int unwind_spec_ehframe(struct dso *dso, struct machine *machine, - u64 offset, u64 *table_data_offset, u64 *fde_count) -{ - struct eh_frame_hdr hdr; - u8 *enc = (u8 *) &hdr.enc; - u8 *end = (u8 *) &hdr.data; - ssize_t r; - - r = dso__data_read_offset(dso, machine, offset, - (u8 *) &hdr, sizeof(hdr)); - if (r != sizeof(hdr)) - return -EINVAL; - - /* We dont need eh_frame_ptr, just skip it. */ - dw_read_encoded_value(enc, end, hdr.eh_frame_ptr_enc); - - *fde_count = dw_read_encoded_value(enc, end, hdr.fde_count_enc); - *table_data_offset = enc - (u8 *) &hdr; - return 0; -} - -struct read_unwind_spec_eh_frame_maps_cb_args { - struct dso *dso; - u64 base_addr; -}; - -static int read_unwind_spec_eh_frame_maps_cb(struct map *map, void *data) -{ - - struct read_unwind_spec_eh_frame_maps_cb_args *args = data; - - if (map__dso(map) == args->dso && map__start(map) - map__pgoff(map) < args->base_addr) - args->base_addr = map__start(map) - map__pgoff(map); - - return 0; -} - - -static int read_unwind_spec_eh_frame(struct dso *dso, struct unwind_info *ui, - u64 *table_data, u64 *segbase, - u64 *fde_count) -{ - struct read_unwind_spec_eh_frame_maps_cb_args args = { - .dso = dso, - .base_addr = UINT64_MAX, - }; - int ret, fd; - - if (dso__data(dso)->eh_frame_hdr_offset == 0) { - if (!dso__data_get_fd(dso, ui->machine, &fd)) - return -EINVAL; - - /* Check the .eh_frame section for unwinding info */ - ret = elf_section_address_and_offset(fd, ".eh_frame_hdr", - &dso__data(dso)->eh_frame_hdr_addr, - &dso__data(dso)->eh_frame_hdr_offset); - dso__data(dso)->elf_base_addr = elf_base_address(fd); - dso__data_put_fd(dso); - if (ret || dso__data(dso)->eh_frame_hdr_offset == 0) - return -EINVAL; - } - - maps__for_each_map(thread__maps(ui->thread), read_unwind_spec_eh_frame_maps_cb, &args); - - args.base_addr -= dso__data(dso)->elf_base_addr; - /* Address of .eh_frame_hdr */ - *segbase = args.base_addr + dso__data(dso)->eh_frame_hdr_addr; - ret = unwind_spec_ehframe(dso, ui->machine, dso__data(dso)->eh_frame_hdr_offset, - table_data, fde_count); - if (ret) - return ret; - /* binary_search_table offset plus .eh_frame_hdr address */ - *table_data += *segbase; - return 0; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static int read_unwind_spec_debug_frame(struct dso *dso, - struct machine *machine, u64 *offset) -{ - int fd; - u64 ofs = dso__data(dso)->debug_frame_offset; - - /* debug_frame can reside in: - * - dso - * - debug pointed by symsrc_filename - * - gnu_debuglink, which doesn't necessary - * has to be pointed by symsrc_filename - */ - if (ofs == 0) { - if (dso__data_get_fd(dso, machine, &fd)) { - ofs = elf_section_offset(fd, ".debug_frame"); - dso__data_put_fd(dso); - } - - if (ofs <= 0) { - fd = open(dso__symsrc_filename(dso), O_RDONLY); - if (fd >= 0) { - ofs = elf_section_offset(fd, ".debug_frame"); - close(fd); - } - } - - if (ofs <= 0) { - char *debuglink = malloc(PATH_MAX); - int ret = 0; - - if (debuglink == NULL) { - pr_err("unwind: Can't read unwind spec debug frame.\n"); - return -ENOMEM; - } - - ret = dso__read_binary_type_filename( - dso, DSO_BINARY_TYPE__DEBUGLINK, - machine->root_dir, debuglink, PATH_MAX); - if (!ret) { - fd = open(debuglink, O_RDONLY); - if (fd >= 0) { - ofs = elf_section_offset(fd, - ".debug_frame"); - close(fd); - } - } - if (ofs > 0) { - if (dso__symsrc_filename(dso) != NULL) { - pr_warning( - "%s: overwrite symsrc(%s,%s)\n", - __func__, - dso__symsrc_filename(dso), - debuglink); - dso__free_symsrc_filename(dso); - } - dso__set_symsrc_filename(dso, debuglink); - } else { - free(debuglink); - } - } - - dso__data(dso)->debug_frame_offset = ofs; - } - - *offset = ofs; - if (*offset) - return 0; - - return -EINVAL; -} -#endif - -static struct map *find_map(unw_word_t ip, struct unwind_info *ui) -{ - struct addr_location al; - struct map *ret; - - addr_location__init(&al); - thread__find_map(ui->thread, PERF_RECORD_MISC_USER, ip, &al); - ret = map__get(al.map); - addr_location__exit(&al); - return ret; -} - -static int -find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, - int need_unwind_info, void *arg) -{ - struct unwind_info *ui = arg; - struct map *map; - struct dso *dso; - unw_dyn_info_t di; - u64 table_data, segbase, fde_count; - int ret = -EINVAL; - - map = find_map(ip, ui); - if (!map) - return -EINVAL; - - dso = map__dso(map); - if (!dso) { - map__put(map); - return -EINVAL; - } - - pr_debug("unwind: find_proc_info dso %s\n", dso__name(dso)); - - /* Check the .eh_frame section for unwinding info */ - if (!read_unwind_spec_eh_frame(dso, ui, &table_data, &segbase, &fde_count)) { - memset(&di, 0, sizeof(di)); - di.format = UNW_INFO_FORMAT_REMOTE_TABLE; - di.start_ip = map__start(map); - di.end_ip = map__end(map); - di.u.rti.segbase = segbase; - di.u.rti.table_data = table_data; - di.u.rti.table_len = fde_count * sizeof(struct table_entry) - / sizeof(unw_word_t); - ret = dwarf_search_unwind_table(as, ip, &di, pi, - need_unwind_info, arg); - } - -#ifndef NO_LIBUNWIND_DEBUG_FRAME - /* Check the .debug_frame section for unwinding info */ - if (ret < 0 && - !read_unwind_spec_debug_frame(dso, ui->machine, &segbase)) { - int fd; - u64 start = map__start(map); - unw_word_t base = start; - const char *symfile; - - if (dso__data_get_fd(dso, ui->machine, &fd)) { - if (elf_is_exec(fd, dso__name(dso))) - base = 0; - dso__data_put_fd(dso); - } - - symfile = dso__symsrc_filename(dso) ?: dso__name(dso); - - memset(&di, 0, sizeof(di)); - if (dwarf_find_debug_frame(0, &di, ip, base, symfile, start, map__end(map))) - ret = dwarf_search_unwind_table(as, ip, &di, pi, - need_unwind_info, arg); - } -#endif - map__put(map); - return ret; -} - -static int access_fpreg(unw_addr_space_t __maybe_unused as, - unw_regnum_t __maybe_unused num, - unw_fpreg_t __maybe_unused *val, - int __maybe_unused __write, - void __maybe_unused *arg) -{ - pr_err("unwind: access_fpreg unsupported\n"); - return -UNW_EINVAL; -} - -static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, - unw_word_t __maybe_unused *dil_addr, - void __maybe_unused *arg) -{ - return -UNW_ENOINFO; -} - -static int resume(unw_addr_space_t __maybe_unused as, - unw_cursor_t __maybe_unused *cu, - void __maybe_unused *arg) -{ - pr_err("unwind: resume unsupported\n"); - return -UNW_EINVAL; -} - -static int -get_proc_name(unw_addr_space_t __maybe_unused as, - unw_word_t __maybe_unused addr, - char __maybe_unused *bufp, size_t __maybe_unused buf_len, - unw_word_t __maybe_unused *offp, void __maybe_unused *arg) -{ - pr_err("unwind: get_proc_name unsupported\n"); - return -UNW_EINVAL; -} - -static int access_dso_mem(struct unwind_info *ui, unw_word_t addr, - unw_word_t *data) -{ - struct map *map; - struct dso *dso; - ssize_t size; - - map = find_map(addr, ui); - if (!map) { - pr_debug("unwind: no map for %lx\n", (unsigned long)addr); - return -1; - } - - dso = map__dso(map); - - if (!dso) { - map__put(map); - return -1; - } - - size = dso__data_read_addr(dso, map, ui->machine, - addr, (u8 *) data, sizeof(*data)); - map__put(map); - return !(size == sizeof(*data)); -} - -static int access_mem(unw_addr_space_t __maybe_unused as, - unw_word_t addr, unw_word_t *valp, - int __write, void *arg) -{ - struct unwind_info *ui = arg; - struct stack_dump *stack = &ui->sample->user_stack; - u64 start, end; - int offset; - int ret; - - /* Don't support write, probably not needed. */ - if (__write || !stack || !ui->sample->user_regs || !ui->sample->user_regs->regs) { - *valp = 0; - return 0; - } - - ret = perf_reg_value(&start, perf_sample__user_regs(ui->sample), - perf_arch_reg_sp(thread__e_machine(ui->thread, - ui->machine, - /*e_flags=*/NULL))); - if (ret) - return ret; - - end = start + stack->size; - - /* Check overflow. */ - if (addr + sizeof(unw_word_t) < addr) - return -EINVAL; - - if (addr < start || addr + sizeof(unw_word_t) >= end) { - ret = access_dso_mem(ui, addr, valp); - if (ret) { - pr_debug("unwind: access_mem %p not inside range" - " 0x%" PRIx64 "-0x%" PRIx64 "\n", - (void *) (uintptr_t) addr, start, end); - *valp = 0; - return ret; - } - return 0; - } - - offset = addr - start; - *valp = *(unw_word_t *)&stack->data[offset]; - pr_debug("unwind: access_mem addr %p val %lx, offset %d\n", - (void *) (uintptr_t) addr, (unsigned long)*valp, offset); - return 0; -} - -static int access_reg(unw_addr_space_t __maybe_unused as, - unw_regnum_t regnum, unw_word_t *valp, - int __write, void *arg) -{ - struct unwind_info *ui = arg; - int id, ret; - u64 val; - - /* Don't support write, I suspect we don't need it. */ - if (__write) { - pr_err("unwind: access_reg w %d\n", regnum); - return 0; - } - - if (!ui->sample->user_regs || !ui->sample->user_regs->regs) { - *valp = 0; - return 0; - } - - id = LIBUNWIND__ARCH_REG_ID(regnum); - if (id < 0) - return -EINVAL; - - ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), id); - if (ret) { - if (!ui->best_effort) - pr_err("unwind: can't read reg %d\n", regnum); - return ret; - } - - *valp = (unw_word_t) val; - pr_debug("unwind: reg %d, val %lx\n", regnum, (unsigned long)*valp); - return 0; -} - -static void put_unwind_info(unw_addr_space_t __maybe_unused as, - unw_proc_info_t *pi __maybe_unused, - void *arg __maybe_unused) -{ - pr_debug("unwind: put_unwind_info called\n"); -} - -static int entry(u64 ip, struct thread *thread, - unwind_entry_cb_t cb, void *arg) -{ - struct unwind_entry e; - struct addr_location al; - int ret; - - addr_location__init(&al); - e.ms.sym = thread__find_symbol(thread, PERF_RECORD_MISC_USER, ip, &al); - e.ip = ip; - e.ms.map = al.map; - e.ms.thread = thread__get(al.thread); - - pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", - al.sym ? al.sym->name : "''", - ip, - al.map ? map__map_ip(al.map, ip) : (u64) 0); - - ret = cb(&e, arg); - addr_location__exit(&al); - return ret; -} - -static void display_error(int err) -{ - switch (err) { - case UNW_EINVAL: - pr_err("unwind: Only supports local.\n"); - break; - case UNW_EUNSPEC: - pr_err("unwind: Unspecified error.\n"); - break; - case UNW_EBADREG: - pr_err("unwind: Register unavailable.\n"); - break; - default: - break; - } -} - -static unw_accessors_t accessors = { - .find_proc_info = find_proc_info, - .put_unwind_info = put_unwind_info, - .get_dyn_info_list_addr = get_dyn_info_list_addr, - .access_mem = access_mem, - .access_reg = access_reg, - .access_fpreg = access_fpreg, - .resume = resume, - .get_proc_name = get_proc_name, -}; - -static int _unwind__prepare_access(struct maps *maps) -{ - void *addr_space = unw_create_addr_space(&accessors, 0); - - maps__set_addr_space(maps, addr_space); - if (!addr_space) { - pr_err("unwind: Can't create unwind address space.\n"); - return -ENOMEM; - } - - unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); - return 0; -} - -static void _unwind__flush_access(struct maps *maps) -{ - unw_flush_cache(maps__addr_space(maps), 0, 0); -} - -static void _unwind__finish_access(struct maps *maps) -{ - unw_destroy_addr_space(maps__addr_space(maps)); -} - -static int get_entries(struct unwind_info *ui, unwind_entry_cb_t cb, - void *arg, int max_stack) -{ - uint16_t e_machine = thread__e_machine(ui->thread, ui->machine, /*e_flags=*/NULL); - u64 val; - unw_word_t ips[max_stack]; - unw_addr_space_t addr_space; - unw_cursor_t c; - int ret, i = 0; - - ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), - perf_arch_reg_ip(e_machine)); - if (ret) - return ret; - - ips[i++] = (unw_word_t) val; - - /* - * If we need more than one entry, do the DWARF - * unwind itself. - */ - if (max_stack - 1 > 0) { - WARN_ONCE(!ui->thread, "WARNING: ui->thread is NULL"); - addr_space = maps__addr_space(thread__maps(ui->thread)); - - if (addr_space == NULL) - return -1; - - ret = unw_init_remote(&c, addr_space, ui); - if (ret && !ui->best_effort) - display_error(ret); - - while (!ret && (unw_step(&c) > 0) && i < max_stack) { - unw_get_reg(&c, UNW_REG_IP, &ips[i]); - - /* - * Decrement the IP for any non-activation frames. - * this is required to properly find the srcline - * for caller frames. - * See also the documentation for dwfl_frame_pc(), - * which this code tries to replicate. - */ - if (unw_is_signal_frame(&c) <= 0) - --ips[i]; - - ++i; - } - - max_stack = i; - } - - /* - * Display what we got based on the order setup. - */ - for (i = 0; i < max_stack && !ret; i++) { - int j = i; - - if (callchain_param.order == ORDER_CALLER) - j = max_stack - i - 1; - ret = ips[j] ? entry(ips[j], ui->thread, cb, arg) : 0; - } - - return ret; -} - -static int _unwind__get_entries(unwind_entry_cb_t cb, void *arg, - struct thread *thread, - struct perf_sample *data, int max_stack, - bool best_effort) -{ - struct unwind_info ui = { - .sample = data, - .thread = thread, - .machine = maps__machine(thread__maps(thread)), - .best_effort = best_effort - }; - - if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; - - if (max_stack <= 0) - return -EINVAL; - - return get_entries(&ui, cb, arg, max_stack); -} - -static struct unwind_libunwind_ops -_unwind_libunwind_ops = { - .prepare_access = _unwind__prepare_access, - .flush_access = _unwind__flush_access, - .finish_access = _unwind__finish_access, - .get_entries = _unwind__get_entries, -}; - -#ifndef REMOTE_UNWIND_LIBUNWIND -struct unwind_libunwind_ops * -local_unwind_libunwind_ops = &_unwind_libunwind_ops; -#endif diff --git a/tools/perf/util/unwind-libunwind.c b/tools/perf/util/unwind-libunwind.c index cb8be6acfb6f..73d191ce51a5 100644 --- a/tools/perf/util/unwind-libunwind.c +++ b/tools/perf/util/unwind-libunwind.c @@ -1,92 +1,683 @@ // SPDX-License-Identifier: GPL-2.0 -#include "unwind.h" +#include "callchain.h" +#include "debug.h" #include "dso.h" +#include "env.h" #include "map.h" -#include "thread.h" +#include "perf_regs.h" #include "session.h" -#include "debug.h" -#include "env.h" -#include "callchain.h" +#include "symbol.h" +#include "thread.h" +#include "unwind.h" +#include "libunwind-arch/libunwind-arch.h" +#include +#include +#include +#include +#include + +#define DW_EH_PE_FORMAT_MASK 0x0f /* format of the encoded value */ +#define DW_EH_PE_APPL_MASK 0x70 /* how the value is to be applied */ -struct unwind_libunwind_ops __weak *local_unwind_libunwind_ops; -struct unwind_libunwind_ops __weak *x86_32_unwind_libunwind_ops; -struct unwind_libunwind_ops __weak *arm64_unwind_libunwind_ops; +/* Pointer-encoding formats: */ +#define DW_EH_PE_omit 0xff +#define DW_EH_PE_ptr 0x00 /* pointer-sized unsigned value */ +#define DW_EH_PE_udata4 0x03 /* unsigned 32-bit value */ +#define DW_EH_PE_udata8 0x04 /* unsigned 64-bit value */ +#define DW_EH_PE_sdata4 0x0b /* signed 32-bit value */ +#define DW_EH_PE_sdata8 0x0c /* signed 64-bit value */ -int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized) +/* Pointer-encoding application: */ +#define DW_EH_PE_absptr 0x00 /* absolute value */ +#define DW_EH_PE_pcrel 0x10 /* rel. to addr. of encoded value */ + +/* + * The following are not documented by LSB v1.3, yet they are used by + * GCC, presumably they aren't documented by LSB since they aren't + * used on Linux: + */ +#define DW_EH_PE_funcrel 0x40 /* start-of-procedure-relative */ +#define DW_EH_PE_aligned 0x50 /* aligned pointer */ + +/* Flags intentionally not handled, since they're not needed: + * #define DW_EH_PE_indirect 0x80 + * #define DW_EH_PE_uleb128 0x01 + * #define DW_EH_PE_udata2 0x02 + * #define DW_EH_PE_sleb128 0x09 + * #define DW_EH_PE_sdata2 0x0a + * #define DW_EH_PE_textrel 0x20 + * #define DW_EH_PE_datarel 0x30 + */ + +#define dw_read(ptr, type, end) ({ \ + type *__p = (type *) ptr; \ + type __v; \ + if ((__p + 1) > (type *) end) \ + return -EINVAL; \ + __v = *__p++; \ + ptr = (typeof(ptr)) __p; \ + __v; \ + }) + +static int __dw_read_encoded_value(u8 **p, u8 *end, u64 *val, + u8 encoding) { - const char *arch; - enum dso_type dso_type; - struct unwind_libunwind_ops *ops = local_unwind_libunwind_ops; - struct dso *dso = map__dso(map); - struct machine *machine; - int err; + u8 *cur = *p; + *val = 0; - if (!dwarf_callchain_users) + switch (encoding) { + case DW_EH_PE_omit: + *val = 0; + goto out; + case DW_EH_PE_ptr: + *val = dw_read(cur, unsigned long, end); + goto out; + default: + break; + } + + switch (encoding & DW_EH_PE_APPL_MASK) { + case DW_EH_PE_absptr: + break; + case DW_EH_PE_pcrel: + *val = (unsigned long) cur; + break; + default: + return -EINVAL; + } + + if ((encoding & 0x07) == 0x00) + encoding |= DW_EH_PE_udata4; + + switch (encoding & DW_EH_PE_FORMAT_MASK) { + case DW_EH_PE_sdata4: + *val += dw_read(cur, s32, end); + break; + case DW_EH_PE_udata4: + *val += dw_read(cur, u32, end); + break; + case DW_EH_PE_sdata8: + *val += dw_read(cur, s64, end); + break; + case DW_EH_PE_udata8: + *val += dw_read(cur, u64, end); + break; + default: + return -EINVAL; + } + + out: + *p = cur; + return 0; +} + +#define dw_read_encoded_value(ptr, end, enc) ({ \ + u64 __v; \ + if (__dw_read_encoded_value(&ptr, end, &__v, enc)) { \ + return -EINVAL; \ + } \ + __v; \ + }) + +static u64 elf_base_address(int fd) +{ + Elf *elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + GElf_Phdr phdr; + u64 retval = 0; + size_t i, phdrnum = 0; + + if (elf == NULL) return 0; + (void)elf_getphdrnum(elf, &phdrnum); + /* PT_LOAD segments are sorted by p_vaddr, so the first has the minimum p_vaddr. */ + for (i = 0; i < phdrnum; i++) { + if (gelf_getphdr(elf, i, &phdr) && phdr.p_type == PT_LOAD) { + retval = phdr.p_vaddr & -getpagesize(); + break; + } + } - if (maps__addr_space(maps)) { - pr_debug("unwind: thread map already set, dso=%s\n", dso__name(dso)); - if (initialized) - *initialized = true; + elf_end(elf); + return retval; +} + +static int unwind_spec_ehframe(struct dso *dso, struct machine *machine, + u64 offset, u64 *table_data_offset, u64 *fde_count) +{ + struct eh_frame_hdr { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + + /* + * The rest of the header is variable-length and consists of the + * following members: + * + * encoded_t eh_frame_ptr; + * encoded_t fde_count; + */ + + /* A single encoded pointer should not be more than 8 bytes. */ + u64 enc[2]; + + /* + * struct { + * encoded_t start_ip; + * encoded_t fde_addr; + * } binary_search_table[fde_count]; + */ + char data[]; + } __packed hdr; + u8 *enc = (u8 *) &hdr.enc; + u8 *end = (u8 *) &hdr.data; + ssize_t r; + + r = dso__data_read_offset(dso, machine, offset, (u8 *) &hdr, sizeof(hdr)); + if (r != sizeof(hdr)) + return -EINVAL; + + /* We dont need eh_frame_ptr, just skip it. */ + dw_read_encoded_value(enc, end, hdr.eh_frame_ptr_enc); + + *fde_count = dw_read_encoded_value(enc, end, hdr.fde_count_enc); + *table_data_offset = enc - (u8 *) &hdr; + return 0; +} + +struct read_unwind_spec_eh_frame_maps_cb_args { + struct dso *dso; + u64 base_addr; +}; + +static int read_unwind_spec_eh_frame_maps_cb(struct map *map, void *data) +{ + + struct read_unwind_spec_eh_frame_maps_cb_args *args = data; + + if (map__dso(map) == args->dso && map__start(map) - map__pgoff(map) < args->base_addr) + args->base_addr = map__start(map) - map__pgoff(map); + + return 0; +} + +static int elf_section_address_and_offset(int fd, const char *name, u64 *address, u64 *offset) +{ + Elf *elf; + GElf_Ehdr ehdr; + GElf_Shdr shdr; + int ret = -1; + + elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + if (elf == NULL) + return -1; + + if (gelf_getehdr(elf, &ehdr) == NULL) + goto out_err; + + if (!elf_section_by_name(elf, &ehdr, &shdr, name, NULL)) + goto out_err; + + *address = shdr.sh_addr; + *offset = shdr.sh_offset; + ret = 0; +out_err: + elf_end(elf); + return ret; +} + +static int read_unwind_spec_eh_frame(struct dso *dso, struct unwind_info *ui, + u64 *table_data, u64 *segbase, + u64 *fde_count) +{ + struct read_unwind_spec_eh_frame_maps_cb_args args = { + .dso = dso, + .base_addr = UINT64_MAX, + }; + int ret, fd; + + if (dso__data(dso)->eh_frame_hdr_offset == 0) { + if (!dso__data_get_fd(dso, ui->machine, &fd)) + return -EINVAL; + + /* Check the .eh_frame section for unwinding info */ + ret = elf_section_address_and_offset(fd, ".eh_frame_hdr", + &dso__data(dso)->eh_frame_hdr_addr, + &dso__data(dso)->eh_frame_hdr_offset); + dso__data(dso)->elf_base_addr = elf_base_address(fd); + dso__data_put_fd(dso); + if (ret || dso__data(dso)->eh_frame_hdr_offset == 0) + return -EINVAL; + } + + maps__for_each_map(thread__maps(ui->thread), read_unwind_spec_eh_frame_maps_cb, &args); + + args.base_addr -= dso__data(dso)->elf_base_addr; + /* Address of .eh_frame_hdr */ + *segbase = args.base_addr + dso__data(dso)->eh_frame_hdr_addr; + ret = unwind_spec_ehframe(dso, ui->machine, dso__data(dso)->eh_frame_hdr_offset, + table_data, fde_count); + if (ret) + return ret; + /* binary_search_table offset plus .eh_frame_hdr address */ + *table_data += *segbase; + return 0; +} + +static u64 elf_section_offset(int fd, const char *name) +{ + u64 address, offset = 0; + + if (elf_section_address_and_offset(fd, name, &address, &offset)) + return 0; + + return offset; +} + +static int read_unwind_spec_debug_frame(struct dso *dso, + struct machine *machine, u64 *offset) +{ + int fd; + u64 ofs = dso__data(dso)->debug_frame_offset; + + /* debug_frame can reside in: + * - dso + * - debug pointed by symsrc_filename + * - gnu_debuglink, which doesn't necessary + * has to be pointed by symsrc_filename + */ + if (ofs == 0) { + if (dso__data_get_fd(dso, machine, &fd)) { + ofs = elf_section_offset(fd, ".debug_frame"); + dso__data_put_fd(dso); + } + + if (ofs <= 0) { + fd = open(dso__symsrc_filename(dso), O_RDONLY); + if (fd >= 0) { + ofs = elf_section_offset(fd, ".debug_frame"); + close(fd); + } + } + + if (ofs <= 0) { + char *debuglink = malloc(PATH_MAX); + int ret = 0; + + if (debuglink == NULL) { + pr_err("unwind: Can't read unwind spec debug frame.\n"); + return -ENOMEM; + } + + ret = dso__read_binary_type_filename( + dso, DSO_BINARY_TYPE__DEBUGLINK, + machine->root_dir, debuglink, PATH_MAX); + if (!ret) { + fd = open(debuglink, O_RDONLY); + if (fd >= 0) { + ofs = elf_section_offset(fd, + ".debug_frame"); + close(fd); + } + } + if (ofs > 0) { + if (dso__symsrc_filename(dso) != NULL) { + pr_warning( + "%s: overwrite symsrc(%s,%s)\n", + __func__, + dso__symsrc_filename(dso), + debuglink); + dso__free_symsrc_filename(dso); + } + dso__set_symsrc_filename(dso, debuglink); + } else { + free(debuglink); + } + } + + dso__data(dso)->debug_frame_offset = ofs; + } + + *offset = ofs; + if (*offset) + return 0; + + return -EINVAL; +} + +static struct map *find_map(uint64_t ip, struct unwind_info *ui) +{ + struct addr_location al; + struct map *ret; + + addr_location__init(&al); + thread__find_map(ui->thread, PERF_RECORD_MISC_USER, ip, &al); + ret = map__get(al.map); + addr_location__exit(&al); + return ret; +} + +static int elf_is_exec(int fd, const char *name) +{ + Elf *elf; + GElf_Ehdr ehdr; + int retval = 0; + + elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + if (elf == NULL) + return 0; + if (gelf_getehdr(elf, &ehdr) == NULL) + goto out; + + retval = (ehdr.e_type == ET_EXEC); + +out: + elf_end(elf); + pr_debug3("unwind: elf_is_exec(%s): %d\n", name, retval); + return retval; +} + +int __libunwind__find_proc_info(void *as, uint64_t ip, void *pi, int need_unwind_info, void *arg) +{ + struct unwind_info *ui = arg; + struct map *map; + struct dso *dso; + u64 table_data, segbase, fde_count; + int ret = -EINVAL; + + map = find_map(ip, ui); + if (!map) + return -EINVAL; + + dso = map__dso(map); + if (!dso) { + map__put(map); + return -EINVAL; + } + + pr_debug3("unwind: find_proc_info dso %s\n", dso__name(dso)); + + /* Check the .eh_frame section for unwinding info */ + if (!read_unwind_spec_eh_frame(dso, ui, &table_data, &segbase, &fde_count)) { + struct table_entry { + u32 start_ip_offset; + u32 fde_offset; + }; + struct libarch_unwind__dyn_info di = { + .start_ip = map__start(map), + .end_ip = map__end(map), + .segbase = segbase, + .table_data = table_data, + .table_len = fde_count * sizeof(struct table_entry) / ui->unw_word_t_size, + }; + + ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, + need_unwind_info, arg); + } + + /* Check the .debug_frame section for unwinding info */ + if (ret < 0 && !read_unwind_spec_debug_frame(dso, ui->machine, &segbase)) { + int fd; + u64 start = map__start(map); + u64 base = start; + const char *symfile; + struct libarch_unwind__dyn_info di = {}; + + if (dso__data_get_fd(dso, ui->machine, &fd)) { + if (elf_is_exec(fd, dso__name(dso))) + base = 0; + dso__data_put_fd(dso); + } + + symfile = dso__symsrc_filename(dso) ?: dso__name(dso); + + if (libunwind_arch__dwarf_find_debug_frame(ui->e_machine, /*found=*/0, &di, ip, + base, symfile, start, map__end(map))) { + ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, + need_unwind_info, arg); + } + } + map__put(map); + return ret; +} + +static int access_dso_mem(struct unwind_info *ui, uint64_t addr, void *data_word) +{ + struct map *map; + struct dso *dso; + ssize_t size; + + map = find_map(addr, ui); + if (!map) { + pr_debug("unwind: no map for %lx\n", (unsigned long)addr); + return -1; + } + + dso = map__dso(map); + + if (!dso) { + map__put(map); + return -1; + } + + size = dso__data_read_addr(dso, map, ui->machine, + addr, + (u8 *) data_word, + ui->unw_word_t_size); + map__put(map); + return !((size_t)size == ui->unw_word_t_size); +} + +int __libunwind__access_mem(void *as __maybe_unused, uint64_t addr, void *valp_word, + int __write, void *arg) +{ + struct unwind_info *ui = arg; + struct stack_dump *stack = &ui->sample->user_stack; + u64 start, end; + int offset; + int ret; + + /* Don't support write, probably not needed. */ + if (__write || !stack || !ui->sample->user_regs || !ui->sample->user_regs->regs) { + uint64_t zero = 0; + + memcpy(valp_word, &zero, ui->unw_word_t_size); return 0; } - machine = maps__machine(maps); - /* env->arch is NULL for live-mode (i.e. perf top) */ - if (!machine->env || !machine->env->arch) - goto out_register; + ret = perf_reg_value(&start, perf_sample__user_regs(ui->sample), + perf_arch_reg_sp(ui->e_machine)); + if (ret) + return ret; + + end = start + stack->size; + + /* Check overflow. */ + if (addr + ui->unw_word_t_size < addr) + return -EINVAL; + + if (addr < start || addr + ui->unw_word_t_size >= end) { + ret = access_dso_mem(ui, addr, valp_word); + if (ret) { + pr_debug3("unwind: access_mem %p not inside range" + " 0x%" PRIx64 "-0x%" PRIx64 "\n", + (void *) (uintptr_t) addr, start, end); + memset(valp_word, 0, ui->unw_word_t_size); + return ret; + } + return 0; + } + + offset = addr - start; + memcpy(valp_word, &stack->data[offset], ui->unw_word_t_size); + pr_debug3("unwind: access_mem addr %p val %lx, offset %d\n", + (void *) (uintptr_t) addr, *((unsigned long *)valp_word), offset); + return 0; +} + +int __libunwind__access_reg(void *as __maybe_unused, int regnum, void *valp_word, int __write, + void *arg) +{ + struct unwind_info *ui = arg; + int id, ret; + u64 val; + + /* Don't support write, I suspect we don't need it. */ + if (__write) { + pr_err("unwind: access_reg w %d\n", regnum); + return 0; + } - dso_type = dso__type(dso, machine); - if (dso_type == DSO__TYPE_UNKNOWN) + if (!ui->sample->user_regs || !ui->sample->user_regs->regs) { + memset(valp_word, 0, ui->unw_word_t_size); return 0; + } + + id = get_perf_regnum_for_unw_regnum(ui->e_machine, regnum); + if (id < 0) + return -EINVAL; + + ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), id); + if (ret) { + if (!ui->best_effort) + pr_err("unwind: can't read reg %d\n", regnum); + return ret; + } + + if (ui->unw_word_t_size == 8) + *(uint64_t *)valp_word = val; + else + *(uint32_t *)valp_word = (uint32_t)val; + pr_debug3("unwind: reg %d, val %lx\n", regnum, val); + return 0; +} + +int unwind__prepare_access(struct maps *maps, uint16_t e_machine) +{ + void *addr_space; - arch = perf_env__arch(machine->env); + if (!dwarf_callchain_users) + return 0; - if (!strcmp(arch, "x86")) { - if (dso_type != DSO__TYPE_64BIT) - ops = x86_32_unwind_libunwind_ops; - } else if (!strcmp(arch, "arm64") || !strcmp(arch, "arm")) { - if (dso_type == DSO__TYPE_64BIT) - ops = arm64_unwind_libunwind_ops; + if (maps__addr_space(maps)) { + pr_debug3("unwind: thread map already set\n"); + return 0; } - if (!ops) { - pr_warning_once("unwind: target platform=%s is not supported\n", arch); + if (e_machine == EM_NONE) return 0; + + maps__set_e_machine(maps, e_machine); + addr_space = libunwind_arch__create_addr_space(e_machine); + + maps__set_addr_space(maps, addr_space); + if (!addr_space) { + pr_err("unwind: Can't create unwind address space.\n"); + return -ENOMEM; } -out_register: - maps__set_unwind_libunwind_ops(maps, ops); - err = maps__unwind_libunwind_ops(maps)->prepare_access(maps); - if (initialized) - *initialized = err ? false : true; - return err; + return 0; } void unwind__flush_access(struct maps *maps) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(maps); - - if (ops) - ops->flush_access(maps); + libunwind_arch__flush_access(maps); } void unwind__finish_access(struct maps *maps) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(maps); + libunwind_arch__finish_access(maps); +} + +static int entry(uint64_t ip, struct thread *thread, unwind_entry_cb_t cb, void *arg) +{ + struct unwind_entry e; + struct addr_location al; + int ret; - if (ops) - ops->finish_access(maps); + addr_location__init(&al); + e.ms.sym = thread__find_symbol(thread, PERF_RECORD_MISC_USER, ip, &al); + e.ip = ip; + e.ms.map = al.map; + e.ms.thread = thread__get(al.thread); + + pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", + al.sym ? al.sym->name : "''", + ip, + al.map ? map__map_ip(al.map, ip) : (u64) 0); + + ret = cb(&e, arg); + addr_location__exit(&al); + return ret; } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, - struct perf_sample *data, int max_stack, + struct perf_sample *sample, int max_stack, bool best_effort) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(thread__maps(thread)); + struct unwind_info *ui; + uint64_t first_ip; + int ret, i = 0, entries = 0; + uint16_t e_machine; - if (ops) - return ops->get_entries(cb, arg, thread, data, max_stack, best_effort); - return 0; + if (!sample->user_regs || !sample->user_regs->regs) + return 0; + + if (max_stack <= 0) + return 0; + + if (!thread) { + pr_warning_once("WARNING: thread is NULL"); + return 0; + } + + e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); + ret = perf_reg_value(&first_ip, perf_sample__user_regs(sample), + perf_arch_reg_ip(e_machine)); + if (ret) + return 0; + + if (max_stack == 1) { + /* Special case for a single entry. */ + ret = entry(first_ip, thread, cb, arg); + return ret ? (ret == -ENOMEM ? -ENOMEM : 0) : 1; + } + + ui = libunwind_arch_unwind_info__new(thread, sample, max_stack, best_effort, e_machine, first_ip); + if (!ui) + return -ENOMEM; + + do { + ret = libunwind_arch__unwind_step(ui); + if (ret < 0) + goto out; + + } while (ret); + + /* + * Display what we got based on the order setup. + */ + for (i = 0; i < ui->cur_ip; i++) { + int j = callchain_param.order == ORDER_CALLEE ? i : ui->cur_ip - i - 1; + + if (ui->ips[j]) { + ret = entry(ui->ips[j], thread, cb, arg); + if (ret) + break; + entries++; + } + } +out: + libunwind_arch_unwind_info__delete(ui); + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. stepping failure) to allow fallback unwinder or kernel callchains. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (ret == -ENOMEM) + return -ENOMEM; + return (entries > 0 || ret == 0) ? entries : 0; } diff --git a/tools/perf/util/unwind.c b/tools/perf/util/unwind.c new file mode 100644 index 000000000000..4ed4b1d55c69 --- /dev/null +++ b/tools/perf/util/unwind.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "debug.h" +#include "symbol_conf.h" +#include "unwind.h" +#include +#include +#include + +int unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) +{ + int ret = 0; + +#if defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + if (symbol_conf.unwind_style[0] == UNWIND_STYLE_UNKNOWN) { + int i = 0; +#ifdef HAVE_LIBDW_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBDW; +#endif +#ifdef HAVE_LIBUNWIND_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBUNWIND; +#endif + } +#endif //defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + + for (size_t i = 0; i < ARRAY_SIZE(symbol_conf.unwind_style); i++) { + switch (symbol_conf.unwind_style[i]) { + case UNWIND_STYLE_LIBDW: + ret = libdw__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_LIBUNWIND: + ret = libunwind__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_UNKNOWN: + default: +#if !defined(HAVE_LIBDW_SUPPORT) && !defined(HAVE_LIBUNWIND_SUPPORT) + pr_warning_once( + "Error: dwarf unwinding not supported, build perf with libdw or libunwind.\n"); +#endif + ret = 0; + break; + } + if (ret > 0) { + ret = 0; + break; + } + if (ret < 0) + break; + } + return ret; +} + +int unwind__configure(const char *var, const char *value, void *cb __maybe_unused) +{ + static const char * const unwind_style_names[] = { + [UNWIND_STYLE_LIBDW] = "libdw", + [UNWIND_STYLE_LIBUNWIND] = "libunwind", + NULL + }; + char *s, *p, *saveptr; + size_t i = 0; + + if (strcmp(var, "unwind.style")) + return 0; + + if (!value) + return -1; + + s = strdup(value); + if (!s) + return -1; + + memset(symbol_conf.unwind_style, 0, sizeof(symbol_conf.unwind_style)); + + p = strtok_r(s, ",", &saveptr); + while (p && i < ARRAY_SIZE(symbol_conf.unwind_style)) { + bool found = false; + char *q = strim(p); + + for (size_t j = UNWIND_STYLE_LIBDW; j < MAX_UNWIND_STYLE; j++) { + if (!strcasecmp(q, unwind_style_names[j])) { + symbol_conf.unwind_style[i++] = j; + found = true; + break; + } + } + if (!found) + pr_warning("Unknown unwind style: %s\n", q); + p = strtok_r(NULL, ",", &saveptr); + } + + free(s); + return 0; +} + +int unwind__option(const struct option *opt __maybe_unused, + const char *arg, + int unset __maybe_unused) +{ + return unwind__configure("unwind.style", arg, NULL); +} diff --git a/tools/perf/util/unwind.h b/tools/perf/util/unwind.h index 9f7164c6d9aa..a2d4af694fde 100644 --- a/tools/perf/util/unwind.h +++ b/tools/perf/util/unwind.h @@ -2,11 +2,13 @@ #ifndef __UNWIND_H #define __UNWIND_H +#include #include #include -#include "util/map_symbol.h" +#include "map_symbol.h" struct maps; +struct option; struct perf_sample; struct thread; @@ -17,16 +19,9 @@ struct unwind_entry { typedef int (*unwind_entry_cb_t)(struct unwind_entry *entry, void *arg); -struct unwind_libunwind_ops { - int (*prepare_access)(struct maps *maps); - void (*flush_access)(struct maps *maps); - void (*finish_access)(struct maps *maps); - int (*get_entries)(unwind_entry_cb_t cb, void *arg, - struct thread *thread, - struct perf_sample *data, int max_stack, bool best_effort); -}; +int unwind__configure(const char *var, const char *value, void *cb); +int unwind__option(const struct option *opt, const char *arg, int unset); -#ifdef HAVE_DWARF_UNWIND_SUPPORT /* * When best_effort is set, don't report errors and fail silently. This could * be expanded in the future to be more permissive about things other than @@ -36,47 +31,55 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, bool best_effort); -/* libunwind specific */ -#ifdef HAVE_LIBUNWIND_SUPPORT -#ifndef LIBUNWIND__ARCH_REG_ID -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__arch_reg_id(regnum) -#endif -int LIBUNWIND__ARCH_REG_ID(int regnum); -int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized); -void unwind__flush_access(struct maps *maps); -void unwind__finish_access(struct maps *maps); +#ifdef HAVE_LIBDW_SUPPORT +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); #else -static inline int unwind__prepare_access(struct maps *maps __maybe_unused, - struct map *map __maybe_unused, - bool *initialized __maybe_unused) +#include "debug.h" +static inline int libdw__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) { + pr_warning_once("Error: libdw dwarf unwinding not built into perf\n"); return 0; } - -static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} -static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} #endif + +#ifdef HAVE_LIBUNWIND_SUPPORT +/* libunwind specific */ +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); +int unwind__prepare_access(struct maps *maps, uint16_t e_machine); +void unwind__flush_access(struct maps *maps); +void unwind__finish_access(struct maps *maps); #else -static inline int -unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, - void *arg __maybe_unused, - struct thread *thread __maybe_unused, - struct perf_sample *data __maybe_unused, - int max_stack __maybe_unused, - bool best_effort __maybe_unused) +#include "debug.h" +static inline int libunwind__get_entries(unwind_entry_cb_t cb __maybe_unused, + void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) { + pr_warning_once("Error: libunwind dwarf unwinding not built into perf\n"); return 0; } static inline int unwind__prepare_access(struct maps *maps __maybe_unused, - struct map *map __maybe_unused, - bool *initialized __maybe_unused) + uint16_t e_machine __maybe_unused) { return 0; } static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} -#endif /* HAVE_DWARF_UNWIND_SUPPORT */ +#endif + #endif /* __UNWIND_H */ diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 25849434f0a4..2c2a5c449ffd 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -419,11 +419,21 @@ int perf_tip(char **strp, const char *dirpath) char *perf_exe(char *buf, int len) { - int n = readlink("/proc/self/exe", buf, len); + int n; + + if (len <= 0) + return buf; + + n = readlink("/proc/self/exe", buf, len - 1); if (n > 0) { buf[n] = 0; return buf; } + if (len < (int)sizeof("perf")) { + buf[0] = '\0'; + return buf; + } + return strcpy(buf, "perf"); } diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c index 57027e0ac7b6..21a0eb58597c 100644 --- a/tools/perf/util/zstd.c +++ b/tools/perf/util/zstd.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include "util/compress.h" #include "util/debug.h" @@ -54,7 +55,13 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ while (input.pos < input.size) { record = dst; + /* process_header writes the event header into record */ + if (dst_size < sizeof(struct perf_event_header)) + goto reset; size = process_header(record, 0); + /* Output buffer full — cannot fit even the record header */ + if (size > dst_size) + goto reset; compressed += size; dst += size; dst_size -= size; @@ -65,10 +72,18 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ if (ZSTD_isError(ret)) { pr_err("failed to compress %ld bytes: %s\n", (long)src_size, ZSTD_getErrorName(ret)); - memcpy(dst, src, src_size); - return src_size; + goto reset; } size = output.pos; + /* + * No progress: ZSTD couldn't emit any bytes into the + * remaining output buffer. Calling process_header + * with size=0 would re-trigger header initialization, + * double-subtracting the header size from dst_size and + * underflowing the unsigned counter. + */ + if (size == 0) + goto reset; size = process_header(record, size); compressed += size; dst += size; @@ -76,6 +91,14 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ } return compressed; + +reset: + /* Reset so the context is usable if the caller retries */ + ret = ZSTD_initCStream(data->cstream, data->comp_level); + if (ZSTD_isError(ret)) + pr_err("failed to reset compression context: %s\n", + ZSTD_getErrorName(ret)); + return -1; } size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, @@ -100,14 +123,26 @@ size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size } } while (input.pos < input.size) { + size_t prev_in = input.pos; + size_t prev_out = output.pos; + ret = ZSTD_decompressStream(data->dstream, &output, &input); if (ZSTD_isError(ret)) { pr_err("failed to decompress (B): %zd -> %zd, dst_size %zd : %s\n", - src_size, output.size, dst_size, ZSTD_getErrorName(ret)); - break; + src_size, output.pos, dst_size, ZSTD_getErrorName(ret)); + return 0; } - output.dst = dst + output.pos; - output.size = dst_size - output.pos; + /* + * Neither stream advanced — decompression is stuck. + * Return 0 (error) rather than partial output: perf + * uses ZSTD_flushStream (not ZSTD_endStream), so the + * stream is continuous across compressed events. + * Discarding unconsumed input would desynchronize the + * decompressor, causing the next call to produce + * garbage that could be misinterpreted as valid events. + */ + if (input.pos == prev_in && output.pos == prev_out) + return 0; } return output.pos; diff --git a/tools/power/x86/intel-speed-select/isst-daemon.c b/tools/power/x86/intel-speed-select/isst-daemon.c index 66df21b2b573..acedb7432849 100644 --- a/tools/power/x86/intel-speed-select/isst-daemon.c +++ b/tools/power/x86/intel-speed-select/isst-daemon.c @@ -148,6 +148,7 @@ static void daemonize(char *rundir, char *pidfile) { int pid, sid, i; char str[10]; + struct stat st; struct sigaction sig_actions; sigset_t sig_set; int ret; @@ -200,11 +201,17 @@ static void daemonize(char *rundir, char *pidfile) if (ret == -1) exit(EXIT_FAILURE); - pid_file_handle = open(pidfile, O_RDWR | O_CREAT, 0600); + pid_file_handle = open(pidfile, O_RDWR | O_CREAT | O_NOFOLLOW, 0600); if (pid_file_handle == -1) { /* Couldn't open lock file */ exit(1); } + + if (fstat(pid_file_handle, &st) == -1) + exit(1); + + if (!S_ISREG(st.st_mode)) + exit(1); /* Try to lock file */ #ifdef LOCKF_SUPPORT if (lockf(pid_file_handle, F_TLOCK, 0) == -1) { diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 344ede2f8546..c905198d0501 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -92,7 +92,7 @@ name as necessary to disambiguate it from others is necessary. Note that option '%x' hex identifier of the PMT MMIO region. .fi .PP -\fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set. If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed. Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed. Otherwise, the system summary plus the specified set of CPUs are printed. The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44 +\fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set. If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed. Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed. Otherwise, the system summary plus the specified set of CPUs are printed. The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44 Multiple invocations of --cpu are OR'd together. .PP \fB--hide column\fP do not show the specified built-in columns. May be invoked multiple times, or with a comma-separated list of column names. .PP @@ -269,7 +269,7 @@ Periodic output goes to stdout, by default, unless --out is used to specify an o The 5-second interval can be changed with the "-i sec" option. .nf sudo turbostat --quiet --show CPU,frequency - Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c7 UncMhz + Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c7 UncMHz - - 524 12.48 4198 3096 74.53 3800 0 0 4 0.09 4081 3096 98.88 3800 0 4 1 0.02 4063 3096 @@ -279,7 +279,7 @@ sudo turbostat --quiet --show CPU,frequency 2 6 3 0.08 4159 3096 3 3 1 0.04 4046 3096 99.66 3 7 0 0.01 3989 3096 - Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c7 UncMhz + Core CPU Avg_MHz Busy% Bzy_MHz TSC_MHz CPU%c7 UncMHz - - 525 12.52 4198 3096 74.54 3800 0 0 4 0.10 4051 3096 99.49 3800 0 4 2 0.04 3993 3096 diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 920694c3c1ec..4ad7cb1df5c5 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -98,6 +98,8 @@ #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL)) +FILE *outf; + enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE }; enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M }; enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE }; @@ -138,8 +140,6 @@ struct msr_counter { enum counter_format format; struct msr_counter *next; unsigned int flags; -#define FLAGS_HIDE (1 << 0) -#define FLAGS_SHOW (1 << 1) #define SYSFS_PERCPU (1 << 1) }; static int use_android_msr_path; @@ -292,33 +292,53 @@ enum bic_names { MAX_BIC }; -void print_bic_set(char *s, cpu_set_t *set) +#define bic_set_t cpu_set_t /* implement bic_set_t using cpu_set_t */ + +/* + * debug helper + */ +static void print_bic_set(const char *s, const bic_set_t *set) { int i; assert(MAX_BIC < CPU_SETSIZE); - printf("%s:", s); + fprintf(outf, "%s:", s); for (i = 0; i < MAX_BIC; ++i) { if (CPU_ISSET(i, set)) - printf(" %s", bic[i].name); + fprintf(outf, " %s", bic[i].name); } - putchar('\n'); + fputc('\n', outf); } -static cpu_set_t bic_group_topology; -static cpu_set_t bic_group_thermal_pwr; -static cpu_set_t bic_group_frequency; -static cpu_set_t bic_group_hw_idle; -static cpu_set_t bic_group_sw_idle; -static cpu_set_t bic_group_idle; -static cpu_set_t bic_group_cache; -static cpu_set_t bic_group_other; -static cpu_set_t bic_group_disabled_by_default; -static cpu_set_t bic_enabled; -static cpu_set_t bic_present; +static bic_set_t bic_group_topology; +static bic_set_t bic_group_thermal_pwr; +static bic_set_t bic_group_frequency; +static bic_set_t bic_group_hw_idle; +static bic_set_t bic_group_sw_idle; +static bic_set_t bic_group_idle; +static bic_set_t bic_group_cache; +static bic_set_t bic_group_other; +static bic_set_t bic_group_disabled_by_default; +static bic_set_t bic_enabled; +static bic_set_t bic_present; + +static void dump_bic_sets(void) +{ + print_bic_set("bic_enabled", &bic_enabled); + print_bic_set("bic_present", &bic_present); + print_bic_set("bic_group_topology", &bic_group_topology); + print_bic_set("bic_group_thermal_pwr", &bic_group_thermal_pwr); + print_bic_set("bic_group_frequency", &bic_group_frequency); + print_bic_set("bic_group_hw_idle", &bic_group_hw_idle); + print_bic_set("bic_group_sw_idle", &bic_group_sw_idle); + print_bic_set("bic_group_idle", &bic_group_idle); + print_bic_set("bic_group_cache", &bic_group_cache); + print_bic_set("bic_group_other", &bic_group_other); + print_bic_set("bic_group_disabled_by_default", &bic_group_disabled_by_default); +} /* modify */ #define BIC_INIT(set) CPU_ZERO(set) @@ -334,7 +354,7 @@ static cpu_set_t bic_present; #define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present) #define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present)) -static void bic_set_all(cpu_set_t *set) +static void bic_set_all(bic_set_t *set) { int i; @@ -348,7 +368,7 @@ static void bic_set_all(cpu_set_t *set) * bic_clear_bits() * clear all the bits from "clr" in "dst" */ -static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr) +static void bic_clear_bits(bic_set_t *dst, bic_set_t *clr) { int i; @@ -485,7 +505,6 @@ static void bic_groups_init(void) #define PCLUNL 15 /* Unlimited */ char *proc_stat = "/proc/stat"; -FILE *outf; int *fd_percpu; int *fd_instr_count_percpu; int *fd_llc_percpu; @@ -504,7 +523,7 @@ unsigned int valid_rapl_msrs; unsigned int summary_only; unsigned int list_header_only; unsigned int dump_only; -unsigned int force_load; +bool force_load; unsigned int cpuid_has_aperf_mperf; unsigned int cpuid_has_hv; unsigned int has_aperf_access; @@ -570,9 +589,8 @@ struct gfx_sysfs_info { static struct gfx_sysfs_info gfx_info[GFX_MAX]; -int get_msr(int cpu, off_t offset, unsigned long long *msr); -int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num); +void get_msr(int cpu, off_t offset, unsigned long long *msr); +int add_counter(unsigned int, const char *, const char *, unsigned int, enum counter_scope, enum counter_type, enum counter_format, int, int); /* Model specific support Start */ @@ -629,8 +647,8 @@ double slm_bclk(void) unsigned int i; double freq; - if (get_msr(master_cpu, MSR_FSB_FREQ, &msr)) - fprintf(outf, "SLM BCLK: unknown\n"); + get_msr(master_cpu, MSR_FSB_FREQ, &msr); + fprintf(outf, "SLM BCLK: unknown\n"); i = msr & 0xf; if (i >= SLM_BCLK_FREQS) { @@ -640,7 +658,7 @@ double slm_bclk(void) freq = slm_freq_table[i]; if (!quiet) - fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq); + fprintf(outf, "SLM BCLK: %.1f MHz\n", freq); return freq; } @@ -1312,7 +1330,6 @@ static struct perf_model_support turbostat_perf_model_support[] = { { INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* GRR */ { INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} }, /* CWF */ - { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, { INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, { INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, @@ -1343,7 +1360,7 @@ void probe_platform_features(unsigned int family, unsigned int model) if (authentic_amd || hygon_genuine) { /* fallback to default features on unsupported models */ - force_load++; + force_load = true; if (max_extended_level >= 0x80000007) { unsigned int eax, ebx, ecx, edx; @@ -1366,7 +1383,7 @@ void probe_platform_features(unsigned int family, unsigned int model) } end: - if (force_load && !platform) { + if ((force_load == true) && !platform) { fprintf(outf, "Forced to run on unsupported platform!\n"); platform = &default_features; } @@ -1407,10 +1424,9 @@ void init_perf_model_support(unsigned int family, unsigned int model) int backwards_count; char *progname; -#define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */ -cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; +cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpuset_cmdline; cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set; -size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; +size_t cpu_setsize; #define MAX_ADDED_THREAD_COUNTERS 24 #define MAX_ADDED_CORE_COUNTERS 8 #define MAX_ADDED_PACKAGE_COUNTERS 16 @@ -1995,15 +2011,17 @@ int pmt_telemdir_filter(const struct dirent *e) { unsigned int dummy; - return sscanf(e->d_name, "telem%u", &dummy); + return (sscanf(e->d_name, "telem%u", &dummy) == 1); } int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b) { unsigned int aidx = 0, bidx = 0; - sscanf((*a)->d_name, "telem%u", &aidx); - sscanf((*b)->d_name, "telem%u", &bidx); + if (sscanf((*a)->d_name, "telem%u", &aidx) != 1) + aidx = 0; + if (sscanf((*b)->d_name, "telem%u", &bidx) != 1) + bidx = 0; return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0; } @@ -2034,8 +2052,11 @@ const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *p return NULL; num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort); - if (num_names == -1) + if (num_names == -1) { + closedir(iter->dir); + iter->dir = NULL; return NULL; + } } iter->current_name_idx = 0; @@ -2063,8 +2084,10 @@ void pmt_diriter_remove(struct pmt_diriter_t *iter) iter->num_names = 0; iter->current_name_idx = 0; - closedir(iter->dir); - iter->dir = NULL; + if (iter->dir) { + closedir(iter->dir); + iter->dir = NULL; + } } unsigned int pmt_counter_get_width(const struct pmt_counter *p) @@ -2436,7 +2459,7 @@ int cpu_is_not_present(int cpu) if (cpu < 0) return 1; - return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set); + return !CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set); } int cpu_is_not_allowed(int cpu) @@ -2444,7 +2467,7 @@ int cpu_is_not_allowed(int cpu) if (cpu < 0) return 1; - return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set); + return !CPU_ISSET_S(cpu, cpu_setsize, cpu_allowed_set); } #define GLOBAL_CORE_ID(core_id, pkg_id) (core_id + pkg_id * (topo.max_core_id + 1)) @@ -2537,9 +2560,9 @@ int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, s int cpu_migrate(int cpu) { - CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); - CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set); - if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1) + CPU_ZERO_S(cpu_setsize, cpu_affinity_set); + CPU_SET_S(cpu, cpu_setsize, cpu_affinity_set); + if (sched_setaffinity(0, cpu_setsize, cpu_affinity_set) == -1) return -1; else return 0; @@ -2625,18 +2648,12 @@ int get_instr_count_fd(int cpu) return fd_instr_count_percpu[cpu]; } -int get_msr(int cpu, off_t offset, unsigned long long *msr) +void get_msr(int cpu, off_t offset, unsigned long long *msr) { - ssize_t retval; - assert(!no_msr); - retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset); - - if (retval != sizeof *msr) + if (pread(get_msr_fd(cpu), msr, sizeof(*msr), offset) != sizeof *msr) err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset); - - return 0; } int add_msr_counter(int cpu, off_t offset) @@ -2780,7 +2797,7 @@ void help(void) * for all the strings in comma separate name_list, * set the approprate bit in return value. */ -void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) +void bic_lookup(bic_set_t *ret_set, char *name_list, enum show_hide_mode mode) { unsigned int i; @@ -3280,8 +3297,6 @@ int dump_counters(PER_THREAD_PARAMS) double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval) { - assert(desired_unit != RAPL_UNIT_INVALID); - /* * For now we don't expect anything other than joules, * so just simplify the logic. @@ -3367,7 +3382,7 @@ int format_counters(PER_THREAD_PARAMS) return 0; /*if not summary line and --cpu is used */ - if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) + if ((t != average.threads) && (cpuset_cmdline && !CPU_ISSET_S(t->cpu_id, cpu_setsize, cpuset_cmdline))) return 0; if (DO_BIC(BIC_USEC)) { @@ -4532,8 +4547,7 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char * { if (mp->msr_num != 0) { assert(!no_msr); - if (get_msr(cpu, mp->msr_num, counterp)) - return -1; + get_msr(cpu, mp->msr_num, counterp); } else { char path[128 + PATH_BYTES]; @@ -4897,13 +4911,10 @@ int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct fprintf(stderr, "Reading rapl counter via msr at %u\n", i); assert(!no_msr); - if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) { - if (get_msr_sum(cpu, rci->msr[i], &rci->data[i])) - return -13 - i; - } else { - if (get_msr(cpu, rci->msr[i], &rci->data[i])) - return -13 - i; - } + if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) + get_msr_sum(cpu, rci->msr[i], &rci->data[i]); + else + get_msr(cpu, rci->msr[i], &rci->data[i]); rci->data[i] &= rci->msr_mask[i]; if (rci->msr_shift[i] >= 0) @@ -5032,8 +5043,7 @@ int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS) case COUNTER_SOURCE_MSR: assert(!no_msr); - if (get_msr(cpu, cci->msr[i], &cci->data[i])) - return -13 - i; + get_msr(cpu, cci->msr[i], &cci->data[i]); if (debug >= 2) fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]); @@ -5131,8 +5141,7 @@ int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t) case COUNTER_SOURCE_MSR: assert(!no_msr); - if (get_msr(cpu, mci->msr[i], &mci->data[i])) - return -2 - i; + get_msr(cpu, mci->msr[i], &mci->data[i]); mci->data[i] &= mci->msr_mask[i]; @@ -5297,12 +5306,10 @@ int get_counters(PER_THREAD_PARAMS) } if (DO_BIC(BIC_Mod_c6)) - if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us)) - return -8; + get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us); if (DO_BIC(BIC_CoreTmp)) { - if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) - return -9; + get_msr(cpu, MSR_IA32_THERM_STATUS, &msr); c->core_temp_c = tj_max - ((msr >> 16) & 0x7F); } @@ -5324,22 +5331,14 @@ int get_counters(PER_THREAD_PARAMS) if (!is_cpu_first_core_in_package(t, p)) goto done; - if (DO_BIC(BIC_Totl_c0)) { - if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0)) - return -10; - } - if (DO_BIC(BIC_Any_c0)) { - if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0)) - return -11; - } - if (DO_BIC(BIC_GFX_c0)) { - if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0)) - return -12; - } - if (DO_BIC(BIC_CPUGFX)) { - if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0)) - return -13; - } + if (DO_BIC(BIC_Totl_c0)) + get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0); + if (DO_BIC(BIC_Any_c0)) + get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0); + if (DO_BIC(BIC_GFX_c0)) + get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0); + if (DO_BIC(BIC_CPUGFX)) + get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0); if (DO_BIC(BIC_CPU_LPI)) p->cpu_lpi = cpuidle_cur_cpu_lpi_us; @@ -5353,8 +5352,7 @@ int get_counters(PER_THREAD_PARAMS) } if (DO_BIC(BIC_PkgTmp)) { - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) - return -17; + get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr); p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F); } @@ -5833,7 +5831,7 @@ void free_fd_percpu(void) if (!fd_percpu) return; - for (i = 0; i < topo.max_cpu_num + 1; ++i) { + for (i = 0; i <= topo.max_cpu_num; ++i) { if (fd_percpu[i] != 0) close(fd_percpu[i]); } @@ -5847,7 +5845,7 @@ void free_fd_instr_count_percpu(void) if (!fd_instr_count_percpu) return; - for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + for (int i = 0; i <= topo.max_cpu_num; ++i) { if (fd_instr_count_percpu[i] != 0) close(fd_instr_count_percpu[i]); } @@ -5861,7 +5859,7 @@ void free_fd_llc_percpu(void) if (!fd_llc_percpu) return; - for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + for (int i = 0; i <= topo.max_cpu_num; ++i) { if (fd_llc_percpu[i] != 0) close(fd_llc_percpu[i]); } @@ -5878,7 +5876,7 @@ void free_fd_l2_percpu(void) if (!fd_l2_percpu) return; - for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + for (int i = 0; i <= topo.max_cpu_num; ++i) { if (fd_l2_percpu[i] != 0) close(fd_l2_percpu[i]); } @@ -5915,7 +5913,7 @@ void free_fd_msr(void) if (!msr_counter_info) return; - for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) { + for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { if (msr_counter_info[cpu].fd_perf != -1) close(msr_counter_info[cpu].fd_perf); } @@ -5978,19 +5976,15 @@ void free_all_buffers(void) CPU_FREE(cpu_present_set); cpu_present_set = NULL; - cpu_present_setsize = 0; CPU_FREE(cpu_effective_set); cpu_effective_set = NULL; - cpu_effective_setsize = 0; CPU_FREE(cpu_allowed_set); cpu_allowed_set = NULL; - cpu_allowed_setsize = 0; CPU_FREE(cpu_affinity_set); cpu_affinity_set = NULL; - cpu_affinity_setsize = 0; if (perf_pcore_set) { CPU_FREE(perf_pcore_set); @@ -6165,7 +6159,7 @@ int get_physical_node_id(struct cpu_topology *thiscpu) static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) { - unsigned int start, end; + int start, end; char *next = cpu_str; while (next && *next) { @@ -6178,7 +6172,7 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) start = strtoul(next, &next, 10); - if (start >= CPU_SUBSET_MAXCPUS) + if (start > topo.max_cpu_num) return 1; CPU_SET_S(start, cpu_set_size, cpu_set); @@ -6205,7 +6199,7 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) return 1; while (++start <= end) { - if (start >= CPU_SUBSET_MAXCPUS) + if (start > topo.max_cpu_num) return 1; CPU_SET_S(start, cpu_set_size, cpu_set); } @@ -6345,7 +6339,7 @@ static void update_effective_set(bool startup) { update_effective_str(startup); - if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize)) + if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_setsize)) err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str); } @@ -6373,26 +6367,6 @@ void re_initialize(void) fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); } -void set_max_cpu_num(void) -{ - FILE *filep; - int current_cpu; - unsigned long dummy; - char pathname[64]; - - current_cpu = sched_getcpu(); - if (current_cpu < 0) - err(1, "cannot find calling cpu ID"); - sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu); - - filep = fopen_or_die(pathname, "r"); - topo.max_cpu_num = 0; - while (fscanf(filep, "%lx,", &dummy) == 1) - topo.max_cpu_num += BITMASK_SIZE; - fclose(filep); - topo.max_cpu_num--; /* 0 based */ -} - /* * count_cpus() * remember the last one seen, it will be the max @@ -6407,7 +6381,7 @@ int count_cpus(int cpu) int mark_cpu_present(int cpu) { - CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set); + CPU_SET_S(cpu, cpu_setsize, cpu_present_set); return 0; } @@ -6710,7 +6684,7 @@ void do_sleep(void) int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) { - int ret, idx; + int idx; unsigned long long msr_cur, msr_last; assert(!no_msr); @@ -6722,9 +6696,7 @@ int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) if (idx < 0) return idx; /* get_msr_sum() = sum + (get_msr() - last) */ - ret = get_msr(cpu, offset, &msr_cur); - if (ret) - return ret; + get_msr(cpu, offset, &msr_cur); msr_last = per_cpu_msr_sum[cpu].entries[idx].last; DELTA_WRAP32(msr_cur, msr_last); *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum; @@ -6737,7 +6709,7 @@ timer_t timerid; /* Timer callback, update the sum of MSRs periodically. */ static int update_msr_sum(PER_THREAD_PARAMS) { - int i, ret; + int i; int cpu = t->cpu_id; UNUSED(c); @@ -6754,11 +6726,7 @@ static int update_msr_sum(PER_THREAD_PARAMS) offset = idx_to_offset(i); if (offset < 0) continue; - ret = get_msr(cpu, offset, &msr_cur); - if (ret) { - fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset); - continue; - } + get_msr(cpu, offset, &msr_cur); msr_last = per_cpu_msr_sum[cpu].entries[i].last; per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff; @@ -7568,27 +7536,21 @@ int print_hwp(PER_THREAD_PARAMS) return -1; } - if (get_msr(cpu, MSR_PM_ENABLE, &msr)) - return 0; - + get_msr(cpu, MSR_PM_ENABLE, &msr); fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-"); /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */ if ((msr & (1 << 0)) == 0) return 0; - if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr)) - return 0; - + get_msr(cpu, MSR_HWP_CAPABILITIES, &msr); fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx " "(high %d guar %d eff %d low %d)\n", cpu, msr, (unsigned int)HWP_HIGHEST_PERF(msr), (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); - if (get_msr(cpu, MSR_HWP_REQUEST, &msr)) - return 0; - + get_msr(cpu, MSR_HWP_REQUEST, &msr); fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx " "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n", cpu, msr, @@ -7598,9 +7560,7 @@ int print_hwp(PER_THREAD_PARAMS) (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); if (has_hwp_pkg) { - if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr)) - return 0; - + get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr); fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx " "(min %d max %d des %d epp 0x%x window 0x%x)\n", cpu, msr, @@ -7609,15 +7569,11 @@ int print_hwp(PER_THREAD_PARAMS) (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); } if (has_hwp_notify) { - if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr)) - return 0; - + get_msr(cpu, MSR_HWP_INTERRUPT, &msr); fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx " "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); } - if (get_msr(cpu, MSR_HWP_STATUS, &msr)) - return 0; - + get_msr(cpu, MSR_HWP_STATUS, &msr); fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx " "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); @@ -7730,9 +7686,10 @@ double get_tdp_intel(void) { unsigned long long msr; - if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) - if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr)) - return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { + get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr); + return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; + } return get_quirk_tdp(); } @@ -7770,8 +7727,7 @@ void rapl_probe_intel(void) CLR_BIC(BIC_RAM__, &bic_enabled); /* units on package 0, verify later other packages match */ - if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr)) - return; + get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr); rapl_power_units = 1.0 / (1 << (msr & 0xF)); if (platform->has_rapl_divisor) @@ -7818,9 +7774,7 @@ void rapl_probe_amd(void) if (!valid_rapl_msrs || no_msr) return; - if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr)) - return; - + get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr); rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf)); rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f)); rapl_power_units = ldexp(1.0, -(msr & 0xf)); @@ -7976,7 +7930,7 @@ static int print_rapl_sysfs(void) snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name); if ((cdir = opendir(path)) == NULL) { - perror("opendir() error"); + warn("%s", path); return 1; } @@ -8019,21 +7973,17 @@ int print_rapl(PER_THREAD_PARAMS) if (valid_rapl_msrs & RAPL_AMD_F17H) { msr_name = "MSR_RAPL_PWR_UNIT"; - if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr)) - return -1; + get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr); } else { msr_name = "MSR_RAPL_POWER_UNIT"; - if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr)) - return -1; + get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr); } fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units); if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { - if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr)) - return -5; - + get_msr(cpu, MSR_PKG_POWER_INFO, &msr); fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, @@ -8043,9 +7993,7 @@ int print_rapl(PER_THREAD_PARAMS) } if (valid_rapl_msrs & RAPL_PKG) { - if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr)) - return -9; - + get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr); fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "PKG Limit #1"); @@ -8055,17 +8003,13 @@ int print_rapl(PER_THREAD_PARAMS) ((msr >> 32) & 0x7FFF) * rapl_power_units, (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS"); - if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr)) - return -9; - + get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr); fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr); fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); } if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) { - if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr)) - return -6; - + get_msr(cpu, MSR_DRAM_POWER_INFO, &msr); fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, @@ -8073,32 +8017,25 @@ int print_rapl(PER_THREAD_PARAMS) ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } if (valid_rapl_msrs & RAPL_DRAM) { - if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr)) - return -9; + get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr); fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "DRAM Limit"); } if (valid_rapl_msrs & RAPL_CORE_POLICY) { - if (get_msr(cpu, MSR_PP0_POLICY, &msr)) - return -7; - + get_msr(cpu, MSR_PP0_POLICY, &msr); fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF); } if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) { - if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr)) - return -9; + get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr); fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "Cores Limit"); } if (valid_rapl_msrs & RAPL_GFX) { - if (get_msr(cpu, MSR_PP1_POLICY, &msr)) - return -8; - + get_msr(cpu, MSR_PP1_POLICY, &msr); fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF); - if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr)) - return -9; + get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr); fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "GFX Limit"); } @@ -8113,7 +8050,6 @@ int print_rapl(PER_THREAD_PARAMS) */ void probe_rapl_msrs(void) { - int ret; off_t offset; unsigned long long msr_value; @@ -8127,12 +8063,7 @@ void probe_rapl_msrs(void) if (offset < 0) return; - ret = get_msr(master_cpu, offset, &msr_value); - if (ret) { - if (debug) - fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset); - return; - } + get_msr(master_cpu, offset, &msr_value); if (msr_value == 0) { if (debug) fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset); @@ -8212,17 +8143,17 @@ int set_temperature_target(PER_THREAD_PARAMS) if (!platform->has_nhm_msrs || no_msr) goto guess; - if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) - goto guess; - + get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr); tcc_default = (msr >> 16) & 0xFF; if (!quiet) { int bits = platform->tcc_offset_bits; unsigned long long enabled = 0; - if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled)) + if (bits) { + get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled); enabled = (enabled >> 30) & 1; + } if (bits && enabled) { tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0); @@ -8274,15 +8205,11 @@ int print_thermal(PER_THREAD_PARAMS) } if (do_ptm && is_cpu_first_core_in_package(t, p)) { - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) - return 0; - + get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr); dts = (msr >> 16) & 0x7F; fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts); - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr)) - return 0; - + get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr); dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); @@ -8291,16 +8218,12 @@ int print_thermal(PER_THREAD_PARAMS) if (do_dts && debug) { unsigned int resolution; - if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) - return 0; - + get_msr(cpu, MSR_IA32_THERM_STATUS, &msr); dts = (msr >> 16) & 0x7F; resolution = (msr >> 27) & 0xF; fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution); - if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr)) - return 0; - + get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr); dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); @@ -8359,9 +8282,9 @@ void decode_feature_control_msr(void) if (quiet) return; - if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr)) - fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", - master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); + get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr); + fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", + master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); } void decode_misc_enable_msr(void) @@ -8374,13 +8297,13 @@ void decode_misc_enable_msr(void) if (!genuine_intel) return; - if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr)) - fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", - master_cpu, msr, - msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); + get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr); + fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", + master_cpu, msr, + msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); } void decode_misc_feature_control(void) @@ -8393,10 +8316,10 @@ void decode_misc_feature_control(void) if (!platform->has_msr_misc_feature_control) return; - if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) - fprintf(outf, - "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", - master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); + get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr); + fprintf(outf, + "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", + master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); } /* @@ -8416,9 +8339,9 @@ void decode_misc_pwr_mgmt_msr(void) if (!platform->has_msr_misc_pwr_mgmt) return; - if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr)) - fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", - master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); + get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr); + fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", + master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); } /* @@ -8437,11 +8360,11 @@ void decode_c6_demotion_policy_msr(void) if (!platform->has_msr_c6_demotion_policy_config) return; - if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr); + fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); - if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr); + fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); } void print_dev_latency(void) @@ -8547,7 +8470,7 @@ int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sy warn("read %s", sysfs_path); goto err; } - if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) { + if (parse_cpu_str(cpuset_buf, cpu_set, cpu_setsize)) { warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str); goto err; } @@ -8583,7 +8506,7 @@ void linux_perf_init_hybrid_cpus(void) perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (perf_pcore_set == NULL) err(3, "CPU_ALLOC"); - CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set); + CPU_ZERO_S(cpu_setsize, perf_pcore_set); initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus"); if (debug) print_cpu_set("perf pcores", perf_pcore_set); @@ -8595,7 +8518,7 @@ void linux_perf_init_hybrid_cpus(void) perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (perf_ecore_set == NULL) err(3, "CPU_ALLOC"); - CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set); + CPU_ZERO_S(cpu_setsize, perf_ecore_set); initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus"); if (debug) print_cpu_set("perf ecores", perf_ecore_set); @@ -8607,7 +8530,7 @@ void linux_perf_init_hybrid_cpus(void) perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (perf_lcore_set == NULL) err(3, "CPU_ALLOC"); - CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set); + CPU_ZERO_S(cpu_setsize, perf_lcore_set); initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus"); if (debug) print_cpu_set("perf lcores", perf_lcore_set); @@ -8691,7 +8614,7 @@ void rapl_perf_init(void) memset(domain_visited, 0, num_domains * sizeof(*domain_visited)); - for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) { + for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { if (cpu_is_not_allowed(cpu)) continue; @@ -9136,13 +9059,10 @@ void process_cpuid() cpuid_has_hv = ecx_flags & (1 << 31); if (!no_msr) { - if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) { - warnx("get_msr(UCODE)"); - } else { - ucode_patch_valid = true; - if (!authentic_amd && !hygon_genuine) - ucode_patch >>= 32; - } + get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch); + ucode_patch_valid = true; + if (!authentic_amd && !hygon_genuine) + ucode_patch >>= 32; } /* @@ -9432,7 +9352,7 @@ void perf_l2_init(void) } continue; } - if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) { + if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_pcore_set)) { fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP); if (fd_l2_percpu[cpu] == -1) { warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs); @@ -9445,7 +9365,7 @@ void perf_l2_init(void) free_fd_l2_percpu(); return; } - } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) { + } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_ecore_set)) { fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP); if (fd_l2_percpu[cpu] == -1) { warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.refs); @@ -9458,7 +9378,7 @@ void perf_l2_init(void) free_fd_l2_percpu(); return; } - } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) { + } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_lcore_set)) { fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP); if (fd_l2_percpu[cpu] == -1) { warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.refs); @@ -9521,6 +9441,35 @@ int set_thread_siblings(struct cpu_topology *thiscpu) return (ht_id - 1); } +/* + * get_max_cpu_num(path) + * return the last number of the cpu list in this file + */ +static int get_max_cpu_num(char *path) +{ + FILE *filep; + int retval, num; + char c; + + filep = fopen(path, "r"); + if (filep == NULL) { + warn("%s", path); + return 255; + } + + while ((retval = fscanf(filep, "%d%c", &num, &c)) == 2) { + if (c == '\n') + break; + if (c != ',' && c != '-') + errx(1, "Bad Format '%s'", path); + } + if (retval != 2) + errx(1, "Bad format '%s'", path); + + fclose(filep); + return num; +} + void topology_probe(bool startup) { int i; @@ -9528,8 +9477,7 @@ void topology_probe(bool startup) int max_package_id = 0; int max_siblings = 0; - /* Initialize num_cpus, max_cpu_num */ - set_max_cpu_num(); + /* Initialize num_cpus */ topo.num_cpus = 0; for_all_proc_cpus(count_cpus); if (!summary_only) @@ -9548,8 +9496,7 @@ void topology_probe(bool startup) cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (cpu_present_set == NULL) err(3, "CPU_ALLOC"); - cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_present_setsize, cpu_present_set); + CPU_ZERO_S(cpu_setsize, cpu_present_set); for_all_proc_cpus(mark_cpu_present); if (debug) print_cpu_set("present set", cpu_present_set); @@ -9560,8 +9507,7 @@ void topology_probe(bool startup) cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (cpu_possible_set == NULL) err(3, "CPU_ALLOC"); - cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set); + CPU_ZERO_S(cpu_setsize, cpu_possible_set); initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible"); if (debug) print_cpu_set("possible set", cpu_possible_set); @@ -9572,8 +9518,7 @@ void topology_probe(bool startup) cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (cpu_effective_set == NULL) err(3, "CPU_ALLOC"); - cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set); + CPU_ZERO_S(cpu_setsize, cpu_effective_set); update_effective_set(startup); if (debug) print_cpu_set("effective set", cpu_effective_set); @@ -9584,48 +9529,47 @@ void topology_probe(bool startup) cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (cpu_allowed_set == NULL) err(3, "CPU_ALLOC"); - cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set); + CPU_ZERO_S(cpu_setsize, cpu_allowed_set); /* * Validate and update cpu_allowed_set. * - * Make sure all cpus in cpu_subset are also in cpu_present_set during startup. - * Give a warning when cpus in cpu_subset become unavailable at runtime. + * Make sure all cpus in cpuset_cmdline are also in cpu_present_set during startup. + * Give a warning when cpus in cpuset_cmdline become unavailable at runtime. * Give a warning when cpus are not effective because of cgroup setting. * - * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset. + * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpuset_cmdline. */ - for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) { - if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset)) + for (i = 0; i <= topo.max_cpu_num; ++i) { + if (cpuset_cmdline && !CPU_ISSET_S(i, cpu_setsize, cpuset_cmdline)) continue; - if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) { - if (cpu_subset) { - /* cpus in cpu_subset must be in cpu_present_set during startup */ + if (!CPU_ISSET_S(i, cpu_setsize, cpu_present_set)) { + if (cpuset_cmdline) { + /* cpus in cpuset_cmdline must be in cpu_present_set during startup */ if (startup) - err(1, "cpu%d not present", i); + errx(1, "cpu%d not present", i); else fprintf(stderr, "cpu%d not present\n", i); } continue; } - if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) { - if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) { + if (CPU_COUNT_S(cpu_setsize, cpu_effective_set)) { + if (!CPU_ISSET_S(i, cpu_setsize, cpu_effective_set)) { fprintf(stderr, "cpu%d not effective\n", i); continue; } } - CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set); + CPU_SET_S(i, cpu_setsize, cpu_allowed_set); } if (debug) print_cpu_set("allowed set", cpu_allowed_set); - if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set)) - err(-ENODEV, "No valid cpus found"); - sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set); + if (!CPU_COUNT_S(cpu_setsize, cpu_allowed_set)) + errx(-ENODEV, "No valid cpus found"); + sched_setaffinity(0, cpu_setsize, cpu_allowed_set); /* * Allocate and initialize cpu_affinity_set @@ -9633,8 +9577,7 @@ void topology_probe(bool startup) cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1)); if (cpu_affinity_set == NULL) err(3, "CPU_ALLOC"); - cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); + CPU_ZERO_S(cpu_setsize, cpu_affinity_set); for_all_proc_cpus(clear_ht_id); @@ -9782,7 +9725,7 @@ void allocate_counters(struct counters *counters) if (counters->threads == NULL) goto error; - for (i = 0; i < topo.max_cpu_num + 1; i++) + for (i = 0; i <= topo.max_cpu_num; i++) (counters->threads)[i].cpu_id = -1; counters->cores = calloc(num_cores, sizeof(struct core_data)); @@ -9911,7 +9854,7 @@ void set_master_cpu(void) { int i; - for (i = 0; i < topo.max_cpu_num + 1; ++i) { + for (i = 0; i <= topo.max_cpu_num; ++i) { if (cpu_is_not_allowed(i)) continue; master_cpu = i; @@ -10028,7 +9971,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited)); - for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) { + for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { next_domain = cpu_to_domain(pinfo, cpu); @@ -10439,7 +10382,7 @@ void pmt_init(void) mod_num = 0; /* Relative module number for current PMT file. */ /* Open the counter for each CPU. */ - for (cpu_num = 0; cpu_num < topo.max_cpu_num;) { + for (cpu_num = 0; cpu_num <= topo.max_cpu_num;) { if (cpu_is_not_allowed(cpu_num)) goto next_loop_iter; @@ -10525,11 +10468,11 @@ void turbostat_init() void affinitize_child(void) { /* Prefer cpu_possible_set, if available */ - if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) { + if (sched_setaffinity(0, cpu_setsize, cpu_possible_set)) { warn("sched_setaffinity cpu_possible_set"); /* Otherwise, allow child to run on same cpu set as turbostat */ - if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set)) + if (sched_setaffinity(0, cpu_setsize, cpu_allowed_set)) warn("sched_setaffinity cpu_allowed_set"); } } @@ -10634,7 +10577,7 @@ void print_bootcmd(void) fclose(fp); } -struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) +struct msr_counter *find_msrp_by_name(struct msr_counter *head, const char *name) { struct msr_counter *mp; @@ -10647,7 +10590,7 @@ struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) return NULL; } -int add_counter(unsigned int msr_num, char *path, char *name, +int add_counter(unsigned int msr_num, const char *path, const char *name, unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id) { struct msr_counter *msrp; @@ -10668,10 +10611,11 @@ int add_counter(unsigned int msr_num, char *path, char *name, fprintf(stderr, "%s: %s FOUND\n", __func__, name); break; } - if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) { + if (sys.added_thread_counters >= MAX_ADDED_THREAD_COUNTERS) { warnx("ignoring thread counter %s", name); return -1; } + sys.added_thread_counters++; break; case SCOPE_CORE: msrp = find_msrp_by_name(sys.cp, name); @@ -10680,10 +10624,11 @@ int add_counter(unsigned int msr_num, char *path, char *name, fprintf(stderr, "%s: %s FOUND\n", __func__, name); break; } - if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) { + if (sys.added_core_counters >= MAX_ADDED_CORE_COUNTERS) { warnx("ignoring core counter %s", name); return -1; } + sys.added_core_counters++; break; case SCOPE_PACKAGE: msrp = find_msrp_by_name(sys.pp, name); @@ -10692,10 +10637,11 @@ int add_counter(unsigned int msr_num, char *path, char *name, fprintf(stderr, "%s: %s FOUND\n", __func__, name); break; } - if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) { + if (sys.added_package_counters >= MAX_ADDED_PACKAGE_COUNTERS) { warnx("ignoring package counter %s", name); return -1; } + sys.added_package_counters++; break; default: warnx("ignoring counter %s with unknown scope", name); @@ -10734,10 +10680,8 @@ int add_counter(unsigned int msr_num, char *path, char *name, struct sysfs_path *sp; sp = calloc(1, sizeof(struct sysfs_path)); - if (sp == NULL) { - perror("calloc"); - exit(1); - } + if (sp == NULL) + err(-1, "%s", path); strncpy(sp->path, path, PATH_BYTES - 1); sp->id = id; sp->next = msrp->sp; @@ -11437,18 +11381,18 @@ void probe_cpuidle_counts(void) /* * parse cpuset with following syntax - * 1,2,4..6,8-10 and set bits in cpu_subset + * 1,2,4..6,8-10 and set bits in cpuset_cmdline */ void parse_cpu_command(char *optarg) { if (!strcmp(optarg, "core")) { - if (cpu_subset) + if (cpuset_cmdline) goto error; show_core_only++; return; } if (!strcmp(optarg, "package")) { - if (cpu_subset) + if (cpuset_cmdline) goto error; show_pkg_only++; return; @@ -11456,15 +11400,23 @@ void parse_cpu_command(char *optarg) if (show_core_only || show_pkg_only) goto error; - cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS); - if (cpu_subset == NULL) - err(3, "CPU_ALLOC"); - cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS); + /* + * cpuset_cmdline is allocated on the first invocation of --cpu + * It is re-used for subsequent invocations. + * It is never freed. + */ + if (!cpuset_cmdline) { + cpuset_cmdline = CPU_ALLOC(topo.max_cpu_num + 1); + if (cpuset_cmdline == NULL) + err(3, "CPU_ALLOC"); - CPU_ZERO_S(cpu_subset_size, cpu_subset); + CPU_ZERO_S(cpu_setsize, cpuset_cmdline); + } - if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size)) + if (parse_cpu_str(optarg, cpuset_cmdline, cpu_setsize)) goto error; + if (debug) + print_cpu_set("--cpu", cpuset_cmdline); return; @@ -11546,7 +11498,7 @@ void cmdline(int argc, char **argv) bic_lookup(&bic_enabled, optarg, SHOW_LIST); break; case 'f': - force_load++; + force_load = true; break; case 'd': debug++; @@ -11558,7 +11510,7 @@ void cmdline(int argc, char **argv) * multiple invocations simply clear more bits in enabled mask */ { - cpu_set_t bic_group_hide; + bic_set_t bic_group_hide; BIC_INIT(&bic_group_hide); @@ -11671,11 +11623,16 @@ int main(int argc, char **argv) ret = write(fd, "0\n", 2); if (ret == -1) - perror("Can't update cgroup\n"); + warn("/sys/fs/cgroup/cgroup.procs"); close(fd); skip_cgroup_setting: + + /* probe max_cpu_num and set cpu_setsize early for use by cmdline() */ + topo.max_cpu_num = get_max_cpu_num("/sys/devices/system/cpu/possible"); + cpu_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + outf = stderr; cmdline(argc, argv); @@ -11698,8 +11655,10 @@ int main(int argc, char **argv) msr_sum_record(); /* dump counters and exit */ - if (dump_only) + if (dump_only) { + dump_bic_sets(); return get_and_dump_counters(); + } /* list header and exit */ if (list_header_only) { diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 9d89bb57e201..db247e42fb45 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * BPF-side helpers for cids and cmasks. See kernel/sched/ext_cid.h for the + * BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the * authoritative layout and semantics. The BPF-side helpers use the cmask_* * naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is * redundant in BPF code. Atomics use __sync_val_compare_and_swap and every @@ -33,7 +33,7 @@ #endif /* - * Mirrors SCX_CMASK_NR_WORDS in kernel/sched/ext_types.h. The u64 cast keeps + * Mirrors SCX_CMASK_NR_WORDS in kernel/sched/ext/types.h. The u64 cast keeps * the +63 from wrapping when @nr_cids is near U32_MAX, so cmask_reframe() * bounds-checking the result against alloc_words catches the overflow instead * of seeing a small value. @@ -281,7 +281,7 @@ static __always_inline void cmask_zero(struct scx_cmask __arena *m) /* * BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_* - * enum in ext_cid.c, which is exported via BTF and reachable through + * enum in ext/cid.c, which is exported via BTF and reachable through * vmlinux.h. */ enum { diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 9591a6e778ce..bd51986c4c42 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -96,7 +96,6 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; bool scx_bpf_task_running(const struct task_struct *p) __ksym; s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; -struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym; struct rq *scx_bpf_locked_rq(void) __ksym; struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; @@ -983,8 +982,8 @@ extern struct irqtime___local cpu_irqtime __ksym __weak; static inline struct rq___local *get_current_rq(u32 cpu) { /* - * This is a workaround to get an rq pointer since we decided to - * deprecate scx_bpf_cpu_rq(). + * This is a workaround to get an rq pointer now that + * scx_bpf_cpu_rq() has been removed. * * WARNING: The caller must hold the rq lock for @cpu. This is * guaranteed when called from scheduling callbacks (ops.running, diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 87f15f296234..133058578668 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -233,23 +233,6 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) -/* - * v6.18: Add a helper to retrieve the current task running on a CPU. - * - * Keep this helper available until v6.20 for compatibility. - */ -static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) -{ - struct rq *rq; - - if (bpf_ksym_exists(scx_bpf_cpu_curr)) - return scx_bpf_cpu_curr(cpu); - - rq = scx_bpf_cpu_rq(cpu); - - return rq ? rq->curr : NULL; -} - /* * v6.19: To work around BPF maximum parameter limit, the following kfuncs are * replaced with variants that pack scalar arguments in a struct. Wrappers are diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 602f07061ee3..23d9ef3e4c9d 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -28,7 +28,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v const struct btf_type *t; const char *n; s32 tid; - int i; + __u32 i; __COMPAT_load_vmlinux_btf(); @@ -42,7 +42,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v if (btf_is_enum(t)) { struct btf_enum *e = btf_enum(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { @@ -53,7 +53,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v } else if (btf_is_enum64(t)) { struct btf_enum64 *e = btf_enum64(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { @@ -85,7 +85,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field const struct btf_member *m; const char *n; s32 tid; - int i; + __u32 i; __COMPAT_load_vmlinux_btf(); tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT); @@ -97,7 +97,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field m = btf_members(t); - for (i = 0; i < BTF_INFO_VLEN(t->info); i++) { + for (i = 0; i < btf_vlen(t); i++) { n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, field)) diff --git a/tools/sched_ext/scx_cpu0.c b/tools/sched_ext/scx_cpu0.c index 4966e3d4c724..84a47aee2f95 100644 --- a/tools/sched_ext/scx_cpu0.c +++ b/tools/sched_ext/scx_cpu0.c @@ -62,7 +62,7 @@ int main(int argc, char **argv) { struct scx_cpu0 *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index fec359581826..0fd214cc61da 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) * and thus can't be updated and repositioned. Instead, we collect the * vtime deltas separately and apply it asynchronously here. */ - delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0); cvtime = cgv_node->cvtime + delta; /* @@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) cgc = find_cgrp_ctx(cgrp); if (cgc) { __sync_fetch_and_add(&cgc->cvtime_delta, - p->se.sum_exec_runtime - taskc->bypassed_at); + (p->se.sum_exec_runtime - taskc->bypassed_at) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); taskc->bypassed_at = 0; } bpf_cgroup_release(cgrp); diff --git a/tools/sched_ext/scx_sdt.c b/tools/sched_ext/scx_sdt.c index bf664b2d3785..ef197b266a36 100644 --- a/tools/sched_ext/scx_sdt.c +++ b/tools/sched_ext/scx_sdt.c @@ -44,7 +44,7 @@ int main(int argc, char **argv) { struct scx_sdt *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); diff --git a/tools/sched_ext/scx_simple.c b/tools/sched_ext/scx_simple.c index c3b48611712b..34f9785335b7 100644 --- a/tools/sched_ext/scx_simple.c +++ b/tools/sched_ext/scx_simple.c @@ -64,7 +64,7 @@ int main(int argc, char **argv) { struct scx_simple *skel; struct bpf_link *link; - __u32 opt; + __s32 opt; __u64 ecode; libbpf_set_print(libbpf_print_fn); diff --git a/tools/sched_ext/scx_userland.c b/tools/sched_ext/scx_userland.c index 616043c165e6..192b79c7e4aa 100644 --- a/tools/sched_ext/scx_userland.c +++ b/tools/sched_ext/scx_userland.c @@ -326,7 +326,7 @@ static int spawn_stats_thread(void) static void pre_bootstrap(int argc, char **argv) { int err; - __u32 opt; + __s32 opt; struct sched_param sched_param = { .sched_priority = sched_get_priority_max(SCHED_EXT), }; diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index a1d170f88fee..a7da279aa3ef 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -2,7 +2,6 @@ // Copyright(c) 2021 Intel Corporation. All rights reserved. #include -#include #include #include #include diff --git a/tools/testing/memblock/linux/mmzone.h b/tools/testing/memblock/linux/mmzone.h index bb682659a12d..8d934ff5b080 100644 --- a/tools/testing/memblock/linux/mmzone.h +++ b/tools/testing/memblock/linux/mmzone.h @@ -35,4 +35,8 @@ typedef struct pglist_data { } pg_data_t; +enum migratetype { + MIGRATE_CMA, +}; + #endif diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index 0607913a3022..d967e76a3c06 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) mt_set_non_kernel(1); /* Spanning store */ mas_set_range(&mas, 1, 100); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); @@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) MT_BUG_ON(mt, mas_allocated(&mas) != 0); mas_set_range(&mas, 0, 200); mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0); allocated = mas_allocated(&mas); height = mas_mt_height(&mas); MT_BUG_ON(mt, allocated != 0); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 8d4db2241cc2..8189f333814c 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -32,6 +32,7 @@ TARGETS += exec TARGETS += fchmodat2 TARGETS += filesystems TARGETS += filesystems/binderfs +TARGETS += filesystems/ceph TARGETS += filesystems/epoll TARGETS += filesystems/fat TARGETS += filesystems/overlayfs diff --git a/tools/testing/selftests/acct/.gitignore b/tools/testing/selftests/acct/.gitignore index 7e78aac19038..9e9c61c5bfd6 100644 --- a/tools/testing/selftests/acct/.gitignore +++ b/tools/testing/selftests/acct/.gitignore @@ -1,3 +1,4 @@ acct_syscall +taskstats_fill_stats_tgid config -process_log \ No newline at end of file +process_log diff --git a/tools/testing/selftests/acct/Makefile b/tools/testing/selftests/acct/Makefile index 7e025099cf65..083cab5ddb72 100644 --- a/tools/testing/selftests/acct/Makefile +++ b/tools/testing/selftests/acct/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := acct_syscall +TEST_GEN_PROGS += taskstats_fill_stats_tgid + CFLAGS += -Wall +LDLIBS += -lpthread -include ../lib.mk \ No newline at end of file +include ../lib.mk diff --git a/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c new file mode 100644 index 000000000000..d6cab4ae26f2 --- /dev/null +++ b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#ifndef NLA_ALIGN +#define NLA_ALIGNTO 4 +#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1)) +#define NLA_HDRLEN ((int)NLA_ALIGN(sizeof(struct nlattr))) +#endif + +#define BUSY_NS (200ULL * 1000 * 1000) + +struct worker_ctx { + pthread_mutex_t lock; + pthread_cond_t cond; + bool ready; + bool release; +}; + +static unsigned long busy_sink; + +static void *taskstats_nla_data(const struct nlattr *na) +{ + return (void *)((char *)na + NLA_HDRLEN); +} + +static bool taskstats_nla_ok(const struct nlattr *na, int remaining) +{ + return remaining >= (int)sizeof(*na) && + na->nla_len >= sizeof(*na) && + na->nla_len <= remaining; +} + +static struct nlattr *taskstats_nla_next(const struct nlattr *na, int *remaining) +{ + int aligned_len = NLA_ALIGN(na->nla_len); + + *remaining -= aligned_len; + return (struct nlattr *)((char *)na + aligned_len); +} + +static uint64_t timespec_diff_ns(const struct timespec *start, + const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +static void burn_cpu_for_ns(uint64_t runtime_ns) +{ + struct timespec start, now; + unsigned long acc = 0; + + if (clock_gettime(CLOCK_MONOTONIC, &start)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + + do { + for (int i = 0; i < 100000; i++) + acc += i; + if (clock_gettime(CLOCK_MONOTONIC, &now)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + } while (timespec_diff_ns(&start, &now) < runtime_ns); + + busy_sink = acc; +} + +static int netlink_open(void) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + }; + int fd; + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + if (fd < 0) + return -errno; + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + int err = -errno; + + close(fd); + return err; + } + + return fd; +} + +static int send_request(int fd, void *buf, size_t len) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + }; + + if (sendto(fd, buf, len, 0, (struct sockaddr *)&addr, sizeof(addr)) < 0) + return -errno; + + return 0; +} + +static int get_family_id(int fd, const char *name) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[8192]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + int len; + int rem; + int ret; + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = GENL_ID_CTRL; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 1; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = CTRL_CMD_GETFAMILY; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = CTRL_ATTR_FAMILY_NAME; + na->nla_len = NLA_HDRLEN + strlen(name) + 1; + memcpy(taskstats_nla_data(na), name, strlen(name) + 1); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (taskstats_nla_ok(na, rem)) { + if (na->nla_type == CTRL_ATTR_FAMILY_ID) + return *(uint16_t *)taskstats_nla_data(na); + na = taskstats_nla_next(na, &rem); + } + } + + return -ENOENT; +} + +static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id, + struct taskstats *stats) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[16384]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + struct nlattr *nested; + int len; + int rem; + int nrem; + int ret; + + memset(stats, 0, sizeof(*stats)); + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = family_id; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 2; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = TASKSTATS_CMD_GET; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = attr_type; + na->nla_len = NLA_HDRLEN + sizeof(id); + memcpy(taskstats_nla_data(na), &id, sizeof(id)); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (taskstats_nla_ok(na, rem)) { + if (na->nla_type == TASKSTATS_TYPE_AGGR_PID || + na->nla_type == TASKSTATS_TYPE_AGGR_TGID) { + nested = (struct nlattr *)taskstats_nla_data(na); + nrem = na->nla_len - NLA_HDRLEN; + while (taskstats_nla_ok(nested, nrem)) { + if (nested->nla_type == TASKSTATS_TYPE_STATS) { + memcpy(stats, taskstats_nla_data(nested), + sizeof(*stats)); + return 0; + } + nested = taskstats_nla_next(nested, &nrem); + } + } + na = taskstats_nla_next(na, &rem); + } + } + + return -ENOENT; +} + +static uint64_t cpu_total(const struct taskstats *stats) +{ + return (uint64_t)stats->ac_utime + (uint64_t)stats->ac_stime; +} + +static void print_stats(const char *label, const struct taskstats *stats) +{ + ksft_print_msg("%s: cpu_total=%llu nvcsw=%llu nivcsw=%llu\n", + label, (unsigned long long)cpu_total(stats), + (unsigned long long)stats->nvcsw, + (unsigned long long)stats->nivcsw); +} + +static void *worker_thread(void *arg) +{ + struct worker_ctx *ctx = arg; + + burn_cpu_for_ns(BUSY_NS); + + pthread_mutex_lock(&ctx->lock); + ctx->ready = true; + pthread_cond_broadcast(&ctx->cond); + while (!ctx->release) + pthread_cond_wait(&ctx->cond, &ctx->lock); + pthread_mutex_unlock(&ctx->lock); + + return NULL; +} + +int main(void) +{ + struct worker_ctx ctx = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, + }; + struct taskstats before, after; + pthread_t thread; + pid_t tgid = getpid(); + int family_id; + int fd; + int ret; + + ksft_print_header(); + ksft_set_plan(1); + + if (geteuid()) + ksft_exit_skip("taskstats_fill_stats_tgid needs root\n"); + + fd = netlink_open(); + if (fd < 0) + ksft_exit_skip("failed to open generic netlink socket: %s\n", + strerror(-fd)); + + family_id = get_family_id(fd, TASKSTATS_GENL_NAME); + if (family_id < 0) + ksft_exit_skip("taskstats generic netlink family unavailable: %s\n", + strerror(-family_id)); + + /* Create worker thread that burns 200ms of CPU */ + if (pthread_create(&thread, NULL, worker_thread, &ctx) != 0) + ksft_exit_fail_msg("pthread_create failed: %s\n", strerror(errno)); + + /* Wait for worker to finish generating activity */ + pthread_mutex_lock(&ctx.lock); + while (!ctx.ready) + pthread_cond_wait(&ctx.cond, &ctx.lock); + pthread_mutex_unlock(&ctx.lock); + + /* + * Snapshot A: TGID stats while worker is alive and sleeping. + * Contains main thread + worker contributions. + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &before); + if (ret) + ksft_exit_fail_msg("TGID query before exit failed: %s\n", + strerror(-ret)); + + /* Release worker so it can exit, then join (deterministic wait). + * + * Kernel exit path ordering guarantees: + * do_exit() + * taskstats_exit() -> fill_tgid_exit() (accumulates worker into signal->stats) + * exit_notify() (releases the thread) + * do_task_dead() -> __schedule() (wakes joiner) + * + * So pthread_join() returns only after fill_tgid_exit() has completed. + */ + pthread_mutex_lock(&ctx.lock); + ctx.release = true; + pthread_cond_broadcast(&ctx.cond); + pthread_mutex_unlock(&ctx.lock); + + pthread_join(thread, NULL); + + /* + * Snapshot B: TGID stats after worker has exited. + * fill_stats_for_tgid() does: + * memcpy(signal->stats) <- includes fill_tgid_exit accumulation + * + scan live threads <- only main thread now + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &after); + if (ret) + ksft_exit_fail_msg("TGID query after exit failed: %s\n", + strerror(-ret)); + + print_stats("TGID before worker exit", &before); + print_stats("TGID after worker exit", &after); + + /* + * The worker burned 200ms of CPU before the first snapshot. + * If the kernel correctly retained its contribution via + * fill_tgid_exit(), then the TGID CPU total after exit must be at + * least as large as the TGID CPU total before exit. + */ + ksft_test_result(cpu_total(&after) >= cpu_total(&before), + "TGID CPU stats should not regress after thread exit\n"); + + close(fd); + ksft_finished(); + return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; +} diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c index d4f845c32804..a329f901c5ed 100644 --- a/tools/testing/selftests/alsa/mixer-test.c +++ b/tools/testing/selftests/alsa/mixer-test.c @@ -84,6 +84,7 @@ static void find_controls(void) if (err < 0) { ksft_print_msg("Failed to get hctl for card %d: %s\n", card, snd_strerror(err)); + free(card_data); goto next_card; } @@ -339,9 +340,9 @@ static bool ctl_value_index_valid(struct ctl_data *ctl, } if (int64_val > snd_ctl_elem_info_get_max64(ctl->info)) { - ksft_print_msg("%s.%d value %lld more than maximum %ld\n", + ksft_print_msg("%s.%d value %lld more than maximum %lld\n", ctl->name, index, int64_val, - snd_ctl_elem_info_get_max(ctl->info)); + snd_ctl_elem_info_get_max64(ctl->info)); return false; } diff --git a/tools/testing/selftests/arm64/gcs/libc-gcs.c b/tools/testing/selftests/arm64/gcs/libc-gcs.c index 72e82bfbecc9..ddb0b2b20155 100644 --- a/tools/testing/selftests/arm64/gcs/libc-gcs.c +++ b/tools/testing/selftests/arm64/gcs/libc-gcs.c @@ -130,7 +130,7 @@ TEST(gcs_find_terminator) * We can access a GCS via ptrace * * This could usefully have a fixture but note that each test is - * fork()ed into a new child whcih causes issues. Might be better to + * fork()ed into a new child which causes issues. Might be better to * lift at least some of this out into a separate, non-harness, test * program. */ diff --git a/tools/testing/selftests/arm64/pauth/pac.c b/tools/testing/selftests/arm64/pauth/pac.c index 67d138057707..f4b859c75a5a 100644 --- a/tools/testing/selftests/arm64/pauth/pac.c +++ b/tools/testing/selftests/arm64/pauth/pac.c @@ -292,7 +292,7 @@ TEST(single_thread_different_keys) /* * fork() does not change keys. Only exec() does so call a worker program. - * Its only job is to sign a value and report back the resutls + * Its only job is to sign a value and report back the results */ TEST(exec_changed_keys) { diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 3d9d2cd7764b..c4a3a6b3eb83 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -539,12 +539,12 @@ extern const struct bench bench_trig_uretprobe_multi_push; extern const struct bench bench_trig_uprobe_multi_ret; extern const struct bench bench_trig_uretprobe_multi_ret; #ifdef __x86_64__ -extern const struct bench bench_trig_uprobe_nop5; -extern const struct bench bench_trig_uretprobe_nop5; -extern const struct bench bench_trig_uprobe_multi_nop5; -extern const struct bench bench_trig_uretprobe_multi_nop5; +extern const struct bench bench_trig_uprobe_nop10; +extern const struct bench bench_trig_uretprobe_nop10; +extern const struct bench bench_trig_uprobe_multi_nop10; +extern const struct bench bench_trig_uretprobe_multi_nop10; extern const struct bench bench_trig_usdt_nop; -extern const struct bench bench_trig_usdt_nop5; +extern const struct bench bench_trig_usdt_nop10; #endif extern const struct bench bench_rb_libbpf; @@ -622,12 +622,12 @@ static const struct bench *benchs[] = { &bench_trig_uprobe_multi_ret, &bench_trig_uretprobe_multi_ret, #ifdef __x86_64__ - &bench_trig_uprobe_nop5, - &bench_trig_uretprobe_nop5, - &bench_trig_uprobe_multi_nop5, - &bench_trig_uretprobe_multi_nop5, + &bench_trig_uprobe_nop10, + &bench_trig_uretprobe_nop10, + &bench_trig_uprobe_multi_nop10, + &bench_trig_uretprobe_multi_nop10, &bench_trig_usdt_nop, - &bench_trig_usdt_nop5, + &bench_trig_usdt_nop10, #endif /* ringbuf/perfbuf benchmarks */ &bench_rb_libbpf, diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c index 2f22ec61667b..61513efc167a 100644 --- a/tools/testing/selftests/bpf/benchs/bench_trigger.c +++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c @@ -396,15 +396,15 @@ static void *uprobe_producer_ret(void *input) } #ifdef __x86_64__ -__nocf_check __weak void uprobe_target_nop5(void) +__nocf_check __weak void uprobe_target_nop10(void) { - asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); + asm volatile (".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); } -static void *uprobe_producer_nop5(void *input) +static void *uprobe_producer_nop10(void *input) { while (true) - uprobe_target_nop5(); + uprobe_target_nop10(); return NULL; } @@ -418,7 +418,7 @@ static void *uprobe_producer_usdt_nop(void *input) return NULL; } -static void *uprobe_producer_usdt_nop5(void *input) +static void *uprobe_producer_usdt_nop10(void *input) { while (true) usdt_2(); @@ -542,24 +542,24 @@ static void uretprobe_multi_ret_setup(void) } #ifdef __x86_64__ -static void uprobe_nop5_setup(void) +static void uprobe_nop10_setup(void) { - usetup(false, false /* !use_multi */, &uprobe_target_nop5); + usetup(false, false /* !use_multi */, &uprobe_target_nop10); } -static void uretprobe_nop5_setup(void) +static void uretprobe_nop10_setup(void) { - usetup(true, false /* !use_multi */, &uprobe_target_nop5); + usetup(true, false /* !use_multi */, &uprobe_target_nop10); } -static void uprobe_multi_nop5_setup(void) +static void uprobe_multi_nop10_setup(void) { - usetup(false, true /* use_multi */, &uprobe_target_nop5); + usetup(false, true /* use_multi */, &uprobe_target_nop10); } -static void uretprobe_multi_nop5_setup(void) +static void uretprobe_multi_nop10_setup(void) { - usetup(true, true /* use_multi */, &uprobe_target_nop5); + usetup(true, true /* use_multi */, &uprobe_target_nop10); } static void usdt_setup(const char *name) @@ -598,7 +598,7 @@ static void usdt_nop_setup(void) usdt_setup("usdt_1"); } -static void usdt_nop5_setup(void) +static void usdt_nop10_setup(void) { usdt_setup("usdt_2"); } @@ -665,10 +665,10 @@ BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop"); BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push"); BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret"); #ifdef __x86_64__ -BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5"); -BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5"); -BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5"); -BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5"); +BENCH_TRIG_USERMODE(uprobe_nop10, nop10, "uprobe-nop10"); +BENCH_TRIG_USERMODE(uretprobe_nop10, nop10, "uretprobe-nop10"); +BENCH_TRIG_USERMODE(uprobe_multi_nop10, nop10, "uprobe-multi-nop10"); +BENCH_TRIG_USERMODE(uretprobe_multi_nop10, nop10, "uretprobe-multi-nop10"); BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop"); -BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5"); +BENCH_TRIG_USERMODE(usdt_nop10, usdt_nop10, "usdt-nop10"); #endif diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh index 9ec59423b949..e490b337e960 100755 --- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh +++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh @@ -2,7 +2,7 @@ set -eufo pipefail -for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5 +for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop10} usdt-nop usdt-nop10 do summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-) printf "%-15s: %s\n" $i "$summary" diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index d1db355e872b..63e4e472fe36 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -364,6 +364,9 @@ extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym; extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, struct bpf_dynptr *value_p) __weak __ksym; +extern int bpf_sock_read_xattr(struct socket *sock, const char *name__str, + struct bpf_dynptr *value_p) __weak __ksym; + #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 #define HARDIRQ_BITS 4 @@ -423,6 +426,8 @@ static inline int get_preempt_count(void) return bpf_get_current_task_btf()->thread_info.preempt_count; #elif defined(bpf_target_s390) return bpf_get_lowcore()->preempt_count; +#elif defined(bpf_target_loongarch) + return bpf_get_current_task_btf()->thread_info.preempt_count; #endif return 0; } @@ -433,6 +438,7 @@ static inline int get_preempt_count(void) * * arm64 * * powerpc64 * * s390x + * * loongarch */ static inline int bpf_in_interrupt(void) { @@ -454,6 +460,7 @@ static inline int bpf_in_interrupt(void) * * arm64 * * powerpc64 * * s390x + * * loongarch */ static inline int bpf_in_nmi(void) { @@ -466,6 +473,7 @@ static inline int bpf_in_nmi(void) * * arm64 * * powerpc64 * * s390x + * * loongarch */ static inline int bpf_in_hardirq(void) { @@ -478,6 +486,7 @@ static inline int bpf_in_hardirq(void) * * arm64 * * powerpc64 * * s390x + * * loongarch */ static inline int bpf_in_serving_softirq(void) { @@ -498,6 +507,7 @@ static inline int bpf_in_serving_softirq(void) * * arm64 * * powerpc64 * * s390x + * * loongarch */ static inline int bpf_in_task(void) { diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index bac60b444551..adb25146e88c 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -45,13 +45,16 @@ CONFIG_IPV6=y CONFIG_IPV6_FOU=y CONFIG_IPV6_FOU_TUNNEL=y CONFIG_IPV6_GRE=y +CONFIG_IPV6_IOAM6_LWTUNNEL=y CONFIG_IPV6_SEG6_BPF=y +CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SIT=y CONFIG_IPV6_TUNNEL=y CONFIG_KEYS=y CONFIG_LIRC=y CONFIG_LIVEPATCH=y CONFIG_LWTUNNEL=y +CONFIG_LWTUNNEL_BPF=y CONFIG_MODULE_SIG=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_UNLOAD=y diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7..db935a9d9fc1 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11..75133119c04a 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index b33dba4b126e..14d4c1793aed 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -5,6 +5,8 @@ #include "test_bpf_nf.skel.h" #include "test_bpf_nf_fail.skel.h" +#define CT_OPTS_ERROR_GUARD 0x12345678 + static char log_buf[1024 * 1024]; struct { @@ -119,6 +121,10 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->bss->test_einval_reserved_new, -EINVAL, "Test EINVAL for reserved in new struct not set to 0"); ASSERT_EQ(skel->bss->test_einval_netns_id, -EINVAL, "Test EINVAL for netns_id < -1"); ASSERT_EQ(skel->bss->test_einval_len_opts, -EINVAL, "Test EINVAL for len__opts != NF_BPF_CT_OPTS_SZ"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_lookup, CT_OPTS_ERROR_GUARD, + "Test no error write for lookup opts__sz before error field"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_alloc, CT_OPTS_ERROR_GUARD, + "Test no error write for alloc opts__sz before error field"); ASSERT_EQ(skel->bss->test_eproto_l4proto, -EPROTO, "Test EPROTO for l4proto != TCP or UDP"); ASSERT_EQ(skel->bss->test_enonet_netns_id, -ENONET, "Test ENONET for bad but valid netns_id"); ASSERT_EQ(skel->bss->test_enoent_lookup, -ENOENT, "Test ENOENT for failed lookup"); diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 96f719a0cec9..66855cbd6b73 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4121,8 +4121,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #3, type tag order", @@ -4141,8 +4139,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #4, type tag order", @@ -4161,8 +4157,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #5, type tag order", @@ -4198,11 +4192,9 @@ static struct btf_raw_test raw_tests[] = { .map_name = "tag_type_check_btf", .key_size = sizeof(int), .value_size = 4, - .key_type_id = 1, - .value_type_id = 1, + .key_type_id = 4, + .value_type_id = 4, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #7, tag with kflag", diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c index d4d583872fa2..d2ccf409dfe3 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c @@ -102,6 +102,82 @@ static int run_getsockopt_test(int cg_parent, int cg_child, int sock_fd, bool al return err; } +/* + * Replacing a link's program (bpf_link_update) must target the correct slot in + * the effective array even when a BPF_F_PREORDER program is attached to the + * same cgroup. All programs here are attached to a single cgroup; "parent" is + * reused only as a third distinct program. + * + * Attach child(1) normally and child_2(2) with BPF_F_PREORDER, so the effective + * order is [2, 1]. Then replace child(1)'s program with parent(3): only the + * non-preorder slot changes, giving [2, 3]. + */ +static int run_link_replace_test(int cgroup_fd, int sock_fd) +{ + LIBBPF_OPTS(bpf_link_create_opts, create_opts); + int err = 0, normal_link = -1, preorder_link = -1; + struct cgroup_preorder *skel = NULL; + enum bpf_attach_type atype; + __u8 *result, buf = 0x00; + socklen_t optlen = 1; + + skel = cgroup_preorder__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load")) + return -1; + + err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1); + if (!ASSERT_OK(err, "setsockopt")) + goto close_skel; + + atype = bpf_program__expected_attach_type(skel->progs.child); + + create_opts.flags = 0; + normal_link = bpf_link_create(bpf_program__fd(skel->progs.child), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(normal_link, 0, "create_normal_link")) { + err = normal_link; + goto close_skel; + } + + create_opts.flags = BPF_F_PREORDER; + preorder_link = bpf_link_create(bpf_program__fd(skel->progs.child_2), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(preorder_link, 0, "create_preorder_link")) { + err = preorder_link; + goto close_links; + } + + result = skel->bss->result; + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-before")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 1, "order before update"); + + /* Replace the normal link's program child(1) -> parent(3). */ + err = bpf_link_update(normal_link, bpf_program__fd(skel->progs.parent), NULL); + if (!ASSERT_OK(err, "bpf_link_update")) + goto close_links; + + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-after")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 3, "order after update"); + +close_links: + if (preorder_link >= 0) + close(preorder_link); + close(normal_link); +close_skel: + cgroup_preorder__destroy(skel); + return err; +} + void test_cgroup_preorder(void) { int cg_parent = -1, cg_child = -1, sock_fd = -1; @@ -120,6 +196,7 @@ void test_cgroup_preorder(void) ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1"); ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2"); + ASSERT_OK(run_link_replace_test(cg_child, sock_fd), "link_replace_test"); out: close(sock_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index f589eefbf9fb..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc451..32dfc1c511af 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 41dfaaabb73f..ac51fd454821 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -10,7 +10,11 @@ #include #include "test_progs.h" -static int duration; +#define BTF_DATA_FILE "resolve_btfids.test.o.BTF" + +#ifndef KF_FASTCALL +#define KF_FASTCALL (1 << 12) +#endif struct symbol { const char *name; @@ -28,12 +32,41 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +struct kfunc_symbol { + const char *name; + s32 id; + u32 flags; +}; + +static struct kfunc_symbol kfunc_symbols[] = { + { "kfunc_a", -1, 0 }, + { "kfunc_b", -1, KF_FASTCALL }, +}; + /* Align the .BTF_ids section to 4 bytes */ asm ( ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" ".balign 4, 0; \n" ".popsection; \n"); +/* + * test_list_local, test_set and test_kfunc_set are .local symbols placed + * in .BTF_ids by inline asm, and are read here directly by C name. To the + * compiler they are plain, default-visibility extern objects. + * + * When test_progs is linked as a position-independent executable (PIE), + * taking the address of such an extern is routed through the GOT. The + * GNU assembler on aarch64 unconditionally converts references to .local + * symbols into section + addend form (".BTF_ids + "), but a GOT + * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so + * the linker resolves it to the .BTF_ids base. + * + * Mark them hidden so the compiler treats them as non-interposable and + * emits a direct, addend-preserving PC-relative access instead of a GOT + * load, in both PIE and non-PIE builds. test_list_global is .globl and + * not affected, so it is left at default visibility. + */ +#pragma GCC visibility push(hidden) BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) @@ -43,24 +76,39 @@ BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -extern __u32 test_list_global[]; -BTF_ID_LIST_GLOBAL(test_list_global, 1) -BTF_ID_UNUSED +BTF_SET_START(test_set) BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) +BTF_SET_END(test_set) -BTF_SET_START(test_set) +BTF_KFUNCS_START(test_kfunc_set) +BTF_ID_FLAGS(func, kfunc_a) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_KFUNCS_END(test_kfunc_set) + +/* + * Same kfuncs in reverse declaration order, so resolve_btfids has to + * actually sort at least one of the two sets. + */ +BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_a) +BTF_KFUNCS_END(test_kfunc_set_rev) +#pragma GCC visibility pop + +extern __u32 test_list_global[]; +BTF_ID_LIST_GLOBAL(test_list_global, 1) +BTF_ID_UNUSED BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -BTF_SET_END(test_set) static int __resolve_symbol(struct btf *btf, int type_id) @@ -70,10 +118,10 @@ __resolve_symbol(struct btf *btf, int type_id) unsigned int i; type = btf__type_by_id(btf, type_id); - if (!type) { - PRINT_FAIL("Failed to get type for ID %d\n", type_id); + if (!ASSERT_OK_PTR(type, "btf__type_by_id")) return -1; - } + + str = btf__name_by_offset(btf, type->name_off); for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { if (test_symbols[i].id >= 0) @@ -82,64 +130,80 @@ __resolve_symbol(struct btf *btf, int type_id) if (BTF_INFO_KIND(type->info) != test_symbols[i].type) continue; - str = btf__name_by_offset(btf, type->name_off); - if (!str) { - PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id); - return -1; - } - if (!strcmp(str, test_symbols[i].name)) test_symbols[i].id = type_id; } + if (!btf_is_func(type)) + return 0; + + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].id >= 0) + continue; + if (!strcmp(str, kfunc_symbols[i].name)) + kfunc_symbols[i].id = type_id; + } + return 0; } -static int resolve_symbols(void) +static int resolve_symbols(struct btf *btf) { - struct btf *btf; + __u32 nr = btf__type_cnt(btf); int type_id; - __u32 nr; - - btf = btf__parse_raw("resolve_btfids.test.o.BTF"); - if (CHECK(libbpf_get_error(btf), "resolve", - "Failed to load BTF from resolve_btfids.test.o.BTF\n")) - return -1; - - nr = btf__type_cnt(btf); for (type_id = 1; type_id < nr; type_id++) { if (__resolve_symbol(btf, type_id)) - break; + return -1; } - - btf__free(btf); return 0; } +static void check_kfunc_set(struct btf_id_set8 *set) +{ + unsigned int i, j; + + ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags"); + ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt"); + + for (i = 0; i < set->cnt; i++) { + for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) { + if (kfunc_symbols[j].id == (s32)set->pairs[i].id) { + ASSERT_EQ(set->pairs[i].flags, + kfunc_symbols[j].flags, "kfunc_flags_check"); + break; + } + } + + ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found"); + + if (i > 0) { + ASSERT_LE(set->pairs[i - 1].id, + set->pairs[i].id, "kfunc_sort_check"); + } + } +} + void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; unsigned int i, j; - int ret = 0; + struct btf *btf; - if (resolve_symbols()) + btf = btf__parse_raw(BTF_DATA_FILE); + if (!ASSERT_OK_PTR(btf, "btf_parse")) return; + if (resolve_symbols(btf)) + goto out; + /* Check BTF_ID_LIST(test_list_local) and * BTF_ID_LIST_GLOBAL(test_list_global) IDs */ for (j = 0; j < ARRAY_SIZE(test_lists); j++) { test_list = test_lists[j]; - for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { - ret = CHECK(test_list[i] != test_symbols[i].id, - "id_check", - "wrong ID for %s (%d != %d)\n", - test_symbols[i].name, - test_list[i], test_symbols[i].id); - if (ret) - return; - } + for (i = 0; i < ARRAY_SIZE(test_symbols); i++) + ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name); } /* Check BTF_SET_START(test_set) IDs */ @@ -153,15 +217,16 @@ void test_resolve_btfids(void) break; } - ret = CHECK(!found, "id_check", - "ID %d not found in test_symbols\n", - test_set.ids[i]); - if (ret) + if (!ASSERT_TRUE(found, "id_in_test_symbols")) break; - if (i > 0) { - if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check")) - return; - } + if (i > 0) + ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"); } + + check_kfunc_set(&test_kfunc_set); + check_kfunc_set(&test_kfunc_set_rev); + +out: + btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/sock_xattr.c b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c new file mode 100644 index 000000000000..b5816e90f01a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Christian Brauner */ + +#include +#include +#include +#include +#include +#include +#include + +#include "sock_read_xattr.skel.h" + +static const char xattr_value[] = "bpf_sock_value"; +static const char xattr_name[] = "user.bpf_test"; + +static void test_read_sock_xattr(void) +{ + struct sockaddr_in addr = {}; + struct sock_read_xattr *skel = NULL; + struct bpf_link *link = NULL; + int sock_fd = -1, err; + + sock_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sock_fd, "socket")) + return; + + err = fsetxattr(sock_fd, xattr_name, xattr_value, sizeof(xattr_value), 0); + if (!ASSERT_OK(err, "fsetxattr")) + goto out; + + skel = sock_read_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "sock_read_xattr__open_and_load")) + goto out; + + skel->bss->monitored_pid = sys_gettid(); + + /* Only attach the functional program; the verifier-only programs + * above are not pid-gated and would clobber the shared globals. + */ + link = bpf_program__attach(skel->progs.read_sock_xattr); + if (!ASSERT_OK_PTR(link, "attach read_sock_xattr")) + goto out; + + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + /* Only the lsm/socket_connect hook matters; the connect may fail. */ + connect(sock_fd, (struct sockaddr *)&addr, sizeof(addr)); + + ASSERT_EQ(skel->data->read_ret, sizeof(xattr_value), "read_ret"); + ASSERT_STREQ(skel->bss->value, xattr_value, "value"); + +out: + bpf_link__destroy(link); + if (sock_fd >= 0) + close(sock_fd); + sock_read_xattr__destroy(skel); +} + +void test_sock_xattr(void) +{ + RUN_TESTS(sock_read_xattr); + + if (test__start_subtest("read_sock_xattr")) + test_read_sock_xattr(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c index 621b3b71888e..1d7231728eaf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c @@ -431,6 +431,35 @@ static void test_sockmap_strp_verdict(int family, int sotype) test_sockmap_strp__destroy(strp); } +static void test_sockmap_strp_parser_reject(void) +{ + struct test_sockmap_strp *strp = NULL; + int parser_mod, parser_ro, link; + int err, map; + + strp = test_sockmap_strp__open_and_load(); + if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load")) + return; + + map = bpf_map__fd(strp->maps.sock_map); + parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize); + parser_ro = bpf_program__fd(strp->progs.prog_skb_parser); + + err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0); + ASSERT_ERR(err, "bpf_prog_attach parser_mod"); + + link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL); + if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro")) + goto out; + + err = bpf_link_update(link, parser_mod, NULL); + ASSERT_ERR(err, "bpf_link_update parser_mod"); +out: + if (link >= 0) + close(link); + test_sockmap_strp__destroy(strp); +} + void test_sockmap_strp(void) { if (test__start_subtest("sockmap strp tcp pass")) @@ -451,4 +480,6 @@ void test_sockmap_strp(void) test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM); if (test__start_subtest("sockmap strp tcp dispatch")) test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM); + if (test__start_subtest("sockmap strp parser reject pkt mod")) + test_sockmap_strp_parser_reject(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c index 64fbda082309..af8968b89ad7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c @@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer) req.n.nlmsg_len += sizeof(struct ifinfomsg); addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer)); addattr_nest_end(&req.n, peer_info); + addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB, + NETKIT_SCRUB_NONE); addattr_nest_end(&req.n, data); addattr_nest_end(&req.n, linkinfo); @@ -405,6 +407,24 @@ static int netns_load_bpf(const struct bpf_program *src_prog, return -1; } +static struct bpf_link *netns_attach_nk(const char *ns, int ifindex, + struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_netkit_opts, optl); + struct nstoken *nstoken = NULL; + struct bpf_link *link = NULL; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "setns")) + goto cleanup; + + link = bpf_program__attach_netkit(prog, ifindex, &optl); +cleanup: + if (nstoken) + close_netns(nstoken); + return link; +} + static void test_tcp(int family, const char *addr, __u16 port) { int listen_fd = -1, accept_fd = -1, client_fd = -1; @@ -1082,6 +1102,53 @@ static void test_tc_redirect_peer(struct netns_setup_result *setup_result) close_netns(nstoken); } +static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result) +{ + struct test_tc_peer *skel; + struct nstoken *nstoken; + int err; + + nstoken = open_netns(NS_FWD); + if (!ASSERT_OK_PTR(nstoken, "setns fwd")) + return; + + skel = test_tc_peer__open(); + if (!ASSERT_OK_PTR(skel, "test_tc_peer__open")) + goto done; + + skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd; + skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd; + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing, + BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing, + BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type"); + + err = test_tc_peer__load(skel); + if (!ASSERT_OK(err, "test_tc_peer__load")) + goto done; + + skel->links.tc_src_ing = netns_attach_nk(NS_SRC, + setup_result->ifindex_src, + skel->progs.tc_src_ing); + if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src")) + goto done; + skel->links.tc_dst_ing = netns_attach_nk(NS_DST, + setup_result->ifindex_dst, + skel->progs.tc_dst_ing); + if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst")) + goto done; + + if (!ASSERT_OK(set_forwarding(false), "disable forwarding")) + goto done; + + test_connectivity(); + +done: + if (skel) + test_tc_peer__destroy(skel); + close_netns(nstoken); +} + static int tun_open(char *name) { struct ifreq ifr; @@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg) RUN_TEST(tc_redirect_peer, MODE_VETH); RUN_TEST(tc_redirect_peer, MODE_NETKIT); + RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT); RUN_TEST(tc_redirect_peer_l3, MODE_VETH); RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT); RUN_TEST(tc_redirect_neigh, MODE_VETH); diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c new file mode 100644 index 000000000000..d0ba2ca587b0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include "map_kptr.skel.h" + +void test_map_uninit_mem_exposure(void) +{ + size_t value_sz, slot_sz, lookup_sz, tail_sz; + int err, key, nr_cpus, cpu, map_fd; + __u8 *value = NULL, *zero = NULL; + struct bpf_program *prog; + struct map_kptr *skel; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = map_kptr__open(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open")) + return; + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_autoload(prog, false); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto out; + } + + err = map_kptr__load(skel); + if (!ASSERT_OK(err, "map_kptr__load")) + goto out; + + value_sz = bpf_map__value_size((skel)->maps.pcpu_array); + slot_sz = roundup(value_sz, 8); + tail_sz = slot_sz - value_sz; + if (!ASSERT_NEQ(tail_sz, 0, "tail_sz")) + goto out; + + lookup_sz = slot_sz * nr_cpus; + map_fd = bpf_map__fd(skel->maps.pcpu_array); + + value = malloc(lookup_sz); + zero = calloc(1, tail_sz); + if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero")) + goto out; + + key = 0; + memset(value, 0x2B, lookup_sz); + err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + memset(value, 0xFF, lookup_sz); + err = bpf_map_lookup_elem(map_fd, &key, value); + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + __u8 *tail = value + cpu * slot_sz + value_sz; + + if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes")) + goto out; + } + +out: + free(zero); + free(value); + map_kptr__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c index 72875071d4f1..477aedbb01ba 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -65,11 +64,6 @@ static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) eth_hdr->h_proto = htons(ETH_P_LOOPBACK); } -static bool is_umem_valid(struct xsk_socket_info *xsk) -{ - return !!xsk->umem->umem; -} - static u32 mode_to_xdp_flags(enum test_mode mode) { return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; @@ -1010,7 +1004,7 @@ static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) return TEST_FAILURE; if (!ret) { - if (!is_umem_valid(test->ifobj_tx->xsk)) + if (test->poll_tmout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); @@ -1149,7 +1143,7 @@ static int receive_pkts(struct test_spec *test) break; res = __receive_pkts(test, xsk); - if (!(res == TEST_PASS || res == TEST_CONTINUE)) + if (res != TEST_CONTINUE) return res; ret = gettimeofday(&tv_now, NULL); @@ -1166,11 +1160,12 @@ static int receive_pkts(struct test_spec *test) return TEST_PASS; } -static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) +static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, + bool test_timeout) { u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct xsk_umem_info *umem = ifobject->xsk_arr[0].umem_real; struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct xsk_umem_info *umem = xsk->umem; bool use_poll = ifobject->use_poll; struct pollfd fds = { }; int ret; @@ -1178,7 +1173,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); /* pkts_in_flight might be negative if many invalid packets are sent */ if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / - buffer_len)) { + buffer_len) && !test_timeout) { ret = kick_tx(xsk); if (ret) return TEST_FAILURE; @@ -1191,7 +1186,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); - if (timeout) { + if (test_timeout) { if (ret < 0) { ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, errno); @@ -1271,7 +1266,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); if (ret <= 0) { - if (ret == 0 && timeout) + if (ret == 0 && test_timeout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); @@ -1279,14 +1274,14 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b } } - if (!timeout) { + if (!test_timeout) { if (complete_pkts(xsk, i)) return TEST_FAILURE; usleep(10); - return TEST_PASS; } + /* Loop completion is driven by send_pkts() stream progress checks. */ return TEST_CONTINUE; } @@ -1322,7 +1317,6 @@ bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) static int send_pkts(struct test_spec *test, struct ifobject *ifobject) { - bool timeout = !is_umem_valid(test->ifobj_rx->xsk); DECLARE_BITMAP(bitmap, test->nb_sockets); u32 i, ret; @@ -1337,19 +1331,18 @@ static int send_pkts(struct test_spec *test, struct ifobject *ifobject) __set_bit(i, bitmap); continue; } - ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); - if (ret == TEST_CONTINUE && !test->fail) - continue; - - if ((ret || test->fail) && !timeout) - return TEST_FAILURE; - - if (ret == TEST_PASS && timeout) + ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], test->poll_tmout); + if (ret != TEST_CONTINUE) return ret; - ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); - if (ret) + if (test->fail) return TEST_FAILURE; + + if (!test->poll_tmout) { + ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); + if (ret) + return TEST_FAILURE; + } } } @@ -1521,7 +1514,7 @@ static int thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobjec umem_tx->base_addr = 0; umem_tx->next_buffer = 0; - ret = xsk_configure(test, ifobject, umem_tx, true); + ret = xsk_configure(test, ifobject, umem_rx, true); if (ret) return ret; ifobject->xsk = &ifobject->xsk_arr[0]; @@ -1677,7 +1670,8 @@ void *worker_testapp_validate_rx(void *arg) strerror(-err)); } - pthread_barrier_wait(&barr); + if (test->use_barrier) + pthread_barrier_wait(&barr); /* We leave only now in case of error to avoid getting stuck in the barrier */ if (err) { @@ -1716,11 +1710,6 @@ static void testapp_clean_xsk_umem(struct ifobject *ifobj) munmap(umem->buffer, umem->mmap_size); } -static void handler(int signum) -{ - pthread_exit(NULL); -} - static bool xdp_prog_changed_rx(struct test_spec *test) { struct ifobject *ifobj = test->ifobj_rx; @@ -1825,9 +1814,18 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i return TEST_FAILURE; } - if (ifobj2) { + err = xsk_attach_xdp_progs(test, ifobj1, ifobj2); + if (err) { + ksft_print_msg("Error: failed to attach XDP programs: %d (%s)\n", + err, strerror(-err)); + return TEST_FAILURE; + } + test->use_barrier = !!ifobj2; + + if (test->use_barrier) { if (pthread_barrier_init(&barr, NULL, 2)) return TEST_FAILURE; + pkt_stream_reset(ifobj2->xsk->pkt_stream); } @@ -1835,27 +1833,26 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i pkt_stream_reset(ifobj1->xsk->pkt_stream); pkts_in_flight = 0; - signal(SIGUSR1, handler); /*Spawn RX thread */ pthread_create(&t0, NULL, ifobj1->func_ptr, test); - if (ifobj2) { + if (test->use_barrier) { pthread_barrier_wait(&barr); if (pthread_barrier_destroy(&barr)) { - pthread_kill(t0, SIGUSR1); + test->use_barrier = false; + pthread_join(t0, NULL); clean_sockets(test, ifobj1); clean_umem(test, ifobj1, NULL); return TEST_FAILURE; } + } + if (ifobj2) { /*Spawn TX thread */ pthread_create(&t1, NULL, ifobj2->func_ptr, test); - pthread_join(t1, NULL); } - if (!ifobj2) - pthread_kill(t0, SIGUSR1); pthread_join(t0, NULL); if (test->total_steps == test->current_step || test->fail) { @@ -1893,8 +1890,6 @@ static int testapp_validate_traffic(struct test_spec *test) } } - if (xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx)) - return TEST_FAILURE; return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); } @@ -2231,16 +2226,33 @@ int testapp_xdp_shared_umem(struct test_spec *test) int testapp_poll_txq_tmout(struct test_spec *test) { + bool shared_umem = test->ifobj_tx->shared_umem; + int ret; + + test->poll_tmout = true; + /* + * POLL_TXQ_FULL exercises TX timeout setup in isolation. + * Keep TX out of shared-UMEM mode here so TX setup does not require + * RX UMEM to be initialized first. + */ + test->ifobj_tx->shared_umem = false; test->ifobj_tx->use_poll = true; /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ test->ifobj_tx->xsk->umem->frame_size = 2048; - if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) + if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) { + test->ifobj_tx->shared_umem = shared_umem; return TEST_FAILURE; - return testapp_validate_traffic_single_thread(test, test->ifobj_tx); + } + + ret = testapp_validate_traffic_single_thread(test, test->ifobj_tx); + test->ifobj_tx->shared_umem = shared_umem; + + return ret; } int testapp_poll_rxq_tmout(struct test_spec *test) { + test->poll_tmout = true; test->ifobj_rx->use_poll = true; return testapp_validate_traffic_single_thread(test, test->ifobj_rx); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.h b/tools/testing/selftests/bpf/prog_tests/test_xsk.h index 4313d0d87235..03753ddc5dcd 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.h +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.h @@ -207,6 +207,8 @@ struct test_spec { bool set_ring; bool adjust_tail; bool adjust_tail_support; + bool poll_tmout; + bool use_barrier; enum test_mode mode; char name[MAX_TEST_NAME_SIZE]; }; diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52..e193206fc5d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -4,6 +4,8 @@ #ifdef __x86_64__ +#define _GNU_SOURCE +#include #include #include #include @@ -13,11 +15,12 @@ #include #include #include +#include #include "uprobe_syscall.skel.h" #include "uprobe_syscall_executed.skel.h" #include "bpf/libbpf_internal.h" -#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 +#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 #include "usdt.h" #pragma GCC diagnostic ignored "-Wattributes" @@ -26,7 +29,7 @@ __attribute__((aligned(16))) __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) { asm volatile ( - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */ + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ "movq $0xdeadbeef, %rax\n" "ret\n" ); @@ -345,9 +348,9 @@ static void test_uretprobe_syscall_call(void) __attribute__((aligned(16))) __nocf_check __weak __naked void uprobe_test(void) { - asm volatile (" \n" - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n" - "ret \n" + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" ); } @@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void) USDT(optimized_uprobe, usdt); } +/* + * Assembly-level red zone clobbering test. Stores known values in the + * red zone (below RSP), executes a nop10 (uprobe site), and checks that + * the values survived. Returns 0 if intact, 1 if clobbered. + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + * + * Align the code at 64 bytes, to make sure nop10 is not on page boundary. + */ +__attribute__((aligned(64))) +__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) +{ + asm volatile ( + "movabs $0x1111111111111111, %%rax\n" + "movq %%rax, -8(%%rsp)\n" + "movabs $0x2222222222222222, %%rax\n" + "movq %%rax, -16(%%rsp)\n" + "movabs $0x3333333333333333, %%rax\n" + "movq %%rax, -24(%%rsp)\n" + + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ + + "movabs $0x1111111111111111, %%rax\n" + "cmpq %%rax, -8(%%rsp)\n" + "jne 1f\n" + "movabs $0x2222222222222222, %%rax\n" + "cmpq %%rax, -16(%%rsp)\n" + "jne 1f\n" + "movabs $0x3333333333333333, %%rax\n" + "cmpq %%rax, -24(%%rsp)\n" + "jne 1f\n" + + "xorl %%eax, %%eax\n" + "retq\n" + "1:\n" + "movl $1, %%eax\n" + "retq\n" + ::: "rax", "memory" + ); +} + static int find_uprobes_trampoline(void *tramp_addr) { void *start, *end; @@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr) return ret; } -static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; -static void *find_nop5(void *fn) +static void *find_nop10(void *fn) { int i; - for (i = 0; i < 10; i++) { - if (!memcmp(nop5, fn + i, 5)) + for (i = 0; i < 128; i++) { + if (!memcmp(nop10, fn + i, 10)) return fn + i; } return NULL; @@ -403,6 +451,16 @@ static void *find_nop5(void *fn) typedef void (__attribute__((nocf_check)) *trigger_t)(void); +static void check_attach_notrigger(struct uprobe_syscall_executed *skel, + void *addr, int executed) +{ + unsigned char *op = addr; + + /* Make sure bpf program was not executed. */ + ASSERT_EQ(skel->bss->executed, executed, "executed"); + ASSERT_EQ(*op, 0xcc, "int3"); +} + static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, void *addr, int executed) { @@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge ASSERT_EQ(skel->bss->executed, executed, "executed"); /* .. and check the trampoline is as expected. */ - call = (struct __arch_relative_insn *) addr; + ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); + call = (struct __arch_relative_insn *)(addr + 5); tramp = (void *) (call + 1) + call->raddr; ASSERT_EQ(call->op, 0xe8, "call"); ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); @@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge return tramp; } -static void check_detach(void *addr, void *tramp) +static bool check_detach(void *addr, void *tramp) { + static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; + bool ok = true; + /* [uprobes_trampoline] stays after detach */ - ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); - ASSERT_OK(memcmp(addr, nop5, 5), "nop5"); + ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); + ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); + return ok; } -static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link, - trigger_t trigger, void *addr, int executed) +static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, + trigger_t trigger, void *addr, int executed) { void *tramp; tramp = check_attach(skel, trigger, addr, executed); bpf_link__destroy(link); check_detach(addr, tramp); + return tramp; } static void test_uprobe_legacy(void) @@ -453,6 +517,7 @@ static void test_uprobe_legacy(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -470,7 +535,30 @@ static void test_uprobe_legacy(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe */ skel->bss->executed = 0; @@ -492,6 +580,7 @@ static void test_uprobe_multi(void) LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -512,7 +601,30 @@ static void test_uprobe_multi(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe.multi */ skel->bss->executed = 0; @@ -536,6 +648,7 @@ static void test_uprobe_session(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -555,7 +668,30 @@ static void test_uprobe_session(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 4); + tramp = check(skel, link, uprobe_test, uprobe_test, 4); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 4); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 8); cleanup: uprobe_syscall_executed__destroy(skel); @@ -565,11 +701,11 @@ static void test_uprobe_usdt(void) { struct uprobe_syscall_executed *skel; struct bpf_link *link; - void *addr; + void *addr, *tramp; errno = 0; - addr = find_nop5(usdt_test); - if (!ASSERT_OK_PTR(addr, "find_nop5")) + addr = find_nop10(usdt_test); + if (!ASSERT_OK_PTR(addr, "find_nop10")) return; skel = uprobe_syscall_executed__open_and_load(); @@ -584,7 +720,32 @@ static void test_uprobe_usdt(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) goto cleanup; - check(skel, link, usdt_test, addr, 2); + tramp = check(skel, link, usdt_test, addr, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check_attach_notrigger(skel, addr, 2); + + bpf_link__destroy(link); + if (!check_detach(addr, tramp)) + goto cleanup; + + usdt_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check(skel, link, usdt_test, addr, 4); cleanup: uprobe_syscall_executed__destroy(skel); @@ -757,12 +918,124 @@ static void test_uprobe_race(void) #define __NR_uprobe 336 #endif +static void test_uprobe_red_zone(void) +{ + struct uprobe_syscall_executed *skel; + struct bpf_link *link; + void *nop10_addr; + size_t offset; + int i; + + nop10_addr = find_nop10(uprobe_red_zone_test); + if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + offset = get_uprobe_offset(nop10_addr); + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "attach_uprobe")) + goto cleanup; + + for (i = 0; i < 10; i++) + ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); + + bpf_link__destroy(link); + +cleanup: + uprobe_syscall_executed__destroy(skel); +} + static void test_uprobe_error(void) { long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); +} + +__attribute__((aligned(16))) +__nocf_check __weak __naked void uprobe_fork_test(void) +{ + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" + ); +} + +static noreturn int child_func(void *arg) +{ + struct uprobe_syscall_executed *skel = arg; + + /* Make sure the child's probe is still there and optimized.. */ + if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) + _exit(1); + + skel->bss->pid = getpid(); + + /* .. and it executes properly. */ + uprobe_fork_test(); + + if (skel->bss->executed != 3) + _exit(2); + + _exit(0); +} + +static void test_uprobe_fork_optimized(bool clone_vm) +{ + struct uprobe_syscall_executed *skel = NULL; + unsigned long offset; + int pid, status, err; + char stack[65535]; + + offset = get_uprobe_offset(&uprobe_fork_test); + if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + -1, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) + goto cleanup; + + skel->bss->pid = getpid(); + + /* Trigger optimization of uprobe in uprobe_fork_test. */ + uprobe_fork_test(); + uprobe_fork_test(); + + /* Make sure it got optimied. */ + if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) + goto cleanup; + + if (clone_vm) { + pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); + if (!ASSERT_GT(pid, 0, "clone")) + goto cleanup; + } else { + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto cleanup; + if (pid == 0) + child_func(skel); + } + + /* Wait for the child and verify it exited properly with 0. */ + err = waitpid(pid, &status, 0); + if (ASSERT_EQ(err, pid, "waitpid")) { + ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); + ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); + } + +cleanup: + uprobe_syscall_executed__destroy(skel); } static void __test_uprobe_syscall(void) @@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void) test_uprobe_usdt(); if (test__start_subtest("uprobe_race")) test_uprobe_race(); + if (test__start_subtest("uprobe_red_zone")) + test_uprobe_red_zone(); + if (test__start_subtest("uprobe_optimized_fork")) + test_uprobe_fork_optimized(false); + if (test__start_subtest("uprobe_optimized_clone_vm")) + test_uprobe_fork_optimized(true); if (test__start_subtest("uprobe_error")) test_uprobe_error(); if (test__start_subtest("uprobe_regs_equal")) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 69759b27794d..8004c9568ffa 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,9 +250,10 @@ static void subtest_basic_usdt(bool optimized) #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; -static unsigned char nop1_nop5_combo[6] = { 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; static void *find_instr(void *fn, unsigned char *instr, size_t cnt) { @@ -271,17 +272,17 @@ static void subtest_optimized_attach(void) __u8 *addr_1, *addr_2; /* usdt_1 USDT probe has single nop instruction */ - addr_1 = find_instr(usdt_1, nop1_nop5_combo, 6); - if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop5_combo")) + addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11); + if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo")) return; addr_1 = find_instr(usdt_1, nop1, 1); if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1")) return; - /* usdt_2 USDT probe has nop,nop5 instructions combo */ - addr_2 = find_instr(usdt_2, nop1_nop5_combo, 6); - if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop5_combo")) + /* usdt_2 USDT probe has nop,nop10 instructions combo */ + addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11); + if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo")) return; skel = test_usdt__open_and_load(); @@ -309,12 +310,12 @@ static void subtest_optimized_attach(void) bpf_link__destroy(skel->links.usdt_executed); - /* we expect the nop5 ip */ + /* we expect the nop10 ip */ skel->bss->expected_ip = (unsigned long) addr_2 + 1; /* * Attach program on top of usdt_2 which is probe defined on top - * of nop1,nop5 combo, so the probe gets optimized on top of nop5. + * of nop1,nop10 combo, so the probe gets optimized on top of nop10. */ skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, 0 /*self*/, "/proc/self/exe", @@ -328,13 +329,64 @@ static void subtest_optimized_attach(void) /* nop stays on addr_2 address */ ASSERT_EQ(*addr_2, 0x90, "nop"); - /* call is on addr_2 + 1 address */ - ASSERT_EQ(*(addr_2 + 1), 0xe8, "call"); + /* + * lea -0x80(%rsp), %rsp + * call ... + */ + static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 }; + + ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call"); ASSERT_EQ(skel->bss->executed, 4, "executed"); cleanup: test_usdt__destroy(skel); } + +/* + * Test that USDT arguments survive nop10 optimization in a function where + * the compiler places operands in the red zone. + * + * Signal handlers are prone to having the compiler place USDT argument + * operands in the red zone (below rsp). + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + */ +static void subtest_optimized_red_zone(void) +{ + struct test_usdt *skel; + int i; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->expected_arg[0] = 0xDEADBEEF; + skel->bss->expected_arg[1] = 0xCAFEBABE; + skel->bss->expected_arg[2] = 0xFEEDFACE; + skel->bss->expected_pid = getpid(); + + skel->links.usdt_check_arg = bpf_program__attach_usdt( + skel->progs.usdt_check_arg, 0, "/proc/self/exe", + "optimized_attach", "usdt_red_zone", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone")) + goto cleanup; + + for (i = 0; i < 10; i++) + usdt_red_zone_trigger(); + + ASSERT_EQ(skel->bss->arg_total, 10, "arg_total"); + ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad"); + ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1"); + ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2"); + ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3"); + +cleanup: + test_usdt__destroy(skel); +} + #endif unsigned short test_usdt_100_semaphore SEC(".probes"); @@ -608,6 +660,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_red_zone")) + subtest_optimized_red_zone(); #endif if (test__start_subtest("multispec")) subtest_multispec_usdt(); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c index 26159e0499c7..448807676176 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include +#include #include "test_xdp_context_test_run.skel.h" #include "test_xdp_meta.skel.h" @@ -8,9 +10,12 @@ #define TX_NAME "veth1" #define TX_NETNS "xdp_context_tx" #define RX_NETNS "xdp_context_rx" +#define RX_MAC "02:00:00:00:00:01" +#define TX_MAC "02:00:00:00:00:02" #define TAP_NAME "tap0" #define DUMMY_NAME "dum0" #define TAP_NETNS "xdp_context_tuntap" +#define LWT_NETNS "xdp_context_lwt" #define TEST_PAYLOAD_LEN 32 static const __u8 test_payload[TEST_PAYLOAD_LEN] = { @@ -187,6 +192,42 @@ static int write_test_packet(int tap_fd) return 0; } +/* Inject Ethernet+IPv6+UDP frame into TAP */ +static int write_test_packet_udp(int tap_fd) +{ + __u8 pkt[sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + + sizeof(struct udphdr) + TEST_PAYLOAD_LEN] = {}; + struct ethhdr *eth = (void *)pkt; + struct ipv6hdr *ip6 = (void *)(eth + 1); + struct udphdr *udp = (void *)(ip6 + 1); + __u8 *payload = (void *)(udp + 1); + const __u8 tap_mac[ETH_ALEN] = { 0x02, 0, 0, 0, 0, 0x01 }; + int n; + + memcpy(eth->h_dest, tap_mac, ETH_ALEN); + eth->h_proto = htons(ETH_P_IPV6); + + ip6->version = 6; + ip6->hop_limit = 64; + ip6->nexthdr = IPPROTO_UDP; + ip6->payload_len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + inet_pton(AF_INET6, "fd00::2", &ip6->saddr); + inet_pton(AF_INET6, "fd00:1::1", &ip6->daddr); + + udp->source = htons(42); + udp->dest = htons(42); + udp->len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + /* UDP checksum is not validated on the forwarding path. */ + + memcpy(payload, test_payload, TEST_PAYLOAD_LEN); + + n = write(tap_fd, pkt, sizeof(pkt)); + if (!ASSERT_EQ(n, sizeof(pkt), "write frame")) + return -1; + + return 0; +} + static void dump_err_stream(const struct bpf_program *prog) { char buf[512]; @@ -518,3 +559,137 @@ void test_xdp_context_tuntap(void) test_xdp_meta__destroy(skel); } + +/* + * Test topology: + * + * tap0 fd00::1 + * RX: injected IPv6 UDP frame, XDP ingress sets metadata + * fwd: encap route prepends outer header(s) + * TX: TC egress validates metadata + * + * A routable IPv6 UDP frame is written into the tap fd, so it enters the RX + * path where XDP stores metadata. Routing then forwards it back out the same + * tap through an encapsulating route that prepends outer header(s). The TC + * egress program checks that the pushed header did not silently corrupt + * metadata. + */ +#define LWT_PIN_PATH "/sys/fs/bpf/xdp_context_lwt_xmit" + +enum lwt_encap_type { + LWT_ENCAP_BPF, + LWT_ENCAP_MPLS, + LWT_ENCAP_SEG6, + LWT_ENCAP_IOAM6, +}; + +static void test_lwt_encap(struct test_xdp_meta *skel, + enum lwt_encap_type type) +{ + LIBBPF_OPTS(bpf_tc_hook, tc_hook, .attach_point = BPF_TC_EGRESS); + LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1); + struct bpf_program *lwt_prog = NULL; + struct netns_obj *ns = NULL; + const char *encap; + bool pinned = false; + int tap_ifindex; + int tap_fd = -1; + int ret; + + skel->bss->test_pass = false; + + switch (type) { + case LWT_ENCAP_BPF: + encap = "encap bpf xmit pinned " LWT_PIN_PATH " via fd00::2"; + lwt_prog = skel->progs.dummy_lwt_xmit; + break; + case LWT_ENCAP_MPLS: + encap = "encap mpls 100 via inet6 fd00::2"; + break; + case LWT_ENCAP_SEG6: + encap = "encap seg6 mode encap segs fd00::2"; + break; + case LWT_ENCAP_IOAM6: + encap = "encap ioam6 mode encap tundst fd00::2 " + "trace prealloc type 0x800000 ns 0 size 4 via fd00::2"; + break; + default: + return; + } + + if (lwt_prog) { + unlink(LWT_PIN_PATH); + ret = bpf_program__pin(lwt_prog, LWT_PIN_PATH); + if (!ASSERT_OK(ret, "pin lwt prog")) + return; + pinned = true; + } + + ns = netns_new(LWT_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto close; + + tap_fd = open_tuntap(TAP_NAME, true); + if (!ASSERT_GE(tap_fd, 0, "open_tuntap")) + goto close; + + SYS(close, "ip link set dev " TAP_NAME " address " RX_MAC); + SYS(close, "sysctl -wq net.ipv6.conf.all.forwarding=1"); + SYS(close, "ip addr add fd00::1/64 dev " TAP_NAME " nodad"); + SYS(close, "ip link set dev " TAP_NAME " up"); + SYS(close, "ip neigh add fd00::2 lladdr " TX_MAC " nud permanent dev " TAP_NAME); + SYS(close, "ip -6 route add fd00:1::/64 %s dev %s", encap, TAP_NAME); + + tap_ifindex = if_nametoindex(TAP_NAME); + if (!ASSERT_GE(tap_ifindex, 0, "if_nametoindex")) + goto close; + + ret = bpf_xdp_attach(tap_ifindex, bpf_program__fd(skel->progs.ing_xdp), + 0, NULL); + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach")) + goto close; + + tc_hook.ifindex = tap_ifindex; + ret = bpf_tc_hook_create(&tc_hook); + if (!ASSERT_OK(ret, "bpf_tc_hook_create")) + goto close; + + tc_opts.prog_fd = bpf_program__fd(skel->progs.tc_is_meta_empty); + ret = bpf_tc_attach(&tc_hook, &tc_opts); + if (!ASSERT_OK(ret, "bpf_tc_attach")) + goto close; + + ret = write_test_packet_udp(tap_fd); + if (!ASSERT_OK(ret, "write_test_packet_udp")) + goto close; + + if (!ASSERT_TRUE(skel->bss->test_pass, "test_pass")) + dump_err_stream(skel->progs.tc_is_meta_empty); + +close: + if (tap_fd >= 0) + close(tap_fd); + netns_free(ns); + if (pinned) + unlink(LWT_PIN_PATH); +} + +void test_xdp_context_lwt_encap(void) +{ + struct test_xdp_meta *skel; + + skel = test_xdp_meta__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + return; + + if (test__start_subtest("bpf_encap")) + test_lwt_encap(skel, LWT_ENCAP_BPF); + if (test__start_subtest("mpls_encap")) + test_lwt_encap(skel, LWT_ENCAP_MPLS); + if (test__start_subtest("seg6_encap")) + test_lwt_encap(skel, LWT_ENCAP_SEG6); + if (test__start_subtest("ioam6_encap")) + test_lwt_encap(skel, LWT_ENCAP_IOAM6); + + test_xdp_meta__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 9eeb5b0b63d6..b0c441384f20 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -158,6 +158,7 @@ #define __arch_arm64 __arch("ARM64") #define __arch_riscv64 __arch("RISCV64") #define __arch_s390x __arch("s390x") +#define __arch_loongarch __arch("LOONGARCH") #define __caps_unpriv(caps) __test_tag("test_caps_unpriv=" EXPAND_QUOTE(caps)) #define __load_if_JITed() __test_tag("load_mode=jited") #define __load_if_no_JITed() __test_tag("load_mode=no_jited") diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c16..c4b438854565 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c index baa525275bde..8587658012c3 100644 --- a/tools/testing/selftests/bpf/progs/btf_data.c +++ b/tools/testing/selftests/bpf/progs/btf_data.c @@ -48,3 +48,13 @@ int func(struct root_struct *root) { return 0; } + +int kfunc_a(struct root_struct *root) +{ + return 0; +} + +int kfunc_b(struct root_struct *root) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c index 3fbefc568e0a..0d87c97dac99 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr.c +++ b/tools/testing/selftests/bpf/progs/map_kptr.c @@ -4,6 +4,18 @@ #include #include "../test_kmods/bpf_testmod_kfunc.h" +struct map_uninit_value { + struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; + __u32 data; +} __attribute__((packed)); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct map_uninit_value); + __uint(max_entries, 1); +} pcpu_array SEC(".maps"); + struct map_value { struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; struct prog_test_ref_kfunc __kptr *ref_ptr; diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a47..e3f8cb54dbe9 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 7247a20c0a3b..024ef2aae200 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -13,12 +13,20 @@ struct node_acquire { struct bpf_refcount refcount; }; +struct node_refcounted { + long key; + struct bpf_list_node list; + struct bpf_refcount refcount; +}; + extern void bpf_rcu_read_lock(void) __ksym; extern void bpf_rcu_read_unlock(void) __ksym; #define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_acquire, node); +private(B) struct bpf_spin_lock lock; +private(B) struct bpf_list_head head __contains(node_refcounted, list); static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -93,6 +101,32 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } +SEC("?tc") +__failure __msg("dereference of modified ptr_ ptr R1") +long refcount_acquire_list_node_offset(void *ctx) +{ + struct node_refcounted *node, *base, *ref; + struct bpf_list_node *list_node; + + node = bpf_obj_new(typeof(*node)); + if (!node) + return 1; + + bpf_spin_lock(&lock); + bpf_list_push_front(&head, &node->list); + list_node = bpf_list_pop_front(&head); + bpf_spin_unlock(&lock); + if (!list_node) + return 2; + + base = container_of(list_node, struct node_refcounted, list); + ref = bpf_refcount_acquire(list_node); + if (ref) + bpf_obj_drop(ref); + bpf_obj_drop(base); + return 0; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("function calls are not allowed while holding a lock") int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu, diff --git a/tools/testing/selftests/bpf/progs/sock_read_xattr.c b/tools/testing/selftests/bpf/progs/sock_read_xattr.c new file mode 100644 index 000000000000..c4a8eae8cc3c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/sock_read_xattr.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner */ + +#include +#include +#include +#include +#include "bpf_experimental.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +char value[16]; +int read_ret = -1; +__u32 monitored_pid = 0; + +static __always_inline void read_xattr(struct socket *sock) +{ + struct bpf_dynptr value_ptr; + + bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr); + bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr); +} + +SEC("lsm.s/socket_connect") +__success +int BPF_PROG(trusted_sock_ptr_sleepable, struct socket *sock) +{ + read_xattr(sock); + return 0; +} + +SEC("lsm/socket_connect") +__success +int BPF_PROG(trusted_sock_ptr_non_sleepable, struct socket *sock) +{ + read_xattr(sock); + return 0; +} + +SEC("lsm.s/socket_connect") +__success +int BPF_PROG(read_sock_xattr, struct socket *sock) +{ + struct bpf_dynptr value_ptr; + __u32 pid = bpf_get_current_pid_tgid() >> 32; + + if (pid != monitored_pid) + return 0; + + bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr); + read_ret = bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c index c9abfe3a11af..56e9aebf05f2 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_parse_prog.c @@ -5,28 +5,6 @@ SEC("sk_skb1") int bpf_prog1(struct __sk_buff *skb) { - void *data_end = (void *)(long) skb->data_end; - void *data = (void *)(long) skb->data; - __u8 *d = data; - int err; - - if (data + 10 > data_end) { - err = bpf_skb_pull_data(skb, 10); - if (err) - return SK_DROP; - - data_end = (void *)(long)skb->data_end; - data = (void *)(long)skb->data; - if (data + 10 > data_end) - return SK_DROP; - } - - /* This write/read is a bit pointless but tests the verifier and - * strparser handler for read/write pkt data and access into sk - * fields. - */ - d = data; - d[7] = 1; return skb->len; } diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c index 0660f29dca95..3177bc5b733a 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c @@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb) __sink(lport); __sink(rport); - if (data + 8 > data_end) - return SK_DROP; + if (data + 8 > data_end) { + if (bpf_skb_pull_data(skb, 8)) + return SK_DROP; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + if (data + 8 > data_end) + return SK_DROP; + + d = data; + } map = d[0]; sk = d[1]; diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index 076fbf03a126..df43649ecb78 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -10,6 +10,8 @@ #define EINVAL 22 #define ENOENT 2 +#define CT_OPTS_ERROR_GUARD 0x12345678 + #define NF_CT_ZONE_DIR_ORIG (1 << IP_CT_DIR_ORIGINAL) #define NF_CT_ZONE_DIR_REPL (1 << IP_CT_DIR_REPLY) @@ -19,6 +21,8 @@ int test_einval_reserved = 0; int test_einval_reserved_new = 0; int test_einval_netns_id = 0; int test_einval_len_opts = 0; +int test_einval_len_opts_small_lookup = 0; +int test_einval_len_opts_small_alloc = 0; int test_eproto_l4proto = 0; int test_enonet_netns_id = 0; int test_enoent_lookup = 0; @@ -124,6 +128,28 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, else test_einval_len_opts = opts_def.error; + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + bpf_ct_release(ct); + test_einval_len_opts_small_lookup = -EINVAL; + } else { + test_einval_len_opts_small_lookup = opts_def.error; + } + + opts_def.error = CT_OPTS_ERROR_GUARD; + ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def.netns_id)); + if (ct) { + ct = bpf_ct_insert_entry(ct); + if (ct) + bpf_ct_release(ct); + test_einval_len_opts_small_alloc = -EINVAL; + } else { + test_einval_len_opts_small_alloc = opts_def.error; + } + opts_def.l4proto = IPPROTO_ICMP; ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c index 137bd6292163..c85081538e93 100644 --- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c +++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c @@ -58,4 +58,10 @@ int BPF_PROG(umulti_run) return 0; } +SEC("fentry.multi") +int BPF_PROG(tmulti_run) +{ + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c index dde3d5bec515..fe88fa6d40bc 100644 --- a/tools/testing/selftests/bpf/progs/test_sockmap_strp.c +++ b/tools/testing/selftests/bpf/progs/test_sockmap_strp.c @@ -50,4 +50,11 @@ int prog_skb_parser_partial(struct __sk_buff *skb) return 10; } +SEC("sk_skb/stream_parser") +int prog_skb_parser_resize(struct __sk_buff *skb) +{ + bpf_skb_change_tail(skb, skb->len, 0); + return skb->len; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_peer.c b/tools/testing/selftests/bpf/progs/test_tc_peer.c index 365eacb5dc34..cfb9ef7f467c 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_peer.c +++ b/tools/testing/selftests/bpf/progs/test_tc_peer.c @@ -34,6 +34,28 @@ int tc_src(struct __sk_buff *skb) return bpf_redirect_peer(IFINDEX_DST, 0); } +SEC("tc") +int tc_dst_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_SRC, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_DST, 0); +} + +SEC("tc") +int tc_src_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_DST, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_SRC, 0); +} + SEC("tc") int tc_dst_l3(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c index f00cb52874e0..0ee78fb050a1 100644 --- a/tools/testing/selftests/bpf/progs/test_usdt.c +++ b/tools/testing/selftests/bpf/progs/test_usdt.c @@ -149,5 +149,30 @@ int usdt_executed(struct pt_regs *ctx) executed++; return 0; } + +int arg_total; +int arg_bad; +long arg_last[3]; +long expected_arg[3]; +int expected_pid; + +SEC("usdt") +int BPF_USDT(usdt_check_arg, long arg1, long arg2, long arg3) +{ + if (expected_pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; + + __sync_fetch_and_add(&arg_total, 1); + arg_last[0] = arg1; + arg_last[1] = arg2; + arg_last[2] = arg3; + + if (arg1 != expected_arg[0] || + arg2 != expected_arg[1] || + arg3 != expected_arg[2]) + __sync_fetch_and_add(&arg_bad, 1); + + return 0; +} #endif char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_xdp_meta.c b/tools/testing/selftests/bpf/progs/test_xdp_meta.c index fa73b17cb999..08b03be0b891 100644 --- a/tools/testing/selftests/bpf/progs/test_xdp_meta.c +++ b/tools/testing/selftests/bpf/progs/test_xdp_meta.c @@ -21,10 +21,6 @@ bool test_pass; -static const __u8 smac_want[ETH_ALEN] = { - 0x12, 0x34, 0xDE, 0xAD, 0xBE, 0xEF, -}; - static const __u8 meta_want[META_SIZE] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, @@ -32,11 +28,6 @@ static const __u8 meta_want[META_SIZE] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, }; -static bool check_smac(const struct ethhdr *eth) -{ - return !__builtin_memcmp(eth->h_source, smac_want, ETH_ALEN); -} - static bool check_metadata(const char *file, int line, __u8 *meta_have) { if (!__builtin_memcmp(meta_have, meta_want, META_SIZE)) @@ -280,18 +271,47 @@ int ing_cls_dynptr_offset_oob(struct __sk_buff *ctx) return TC_ACT_SHOT; } +/* Test packets carry test metadata pattern as payload. */ +static bool is_test_packet_xdp(struct xdp_md *ctx) +{ + __u8 meta_have[META_SIZE]; + __u32 len; + + len = bpf_xdp_get_buff_len(ctx); + if (len < META_SIZE) + return false; + if (bpf_xdp_load_bytes(ctx, len - META_SIZE, meta_have, META_SIZE)) + return false; + if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) + return false; + + return true; +} + +/* Test packets carry test metadata pattern as payload. */ +static bool is_test_packet_tc(struct __sk_buff *ctx) +{ + __u8 meta_have[META_SIZE]; + + if (ctx->len < META_SIZE) + return false; + if (bpf_skb_load_bytes(ctx, ctx->len - META_SIZE, meta_have, META_SIZE)) + return false; + if (__builtin_memcmp(meta_have, meta_want, META_SIZE)) + return false; + + return true; +} + /* Reserve and clear space for metadata but don't populate it */ SEC("xdp") int ing_xdp_zalloc_meta(struct xdp_md *ctx) { - struct ethhdr *eth = ctx_ptr(ctx, data); __u8 *meta; int ret; /* Drop any non-test packets */ - if (eth + 1 > ctx_ptr(ctx, data_end)) - return XDP_DROP; - if (!check_smac(eth)) + if (!is_test_packet_xdp(ctx)) return XDP_DROP; ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); @@ -310,33 +330,24 @@ int ing_xdp_zalloc_meta(struct xdp_md *ctx) SEC("xdp") int ing_xdp(struct xdp_md *ctx) { - __u8 *data, *data_meta, *data_end, *payload; - struct ethhdr *eth; + __u8 *data, *data_meta; int ret; + /* Drop any non-test packets */ + if (!is_test_packet_xdp(ctx)) + return XDP_DROP; + ret = bpf_xdp_adjust_meta(ctx, -META_SIZE); if (ret < 0) return XDP_DROP; data_meta = ctx_ptr(ctx, data_meta); - data_end = ctx_ptr(ctx, data_end); data = ctx_ptr(ctx, data); - eth = (struct ethhdr *)data; - payload = data + sizeof(struct ethhdr); - - if (payload + META_SIZE > data_end || - data_meta + META_SIZE > data) + if (data_meta + META_SIZE > data) return XDP_DROP; - /* The Linux networking stack may send other packets on the test - * interface that interfere with the test. Just drop them. - * The test packets can be recognized by their source MAC address. - */ - if (!check_smac(eth)) - return XDP_DROP; - - __builtin_memcpy(data_meta, payload, META_SIZE); + __builtin_memcpy(data_meta, meta_want, META_SIZE); return XDP_PASS; } @@ -353,7 +364,7 @@ int clone_data_meta_survives_data_write(struct __sk_buff *ctx) if (eth + 1 > ctx_ptr(ctx, data_end)) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; if (meta_have + META_SIZE > eth) @@ -383,7 +394,7 @@ int clone_data_meta_survives_meta_write(struct __sk_buff *ctx) if (eth + 1 > ctx_ptr(ctx, data_end)) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; if (meta_have + META_SIZE > eth) @@ -416,7 +427,7 @@ int clone_meta_dynptr_survives_data_slice_write(struct __sk_buff *ctx) if (!eth) goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; bpf_dynptr_from_skb_meta(ctx, 0, &meta); @@ -436,16 +447,11 @@ int clone_meta_dynptr_survives_data_slice_write(struct __sk_buff *ctx) SEC("tc") int clone_meta_dynptr_survives_meta_slice_write(struct __sk_buff *ctx) { - struct bpf_dynptr data, meta; - const struct ethhdr *eth; + struct bpf_dynptr meta; __u8 *meta_have; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; bpf_dynptr_from_skb_meta(ctx, 0, &meta); @@ -471,15 +477,10 @@ int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) { struct bpf_dynptr data, meta; __u8 meta_have[META_SIZE]; - const struct ethhdr *eth; int err; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; /* Expect read-write metadata before unclone */ @@ -492,6 +493,7 @@ int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) goto out; /* Helper write to payload will unclone the packet */ + bpf_dynptr_from_skb(ctx, 0, &data); bpf_dynptr_write(&data, offsetof(struct ethhdr, h_proto), "x", 1, 0); err = bpf_dynptr_read(meta_have, META_SIZE, &meta, 0, 0); @@ -511,17 +513,12 @@ int clone_meta_dynptr_rw_before_data_dynptr_write(struct __sk_buff *ctx) SEC("tc") int clone_meta_dynptr_rw_before_meta_dynptr_write(struct __sk_buff *ctx) { - struct bpf_dynptr data, meta; + struct bpf_dynptr meta; __u8 meta_have[META_SIZE]; - const struct ethhdr *eth; int err; - bpf_dynptr_from_skb(ctx, 0, &data); - eth = bpf_dynptr_slice(&data, 0, NULL, sizeof(*eth)); - if (!eth) - goto out; /* Ignore non-test packets */ - if (!check_smac(eth)) + if (!is_test_packet_tc(ctx)) goto out; /* Expect read-write metadata before unclone */ @@ -545,6 +542,28 @@ int clone_meta_dynptr_rw_before_meta_dynptr_write(struct __sk_buff *ctx) return TC_ACT_SHOT; } +SEC("lwt_xmit") +int dummy_lwt_xmit(struct __sk_buff *ctx) +{ + if (bpf_skb_change_head(ctx, sizeof(struct ipv6hdr), 0)) + return BPF_DROP; + + return BPF_OK; +} + +SEC("tc") +int tc_is_meta_empty(struct __sk_buff *ctx) +{ + if (!is_test_packet_tc(ctx)) + return TC_ACT_OK; + + if (ctx->data_meta != ctx->data) + return TC_ACT_OK; + + test_pass = true; + return TC_ACT_OK; +} + SEC("tc") int helper_skb_vlan_push_pop(struct __sk_buff *ctx) { diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 76d80605ec7f..02e562f56f9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -12,6 +12,8 @@ __arch_arm64 __jited(" mrs x8, SP_EL0") __arch_riscv64 __jited(" mv a5, tp") +__arch_loongarch +__jited(" move $a5, $tp") int inline_bpf_get_current_task(void) { bpf_get_current_task(); @@ -19,4 +21,15 @@ int inline_bpf_get_current_task(void) return 0; } +SEC("fentry/bpf_fentry_test2") +__success __retval(0) +__arch_loongarch +__jited(" ld.wu $a5, $tp, 16") +int inline_bpf_get_smp_processor_id(void) +{ + bpf_get_smp_processor_id(); + + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_lsm.c b/tools/testing/selftests/bpf/progs/verifier_lsm.c index 2f8103bfa14e..c724bf389f5c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_lsm.c +++ b/tools/testing/selftests/bpf/progs/verifier_lsm.c @@ -197,4 +197,19 @@ int BPF_PROG(sleepable_lsm_cgroup) return 0; } +SEC("lsm/file_mprotect") +__description("lsm retval load must reset stale register bounds") +__failure __msg("div by zero") +__naked int retval_load_resets_bounds(void *ctx) +{ + asm volatile ( + "r6 = 0;" + "r6 = *(u64 *)(r1 + 24);" + "if r6 == 0 goto +1;" + "r6 /= 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 6bc721accbae..72c691333703 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -634,6 +634,32 @@ __naked void partial_stack_load_preserves_partial_zeros(void) : __clobber_common); } +SEC("raw_tp") +__log_level(2) +__success +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R2=0") +__naked void stack_load_preserves_mixed_zero_and_zero_spill(void) +{ + asm volatile ( + /* fp-8 has scalar const-zero spill bytes and STACK_ZERO bytes. */ + ".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */ + "r0 = 0;" + "*(u32 *)(r10 -8) = r0;" + + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(single_byte_buf), + __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0)) + : __clobber_common); +} + char two_byte_buf[2] SEC(".data.two_byte_buf"); SEC("raw_tp") @@ -1359,4 +1385,22 @@ __naked void var_off_write_over_scalar_spill(void) : __clobber_all); } +SEC("socket") +__description("partial fill from cleaned pointer spill") +__failure +__log_level(2) +__msg("1: (05) goto pc+0") +__msg("2: (61) r0 = *(u32 *)(r10 -4)") +__msg("invalid size of register fill") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void partial_fill_from_cleaned_pointer_spill(void) +{ + /* Spill R1(ctx), then force a checkpoint and half-slot cleanup. */ + asm volatile ("*(u64 *)(r10 - 8) = r1;" + "goto +0;" + "r0 = *(u32 *)(r10 - 4);" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index c16f8382cf17..42de5cff7e52 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -6,6 +6,8 @@ #include "../../../include/linux/filter.h" #include "bpf_misc.h" +extern const int bpf_prog_active __ksym; + #define BPF_SK_LOOKUP(func) \ /* struct bpf_sock_tuple tuple = {} */ \ "r2 = 0;" \ @@ -77,6 +79,23 @@ __naked void dummy_prog_loop1_socket(void) : __clobber_all); } +SEC("socket") +__description("unpriv: pseudo btf id log masks address") +__success_unpriv +__msg_unpriv("0: (18) r1 = 0x0") +__not_msg_unpriv("0: (18) r1 = 0x{{[1-9a-f][0-9a-f]*}}") +__retval_unpriv(0) +__log_level(2) +__naked void pseudo_btf_id_log_masks_address(void) +{ + asm volatile ("r1 = %[bpf_prog_active] ll;" + "r0 = 0;" + "exit;" + : + : __imm_addr(bpf_prog_active) + : __clobber_all); +} + SEC("socket") __description("unpriv: return pointer") __success __failure_unpriv __msg_unpriv("R0 leaks addr") @@ -976,4 +995,26 @@ l0_%=: exit; \ : __clobber_all); } +SEC("socket") +__description("unpriv: Spectre v4 stack write slot index") +__success __success_unpriv +__retval(0) +#ifdef SPEC_V4 +__xlated_unpriv("r0 = 0") +__xlated_unpriv("*(u32 *)(r10 -4) = r0") +__xlated_unpriv("nospec") +__xlated_unpriv("*(u32 *)(r10 -8) = r0") +__xlated_unpriv("nospec") +__xlated_unpriv("exit") +#endif +__naked void stack_write_nospec_slot_index(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32 *)(r10 - 4) = r0; \ + *(u32 *)(r10 - 8) = r0; \ + exit; \ +" ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index f345466bca68..24cd0a763673 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -59,6 +59,116 @@ __naked void stack_read_priv_vs_unpriv(void) " ::: __clobber_all); } +SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero across slots") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8,-16") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero_across_slots(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + *(u64*)(r10 - 16) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + r2 -= 16; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves partial spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_partial_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u8*)(r10 - 9) = r0; \ + *(u8*)(r10 - 10) = r0; \ + *(u8*)(r10 - 11) = r0; \ + *(u8*)(r10 - 12) = r0; \ + *(u8*)(r10 - 13) = r0; \ + *(u8*)(r10 - 14) = r0; \ + *(u8*)(r10 - 15) = r0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + if r2 > 10 goto l0_%=; \ + r2 -= 15; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ +l0_%=: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read partial spill with misc data") +__failure +__msg("invalid variable-offset write to stack R1") +__naked void stack_read_var_off_partial_spill_with_misc_data(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = 0; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + SEC("cgroup/skb") __description("variable-offset stack read, uninitialized") __success diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index abdb9e6e3713..3ce32d134e2c 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -377,6 +377,7 @@ enum arch { ARCH_ARM64 = 0x4, ARCH_RISCV64 = 0x8, ARCH_S390X = 0x10, + ARCH_LOONGARCH = 0x20, }; static int get_current_arch(void) @@ -389,6 +390,8 @@ static int get_current_arch(void) return ARCH_RISCV64; #elif defined(__s390x__) return ARCH_S390X; +#elif defined(__loongarch__) + return ARCH_LOONGARCH; #endif return ARCH_UNKNOWN; } @@ -580,6 +583,8 @@ static int parse_test_spec(struct test_loader *tester, arch = ARCH_RISCV64; } else if (strcmp(val, "s390x") == 0) { arch = ARCH_S390X; + } else if (strcmp(val, "LOONGARCH") == 0) { + arch = ARCH_LOONGARCH; } else { PRINT_FAIL("bad arch spec: '%s'\n", val); err = -EINVAL; diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h index c71e21df38b3..75687f50f4e2 100644 --- a/tools/testing/selftests/bpf/usdt.h +++ b/tools/testing/selftests/bpf/usdt.h @@ -313,7 +313,7 @@ struct usdt_sema { volatile unsigned short active; }; #if defined(__ia64__) || defined(__s390__) || defined(__s390x__) #define USDT_NOP nop 0 #elif defined(__x86_64__) -#define USDT_NOP .byte 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x0 /* nop, nop5 */ +#define USDT_NOP .byte 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 /* nop, nop10 */ #else #define USDT_NOP nop #endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 789883aaca4c..5e38f8605b02 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -3,7 +3,7 @@ #if defined(__x86_64__) /* - * Include usdt.h with default nop,nop5 instructions combo. + * Include usdt.h with default nop,nop10 instructions combo. */ #include "usdt.h" @@ -13,4 +13,17 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; +static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; +static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; + +void __attribute__((noinline)) usdt_red_zone_trigger(void) +{ + unsigned long a1 = usdt_red_zone_arg1; + unsigned long a2 = usdt_red_zone_arg2; + unsigned long a3 = usdt_red_zone_arg3; + + USDT(optimized_attach, usdt_red_zone, a1, a2, a3); +} + #endif diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h index febc1723d090..8ebb2b4d4ec0 100644 --- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h +++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h @@ -8,6 +8,7 @@ #define MB(x) (x << 20) +#define NSEC_PER_USEC 1000L #define USEC_PER_SEC 1000000L #define NSEC_PER_SEC 1000000000L diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index 7a40d76b9548..a5eccfcabef5 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -639,6 +639,31 @@ test_cpucg_nested_weight_underprovisioned(const char *root) return run_cpucg_nested_weight_test(root, false); } +/* + * Best effort attempt to get the kernel's HZ value from the config. + * Return the HZ value if found otherwise return 1000 (the default) to + * indicate failure. + */ +static long +get_config_hz(void) +{ + long hz = 1000; + FILE *f; + char cmd[256] = "zcat /proc/config.gz 2>/dev/null | grep '^CONFIG_HZ='"; + + f = popen(cmd, "r"); + + if (!f) + return hz; + + if (fscanf(f, "CONFIG_HZ=%ld", &hz) == EOF) + goto out; + +out: + pclose(f); + return hz; +} + /* * This test creates a cgroup with some maximum value within a period, and * verifies that a process in the cgroup is not overscheduled. @@ -646,15 +671,18 @@ test_cpucg_nested_weight_underprovisioned(const char *root) static int test_cpucg_max(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *cpucg; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); cpucg = cg_name(root, "cpucg_test"); @@ -670,8 +698,8 @@ static int test_cpucg_max(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; @@ -710,15 +738,18 @@ static int test_cpucg_max(const char *root) static int test_cpucg_max_nested(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *parent, *child; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); parent = cg_name(root, "cpucg_parent"); @@ -741,8 +772,8 @@ static int test_cpucg_max_nested(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 0d41aa0d343d..ca9bc38fdb95 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -495,13 +495,20 @@ REMOTE_TEST_MATRIX=( # Narrowing cpuset.cpus to previously sibling-excluded CPUs should # not return CPUs that were never actually owned. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . . C3 . . p1:4|c11:1-2|c12:3 \ + . . . C3 . . p1:4|c11:1-2|c12:3 \ p1:P1|c11:P1|c12:P2 3" # Expanding cpuset.cpus to include a previously sibling-excluded CPU # after the sibling has become a member should correctly request it. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ + . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ + p1:P1|p2:P1|c11:P1" + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . O5=0 . p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \ + p1:P1|p2:P1|c11:P1" ) # diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 2180c328a825..ece244e5c5b9 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -13,6 +13,7 @@ TEST_PROGS += sysfs.py TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py TEST_PROGS += damos_tried_regions.py damon_nr_regions.py +TEST_PROGS += sysfs_refresh.py TEST_PROGS += reclaim.sh lru_sort.sh # regression tests (reproducers of previously found bugs) diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py index 8b12cc048440..e6a2265d721e 100644 --- a/tools/testing/selftests/damon/_damon_sysfs.py +++ b/tools/testing/selftests/damon/_damon_sysfs.py @@ -271,7 +271,7 @@ def __init__(self, type_='anon', matching=False, allow=False, self.type_ = type_ self.matching = matching self.allow = allow - self.memcg_path = memcg_path, + self.memcg_path = memcg_path self.addr_start = addr_start self.addr_end = addr_end self.target_idx = target_idx @@ -624,17 +624,23 @@ class DamonCtx: pause = None idx = None - def __init__(self, ops='paddr', monitoring_attrs=DamonAttrs(), targets=[], - schemes=[], pause=False): + def __init__(self, ops='paddr', monitoring_attrs=None, targets=None, + schemes=None, pause=False): self.ops = ops + if monitoring_attrs is None: + monitoring_attrs = DamonAttrs() self.monitoring_attrs = monitoring_attrs self.monitoring_attrs.context = self + if targets is None: + targets = [] self.targets = targets for idx, target in enumerate(self.targets): target.idx = idx target.context = self + if schemes is None: + schemes = [] self.schemes = schemes for idx, scheme in enumerate(self.schemes): scheme.idx = idx @@ -692,12 +698,14 @@ def stage(self): class Kdamond: state = None pid = None + refresh_ms = None contexts = None idx = None # index of this kdamond between siblings kdamonds = None # parent - def __init__(self, contexts=[]): + def __init__(self, contexts=[], refresh_ms=None): self.contexts = contexts + self.refresh_ms = refresh_ms for idx, context in enumerate(self.contexts): context.idx = idx context.kdamond = self @@ -720,6 +728,11 @@ def start(self): err = context.stage() if err is not None: return err + if self.refresh_ms is not None: + err = write_file(os.path.join(self.sysfs_dir(), 'refresh_ms'), + '%d' % self.refresh_ms) + if err is not None: + return err err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on') if err is not None: return err @@ -831,7 +844,7 @@ def commit_schemes_quota_goals(self): for goal in scheme.quota.goals: err = goal.stage() if err is not None: - print('commit_schemes_quota_goals failed stagign: %s'% + print('commit_schemes_quota_goals failed staging: %s'% err) exit(1) return write_file(os.path.join(self.sysfs_dir(), 'state'), diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py index f04d43702481..0f2f36584e48 100755 --- a/tools/testing/selftests/damon/damos_apply_interval.py +++ b/tools/testing/selftests/damon/damos_apply_interval.py @@ -56,7 +56,7 @@ def main(): # Because the second scheme was having the apply interval that is ten times # lower than that of the first scheme, the second scheme should be tried # about ten times more frequently than the first scheme. For possible - # timing errors, check if it was at least nine times more freuqnetly tried. + # timing errors, check if it was at least nine times more frequently tried. ratio = nr_tried_stats[1] / nr_tried_stats[0] if ratio < 9: print('%d / %d = %f (< 9)' % diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py index f76e0412b564..661e4ba4765a 100755 --- a/tools/testing/selftests/damon/damos_quota_goal.py +++ b/tools/testing/selftests/damon/damos_quota_goal.py @@ -66,7 +66,7 @@ def main(): # effective quota was already minimum that cannot be more reduced if expect_increase is False and last_effective_bytes == 1: continue - print('efective bytes not changed: %d' % goal.effective_bytes) + print('effective bytes not changed: %d' % goal.effective_bytes) exit(1) increased = last_effective_bytes < goal.effective_bytes diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py index 3b347eb28bd2..d6472e6a6e08 100755 --- a/tools/testing/selftests/damon/damos_tried_regions.py +++ b/tools/testing/selftests/damon/damos_tried_regions.py @@ -55,10 +55,10 @@ def main(): collected_nr_regions.sort() sample = collected_nr_regions[4] print('50-th percentile nr_regions: %d' % sample) - print('expectation (>= 14) is %s' % 'met' if sample >= 14 else 'not met') + print('expectation (>= 14) is %s' % ('met' if sample >= 14 else 'not met')) if collected_nr_regions[4] < 14: print('full nr_regions:') - print('\n'.join(collected_nr_regions)) + print('\n'.join(['%d' % x for x in collected_nr_regions])) exit(1) if __name__ == '__main__': diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py index 972948e6215f..09552e91bc78 100755 --- a/tools/testing/selftests/damon/drgn_dump_damon_status.py +++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py @@ -59,7 +59,6 @@ def region_to_dict(region): ['ar', addr_range_to_dict], ['sampling_addr', int], ['nr_accesses', int], - ['nr_accesses_bp', int], ['age', int], ]) @@ -163,7 +162,7 @@ def damos_filter_to_dict(damos_filter): int(damos_filter.addr_range.end)] elif type_ == 'target': dict_['target_idx'] = int(damos_filter.target_idx) - elif type_ == 'hugeapge_size': + elif type_ == 'hugepage_size': dict_['sz_range'] = [int(damos_filter.sz_range.min), int(damos_filter.sz_range.max)] return dict_ diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py index aa03a1187489..3ffa054b6386 100755 --- a/tools/testing/selftests/damon/sysfs.py +++ b/tools/testing/selftests/damon/sysfs.py @@ -119,7 +119,7 @@ def assert_access_pattern_committed(pattern, dump): 'max_nr_accesses', dump) assert_true(dump['min_age_region'] == pattern.age[0], 'min_age_region', dump) - assert_true(dump['max_age_region'] == pattern.age[1], 'miaxage_region', + assert_true(dump['max_age_region'] == pattern.age[1], 'max_age_region', dump) def assert_scheme_committed(scheme, dump): @@ -129,7 +129,7 @@ def assert_scheme_committed(scheme, dump): 'cold': 1, 'pageout': 2, 'hugepage': 3, - 'nohugeapge': 4, + 'nohugepage': 4, 'collapse': 5, 'lru_prio': 6, 'lru_deprio': 7, @@ -250,6 +250,35 @@ def assert_ctxs_committed(kdamonds): if ctx in ctxs_paused_for_dump: ctx.pause = False +def test_memcg_filter_memcg_path_staging(): + global kdamonds + memcg_filter = _damon_sysfs.DamosFilter( + type_='memcg', matching=True, allow=True, memcg_path='/') + kdamonds = _damon_sysfs.Kdamonds( + [_damon_sysfs.Kdamond( + contexts=[_damon_sysfs.DamonCtx( + targets=[_damon_sysfs.DamonTarget(pid=-1)], + schemes=[_damon_sysfs.Damos( + ops_filters=[memcg_filter])], + )])]) + kdamonds.start() + + shown, rd_err = _damon_sysfs.read_file( + os.path.join(memcg_filter.sysfs_dir(), 'memcg_path')) + if rd_err is not None: + print('memcg_path staging: sysfs read (%s)' % rd_err) + kdamonds.stop() + exit(1) + if shown.rstrip('\n') != memcg_filter.memcg_path: + print('memcg_path staging: memcg_path readback ' + '(shown=%s, expected=%s)' % + (shown.rstrip('\n'), memcg_filter.memcg_path)) + kdamonds.stop() + exit(1) + + kdamonds.stop() + kdamonds = None + def main(): global kdamonds kdamonds = _damon_sysfs.Kdamonds( @@ -356,5 +385,7 @@ def main(): assert_ctxs_committed(kdamonds) kdamonds.stop() + test_memcg_filter_memcg_path_staging() + if __name__ == '__main__': main() diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh index 78f4badb5beb..b43deee60fe9 100755 --- a/tools/testing/selftests/damon/sysfs.sh +++ b/tools/testing/selftests/damon/sysfs.sh @@ -3,7 +3,7 @@ source _common.sh -# Kselftest frmework requirement - SKIP code is 4. +# Kselftest framework requirement - SKIP code is 4. ksft_skip=4 ensure_write_succ() @@ -28,7 +28,7 @@ ensure_write_fail() if (echo "$content" > "$file") 2> /dev/null then - echo "writing $content to $file succeed ($fail_reason)" + echo "writing $content to $file succeeded ($reason)" echo "expected failure because $reason" exit 1 fi @@ -99,14 +99,55 @@ test_stats() done } +test_dest() +{ + dest_dir=$1 + ensure_file "$dest_dir/id" "exist" "600" + ensure_file "$dest_dir/weight" "exist" "600" +} + +test_dests() +{ + dests_dir=$1 + ensure_file "$dests_dir/nr_dests" "exist" "600" + ensure_write_succ "$dests_dir/nr_dests" "1" "valid input" + test_dest "$dests_dir/0" + + ensure_write_succ "$dests_dir/nr_dests" "2" "valid input" + test_dest "$dests_dir/0" + test_dest "$dests_dir/1" + + ensure_write_succ "$dests_dir/nr_dests" "0" "valid input" + ensure_dir "$dests_dir/0" "not_exist" + ensure_dir "$dests_dir/1" "not_exist" +} + test_filter() { filter_dir=$1 ensure_file "$filter_dir/type" "exist" "600" - ensure_write_succ "$filter_dir/type" "anon" "valid input" - ensure_write_succ "$filter_dir/type" "memcg" "valid input" - ensure_write_succ "$filter_dir/type" "addr" "valid input" - ensure_write_succ "$filter_dir/type" "target" "valid input" + + local dir_name=$(basename "$(dirname "$filter_dir")") + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "ops_filters" ] + then + ensure_write_succ "$filter_dir/type" "anon" "valid input" + ensure_write_succ "$filter_dir/type" "memcg" "valid input" + fi + if [ "$dir_name" = "filters" ] || [ "$dir_name" = "core_filters" ] + then + ensure_write_succ "$filter_dir/type" "addr" "valid input" + ensure_write_succ "$filter_dir/type" "target" "valid input" + fi + if [ "$dir_name" = "core_filters" ] + then + ensure_write_fail "$filter_dir/type" "anon" "ops type" + ensure_write_fail "$filter_dir/type" "memcg" "ops type" + fi + if [ "$dir_name" = "ops_filters" ] + then + ensure_write_fail "$filter_dir/type" "addr" "core type" + ensure_write_fail "$filter_dir/type" "target" "core type" + fi ensure_write_fail "$filter_dir/type" "foo" "invalid input" ensure_file "$filter_dir/matching" "exist" "600" ensure_file "$filter_dir/memcg_path" "exist" "600" @@ -158,6 +199,20 @@ test_goal() ensure_dir "$goal_dir" "exist" ensure_file "$goal_dir/target_value" "exist" "600" ensure_file "$goal_dir/current_value" "exist" "600" + ensure_file "$goal_dir/target_metric" "exist" "600" + local fpath="$goal_dir/target_metric" + ensure_write_succ "$fpath" "user_input" "valid input" + ensure_write_succ "$fpath" "some_mem_psi_us" "valid input" + ensure_write_succ "$fpath" "node_mem_used_bp" "valid input" + ensure_write_succ "$fpath" "node_mem_free_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_used_bp" "valid input" + ensure_write_succ "$fpath" "node_memcg_free_bp" "valid input" + ensure_write_succ "$fpath" "active_mem_bp" "valid input" + ensure_write_succ "$fpath" "inactive_mem_bp" "valid input" + ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input" + ensure_write_fail "$fpath" "foo" "invalid input" + ensure_file "$goal_dir/nid" "exist" "600" + ensure_file "$goal_dir/path" "exist" "600" } test_goals() @@ -207,7 +262,10 @@ test_scheme() ensure_file "$scheme_dir/apply_interval_us" "exist" "600" test_quotas "$scheme_dir/quotas" test_watermarks "$scheme_dir/watermarks" + test_dests "$scheme_dir/dests" test_filters "$scheme_dir/filters" + test_filters "$scheme_dir/core_filters" + test_filters "$scheme_dir/ops_filters" test_stats "$scheme_dir/stats" test_tried_regions "$scheme_dir/tried_regions" } @@ -346,8 +404,13 @@ test_probes() ensure_write_succ "$probes_dir/nr_probes" "1" "valid input" test_probe "$probes_dir/0" + ensure_write_succ "$probes_dir/nr_probes" "2" "valid input" + test_probe "$probes_dir/0" + test_probe "$probes_dir/1" + ensure_write_succ "$probes_dir/nr_probes" "0" "valid input" ensure_dir "$probes_dir/0" "not_exist" + ensure_dir "$probes_dir/1" "not_exist" } test_monitoring_attrs() @@ -363,7 +426,7 @@ test_context() { context_dir=$1 ensure_dir "$context_dir" "exist" - ensure_file "$context_dir/avail_operations" "exit" 400 + ensure_file "$context_dir/avail_operations" "exist" 400 ensure_file "$context_dir/operations" "exist" 600 ensure_file "$context_dir/addr_unit" "exist" 600 ensure_file "$context_dir/pause" "exist" 600 diff --git a/tools/testing/selftests/damon/sysfs_refresh.py b/tools/testing/selftests/damon/sysfs_refresh.py new file mode 100755 index 000000000000..012b7e8f509f --- /dev/null +++ b/tools/testing/selftests/damon/sysfs_refresh.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import os +import subprocess +import time + +import _damon_sysfs + +def main(): + # Continuously access a memory region for far longer than the test needs, + # so the kdamond always has a live target to monitor while we poll. + sz_region = 10 * 1024 * 1024 + proc = subprocess.Popen( + ['./access_memory', '1', '%d' % sz_region, '60000', 'repeat']) + + # A 'stat' scheme with the default (maximally wide) access pattern matches + # every monitored region, so its 'nr_tried' stat increases as the kdamond + # runs. refresh_ms should make DAMON update the schemes' stats files under + # sysfs on its own, without a manual 'update_schemes_stats' request. + kdamond = _damon_sysfs.Kdamond( + refresh_ms=100, + contexts=[_damon_sysfs.DamonCtx( + ops='vaddr', + targets=[_damon_sysfs.DamonTarget(pid=proc.pid)], + schemes=[_damon_sysfs.Damos(action='stat')], + )]) + kdamonds = _damon_sysfs.Kdamonds([kdamond]) + + err = kdamonds.start() + if err is not None: + # Kernels older than the refresh_ms feature have no such file; treat + # that as unsupported rather than a failure. + if not os.path.exists(os.path.join(kdamond.sysfs_dir(), 'refresh_ms')): + proc.terminate() + proc.wait() + print('kdamond has no refresh_ms file; skipping') + exit(_damon_sysfs.ksft_skip) + proc.terminate() + proc.wait() + print('kdamond start failed: %s' % err) + exit(1) + + scheme = kdamond.contexts[0].schemes[0] + nr_tried_path = os.path.join(scheme.sysfs_dir(), 'stats', 'nr_tried') + + try: + # Poll the stat file directly. We never request an update (e.g. + # 'update_schemes_stats'), so 'nr_tried' can become non-zero only + # through the periodic refresh that refresh_ms enables. + nr_tried = 0 + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if proc.poll() is not None: + print('the access_memory target exited unexpectedly') + exit(1) + content, err = _damon_sysfs.read_file(nr_tried_path) + if err is not None: + print('reading %s failed: %s' % (nr_tried_path, err)) + exit(1) + nr_tried = int(content) + if nr_tried > 0: + break + time.sleep(0.1) + finally: + kdamonds.stop() + proc.terminate() + proc.wait() + + if nr_tried == 0: + print('refresh_ms did not auto-update the schemes stats') + exit(1) + +if __name__ == '__main__': + main() diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py index 35c724a63f6c..16fdc6e7fc56 100755 --- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py +++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py @@ -7,7 +7,7 @@ import _damon_sysfs def pass_wss_estimation(sz_region): - # access two regions of given size, 2 seocnds per each region + # access two regions of given size, 2 seconds per each region proc = subprocess.Popen( ['./access_memory', '2', '%d' % sz_region, '2000', 'repeat']) kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond( diff --git a/tools/testing/selftests/drivers/net/bonding/Makefile b/tools/testing/selftests/drivers/net/bonding/Makefile index be130bf585a4..6364ca02642d 100644 --- a/tools/testing/selftests/drivers/net/bonding/Makefile +++ b/tools/testing/selftests/drivers/net/bonding/Makefile @@ -13,6 +13,7 @@ TEST_PROGS := \ bond_options.sh \ bond_passive_lacp.sh \ bond_stacked_header_parse.sh \ + bond_vlan_real_dev.sh \ dev_addr_lists.sh \ mode-1-recovery-updelay.sh \ mode-2-recovery-updelay.sh \ diff --git a/tools/testing/selftests/drivers/net/bonding/bond_vlan_real_dev.sh b/tools/testing/selftests/drivers/net/bonding/bond_vlan_real_dev.sh new file mode 100755 index 000000000000..542d9ffc4819 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/bond_vlan_real_dev.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test propagation of a real device's state to the VLANs stacked on top of it +# when the real device is (or becomes) a bond member. +# +# The kernel mirrors a real device's UP/DOWN, MTU and feature changes onto its +# VLANs. This is done asynchronously (netdev_work): doing it synchronously from +# the real device's notifier could deadlock. If the real device is brought up +# while enslaved to a bond - so its instance lock is held across NETDEV_UP - and +# a VLAN on top of it is itself a bond member, the synchronous propagation +# re-entered the stack and tried to take the same instance lock again. +# +# Cover both halves: +# - the deferred UP/DOWN, MTU and feature propagation actually lands on the +# VLAN (link state and MTU use an ops-locked dummy, i.e. the deferral path), +# - the deadlock-prone topology - a VLAN on a dummy, with the VLAN and the +# dummy each enslaved to a different bond - can be built without hanging. + +ALL_TESTS=" + vlan_link_state + vlan_mtu + vlan_features + vlan_real_dev_enslave +" + +REQUIRE_MZ=no +NUM_NETIFS=0 +lib_dir=$(dirname "$0") +source "$lib_dir"/../../../net/forwarding/lib.sh + +# Return 0 if $dev in netns $ns has flag $flag set (e.g. UP) in its <...> flags. +link_has_flag() +{ + local ns=$1 dev=$2 flag=$3 + + ip -n "$ns" link show dev "$dev" 2>/dev/null | grep -q "[<,]${flag}[,>]" +} + +link_lacks_flag() +{ + ! link_has_flag "$@" +} + +link_mtu_is() +{ + local ns=$1 dev=$2 want=$3 cur + + cur=$(ip -n "$ns" link show dev "$dev" 2>/dev/null | \ + sed -n 's/.* mtu \([0-9]\+\).*/\1/p') + [ "$cur" = "$want" ] +} + +vlan_feature_is() +{ + local ns=$1 dev=$2 feature=$3 value=$4 + + ip netns exec "$ns" ethtool -k "$dev" 2>/dev/null | \ + grep -q "^$feature: $value" +} + +link_has_master() +{ + local ns=$1 dev=$2 master=$3 + + ip -n "$ns" -o link show dev "$dev" 2>/dev/null | grep -q "master $master" +} + +vlan_link_state() +{ + RET=0 + + ip -n "$NS" link add ls_dummy type dummy + ip -n "$NS" link add link ls_dummy name ls_vlan type vlan id 100 + + # Bringing the real device up must propagate UP to the VLAN. + ip -n "$NS" link set ls_dummy up + busywait "$BUSYWAIT_TIMEOUT" link_has_flag "$NS" ls_vlan UP + check_err $? "VLAN did not go UP after the real device went UP" + + # ... and likewise for DOWN. + ip -n "$NS" link set ls_dummy down + busywait "$BUSYWAIT_TIMEOUT" link_lacks_flag "$NS" ls_vlan UP + check_err $? "VLAN did not go DOWN after the real device went DOWN" + + ip -n "$NS" link del ls_vlan + ip -n "$NS" link del ls_dummy + + log_test "VLAN link state follows the real device" +} + +vlan_mtu() +{ + RET=0 + + # The VLAN inherits the real device's MTU (2000) at creation time. + ip -n "$NS" link add mtu_dummy mtu 2000 type dummy + ip -n "$NS" link add link mtu_dummy name mtu_vlan type vlan id 100 + + # Shrinking the real device's MTU must clamp the VLAN's MTU. + ip -n "$NS" link set mtu_dummy mtu 1500 + busywait "$BUSYWAIT_TIMEOUT" link_mtu_is "$NS" mtu_vlan 1500 + check_err $? "VLAN MTU not clamped after the real device's MTU shrank" + + ip -n "$NS" link del mtu_vlan + ip -n "$NS" link del mtu_dummy + + log_test "VLAN MTU clamped to the real device" +} + +vlan_features() +{ + RET=0 + + # Use veth as the real device: unlike dummy it exports vlan_features, so + # the VLAN actually inherits a toggleable offload to assert on. + ip -n "$NS" link add ft_veth type veth peer name ft_veth_pr + ip -n "$NS" link add link ft_veth name ft_vlan type vlan id 100 + + vlan_feature_is "$NS" ft_vlan scatter-gather on + check_err $? "VLAN did not inherit scatter-gather from the real device" + + # Toggling the offload on the real device must propagate to the VLAN. + ip netns exec "$NS" ethtool -K ft_veth sg off + busywait "$BUSYWAIT_TIMEOUT" \ + vlan_feature_is "$NS" ft_vlan scatter-gather off + check_err $? "VLAN scatter-gather still on after disabling it on real dev" + + ip netns exec "$NS" ethtool -K ft_veth sg on + busywait "$BUSYWAIT_TIMEOUT" \ + vlan_feature_is "$NS" ft_vlan scatter-gather on + check_err $? "VLAN scatter-gather still off after enabling it on real dev" + + ip -n "$NS" link del ft_vlan + ip -n "$NS" link del ft_veth + + log_test "VLAN features follow the real device" +} + +vlan_real_dev_enslave() +{ + RET=0 + + # dummy <- VLAN -> bond0, then enslave the dummy itself to bond1. The + # last step brings the dummy up under bond1's instance lock, which used + # to deadlock while synchronously propagating UP to the (bond-enslaved) + # VLAN on top. + ip -n "$NS" link add dl_dummy type dummy + ip -n "$NS" link set dl_dummy up + ip -n "$NS" link add link dl_dummy name dl_vlan type vlan id 100 + + ip -n "$NS" link add dl_bond0 type bond mode active-backup + ip -n "$NS" link set dl_vlan down + ip -n "$NS" link set dl_vlan master dl_bond0 + check_err $? "could not enslave the VLAN to bond0" + + ip -n "$NS" link add dl_bond1 type bond mode active-backup + ip -n "$NS" link set dl_dummy down + ip -n "$NS" link set dl_dummy master dl_bond1 + check_err $? "could not enslave the real device to bond1" + + # If we got here the kernel did not deadlock; make sure it is still + # responsive and the enslave really took effect. + link_has_master "$NS" dl_dummy dl_bond1 + check_err $? "real device not enslaved to bond1" + + ip -n "$NS" link del dl_bond1 + ip -n "$NS" link del dl_bond0 + ip -n "$NS" link del dl_vlan + ip -n "$NS" link del dl_dummy + + log_test "VLAN real device enslaved to a second bond" +} + +setup_ns NS +trap 'cleanup_ns $NS' EXIT + +tests_run + +exit "$EXIT_STATUS" diff --git a/tools/testing/selftests/drivers/net/hw/toeplitz.py b/tools/testing/selftests/drivers/net/hw/toeplitz.py index cd7e080e6f84..571732198b93 100755 --- a/tools/testing/selftests/drivers/net/hw/toeplitz.py +++ b/tools/testing/selftests/drivers/net/hw/toeplitz.py @@ -21,6 +21,8 @@ ETH_RSS_HASH_TOP = 1 # Must match RPS_MAX_CPUS in toeplitz.c RPS_MAX_CPUS = 16 +# Cap Rx queues so IRQ pinning leaves free CPUs in the RPS_MAX_CPUS range +QUEUE_CAP = 8 def _check_rps_and_rfs_not_configured(cfg): @@ -48,6 +50,25 @@ def _get_cpu_for_irq(irq): return int(data) +def _cap_queue_count(cfg): + ehdr = {"header": {"dev-index": cfg.ifindex}} + chans = cfg.ethnl.channels_get(ehdr) + + config = {} + restore = {} + for key in ("combined-count", "rx-count"): + cur = chans.get(key, 0) + if cur > QUEUE_CAP: + config[key] = QUEUE_CAP + restore[key] = cur + + if not config: + return + + cfg.ethnl.channels_set(ehdr | config) + defer(cfg.ethnl.channels_set, ehdr | restore) + + def _get_irq_cpus(cfg): """ Read the list of IRQs for the device Rx queues. @@ -177,6 +198,7 @@ def test(cfg, proto_flag, ipver, grp): ] if grp: + _cap_queue_count(cfg) _check_rps_and_rfs_not_configured(cfg) if grp == "rss": irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)]) diff --git a/tools/testing/selftests/drivers/net/hw/tso.py b/tools/testing/selftests/drivers/net/hw/tso.py index 1b789fea8929..802bb4868046 100755 --- a/tools/testing/selftests/drivers/net/hw/tso.py +++ b/tools/testing/selftests/drivers/net/hw/tso.py @@ -187,28 +187,24 @@ def query_nic_features(cfg) -> None: cfg.wanted_features.add(f["name"]) cfg.hw_features = set() - hw_all_features_cmd = "" for f in features["hw"]["bits"]["bit"]: if f.get("value", False): - feature = f["name"] - cfg.hw_features.add(feature) - hw_all_features_cmd += f" {feature} on" - try: - ethtool(f"-K {cfg.ifname} {hw_all_features_cmd}") - except Exception as e: - ksft_pr(f"WARNING: failure enabling all hw features: {e}") - ksft_pr("partial gso feature detection may be impacted") + cfg.hw_features.add(f["name"]) # Check which features are supported via GSO partial cfg.partial_features = set() if 'tx-gso-partial' in cfg.hw_features: + seg_features = {f for f in cfg.hw_features if "segmentation" in f} + ethtool(f"-K {cfg.ifname} " + + " ".join(f"{f} on" for f in seg_features)) + ethtool(f"-K {cfg.ifname} tx-gso-partial off") no_partial = set() features = cfg.ethnl.features_get({"header": {"dev-index": cfg.ifindex}}) for f in features["active"]["bits"]["bit"]: no_partial.add(f["name"]) - cfg.partial_features = cfg.hw_features - no_partial + cfg.partial_features = seg_features - no_partial ethtool(f"-K {cfg.ifname} tx-gso-partial on") restore_wanted_features(cfg) diff --git a/tools/testing/selftests/drivers/net/so_txtime.c b/tools/testing/selftests/drivers/net/so_txtime.c index 75f3beef13d9..55a386f3d1b9 100644 --- a/tools/testing/selftests/drivers/net/so_txtime.c +++ b/tools/testing/selftests/drivers/net/so_txtime.c @@ -37,7 +37,7 @@ static int cfg_clockid = CLOCK_TAI; static uint16_t cfg_port = 8000; -static int cfg_variance_us = 4000; +static int cfg_variance_us = 8000; static bool cfg_machine_slow; static uint64_t cfg_start_time_ns; static int cfg_mark; diff --git a/tools/testing/selftests/filelock/.gitignore b/tools/testing/selftests/filelock/.gitignore new file mode 100644 index 000000000000..825e899a121b --- /dev/null +++ b/tools/testing/selftests/filelock/.gitignore @@ -0,0 +1 @@ +ofdlocks diff --git a/tools/testing/selftests/filelock/ofdlocks.c b/tools/testing/selftests/filelock/ofdlocks.c index ff8d47fc373a..68bac28b234b 100644 --- a/tools/testing/selftests/filelock/ofdlocks.c +++ b/tools/testing/selftests/filelock/ofdlocks.c @@ -16,7 +16,7 @@ static int lock_set(int fd, struct flock *fl) fl->l_whence = SEEK_SET; ret = fcntl(fd, F_OFD_SETLK, fl); if (ret) - perror("fcntl()"); + ksft_perror("fcntl()"); return ret; } @@ -28,7 +28,7 @@ static int lock_get(int fd, struct flock *fl) fl->l_whence = SEEK_SET; ret = fcntl(fd, F_OFD_GETLK, fl); if (ret) - perror("fcntl()"); + ksft_perror("fcntl()"); return ret; } @@ -39,94 +39,82 @@ int main(void) int fd = open("/tmp/aa", O_RDWR | O_CREAT | O_EXCL, 0600); int fd2 = open("/tmp/aa", O_RDONLY); + ksft_print_header(); + ksft_set_plan(4); + unlink("/tmp/aa"); assert(fd != -1); assert(fd2 != -1); - ksft_print_msg("[INFO] opened fds %i %i\n", fd, fd2); + ksft_print_msg("opened fds %i %i\n", fd, fd2); /* Set some read lock */ fl.l_type = F_RDLCK; fl.l_start = 5; fl.l_len = 3; rc = lock_set(fd, &fl); - if (rc == 0) { - ksft_print_msg - ("[SUCCESS] set OFD read lock on first fd\n"); - } else { - ksft_print_msg("[FAIL] to set OFD read lock on first fd\n"); - return -1; - } + ksft_test_result(rc == 0, "set OFD read lock on first fd\n"); + if (rc != 0) + ksft_finished(); + /* Make sure read locks do not conflict on different fds. */ fl.l_type = F_RDLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd2, &fl); if (rc != 0) - return -1; - if (fl.l_type != F_UNLCK) { - ksft_print_msg("[FAIL] read locks conflicted\n"); - return -1; - } + ksft_finished(); + if (fl.l_type != F_UNLCK) + ksft_exit_fail_msg("read locks conflicted\n"); + /* Make sure read/write locks do conflict on different fds. */ fl.l_type = F_WRLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd2, &fl); if (rc != 0) - return -1; - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[SUCCESS] read and write locks conflicted\n"); - } else { - ksft_print_msg - ("[SUCCESS] read and write locks not conflicted\n"); - return -1; - } + ksft_finished(); + ksft_test_result(fl.l_type != F_UNLCK, + "read and write locks conflicted\n"); + if (fl.l_type == F_UNLCK) + ksft_finished(); + /* Get info about the lock on first fd. */ fl.l_type = F_UNLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd, &fl); - if (rc != 0) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK not supported\n"); - return -1; - } - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[SUCCESS] F_UNLCK test returns: locked, type %i pid %i len %zi\n", - fl.l_type, fl.l_pid, fl.l_len); - } else { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK did not return lock info\n"); - return -1; - } + if (rc != 0) + ksft_exit_fail_msg("F_OFD_GETLK with F_UNLCK not supported\n"); + ksft_test_result(fl.l_type != F_UNLCK, + "F_OFD_GETLK with F_UNLCK returned lock info\n"); + if (fl.l_type == F_UNLCK) + ksft_exit_fail(); + ksft_print_msg("F_UNLCK test returns: locked, type %i pid %i len %zi\n", + fl.l_type, fl.l_pid, fl.l_len); + /* Try the same but by locking everything by len==0. */ fl2.l_type = F_UNLCK; fl2.l_start = 0; fl2.l_len = 0; rc = lock_get(fd, &fl2); - if (rc != 0) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK not supported\n"); - return -1; - } + if (rc != 0) + ksft_exit_fail_msg + ("F_OFD_GETLK with F_UNLCK not supported\n"); + ksft_test_result(memcmp(&fl, &fl2, sizeof(fl)) == 0, + "F_UNLCK with len==0 returned the same\n"); if (memcmp(&fl, &fl2, sizeof(fl))) { - ksft_print_msg - ("[FAIL] F_UNLCK test returns: locked, type %i pid %i len %zi\n", + ksft_exit_fail_msg + ("F_UNLCK test returns: locked, type %i pid %i len %zi\n", fl.l_type, fl.l_pid, fl.l_len); - return -1; } - ksft_print_msg("[SUCCESS] F_UNLCK with len==0 returned the same\n"); + /* Get info about the lock on second fd - no locks on it. */ fl.l_type = F_UNLCK; fl.l_start = 0; fl.l_len = 0; lock_get(fd2, &fl); - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK return lock info from another fd\n"); - return -1; - } - return 0; + ksft_test_result(fl.l_type == F_UNLCK, + "F_OFD_GETLK with F_UNLCK return lock info from another fd\n"); + + ksft_finished(); } diff --git a/tools/testing/selftests/filesystems/.gitignore b/tools/testing/selftests/filesystems/.gitignore index 64ac0dfa46b7..9eb185fb2f9d 100644 --- a/tools/testing/selftests/filesystems/.gitignore +++ b/tools/testing/selftests/filesystems/.gitignore @@ -5,3 +5,5 @@ fclog file_stressor anon_inode_test kernfs_test +idmapped_tmpfile +ustat_test diff --git a/tools/testing/selftests/filesystems/Makefile b/tools/testing/selftests/filesystems/Makefile index 85427d7f19b9..03be337c1f35 100644 --- a/tools/testing/selftests/filesystems/Makefile +++ b/tools/testing/selftests/filesystems/Makefile @@ -1,7 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 CFLAGS += $(KHDR_INCLUDES) -TEST_GEN_PROGS := devpts_pts file_stressor anon_inode_test kernfs_test fclog +TEST_GEN_PROGS := devpts_pts file_stressor anon_inode_test kernfs_test fclog ustat_test +TEST_GEN_PROGS += idmapped_tmpfile TEST_GEN_PROGS_EXTENDED := dnotify_test include ../lib.mk + +$(OUTPUT)/idmapped_tmpfile: LDLIBS += -lcap +$(OUTPUT)/idmapped_tmpfile: utils.c diff --git a/tools/testing/selftests/filesystems/ceph/Makefile b/tools/testing/selftests/filesystems/ceph/Makefile new file mode 100644 index 000000000000..4ad3e8d40d90 --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +TEST_PROGS := run_validation.sh +TEST_FILES := reset_stress.sh reset_corner_cases.sh \ + validate_consistency.py README settings + +include ../../lib.mk diff --git a/tools/testing/selftests/filesystems/ceph/README b/tools/testing/selftests/filesystems/ceph/README new file mode 100644 index 000000000000..eb0092b38f80 --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/README @@ -0,0 +1,84 @@ +# CephFS Client Reset Test Suite + +Test suite for the CephFS kernel client manual session reset feature. +This trimmed set contains the single-client stress test, the targeted +corner-case test, and the one-shot validation harness used during +feature bring-up. + +## Prerequisites + +- Linux kernel with the CephFS client reset feature (this branch) +- A running Ceph cluster with at least one MDS +- Root access (debugfs requires it) +- Python 3 (for validators) +- flock utility (for lock tests, usually in util-linux) + +## Test inventory + +| Test | Script(s) | What it covers | +|------|-----------|----------------| +| Single-client stress | `reset_stress.sh` | I/O + resets + data integrity on one mount | +| Corner cases | `reset_corner_cases.sh` | EBUSY, dirty caps, flock reclaim, unmount-during-reset | +| Validation harness | `run_validation.sh` | baseline + corner cases + moderate/aggressive stress + final status check | + +## Quick start + +Stress run: + + sudo ./reset_stress.sh --mount-point /mnt/cephfs --profile moderate + +Corner cases: + + sudo ./reset_corner_cases.sh --mount-point /mnt/cephfs + +End-to-end validation: + + sudo ./run_validation.sh --mount-point /mnt/cephfs + +## Stress profiles + + baseline - no resets, 1 IO + 1 rename, 600s + moderate - reset every 5-15s, 2 IO + 1 rename, 900s + aggressive - reset every 1-5s, 4 IO + 2 rename, 900s + soak - reset every 5-15s, 2 IO + 1 rename, 3600s + +## Key options (all scripts) + + --mount-point PATH CephFS mount point (required) + --client-id ID Debugfs client id (auto-detected if one) + +reset_stress.sh additionally accepts: + + --profile NAME baseline|moderate|aggressive|soak + --duration-sec N Override profile runtime + --no-reset Disable reset injection + --out-dir PATH Artifact directory + +## Corner case tests + + [1/4] ebusy_rejection Second reset rejected while first in-flight + [2/4] dirty_caps_at_reset Reset with unflushed dirty caps + [3/4] flock_after_reset Stale lock EIO + fresh lock after holder exit + [4/4] unmount_during_reset umount during active reset (destroy-path wakeup) + +Test 4 requires creating a second CephFS mount instance and SKIPs if +the host cannot do so. See `--help` output for details. + +## Troubleshooting + +**No writable Ceph reset interface found:** +Kernel lacks the reset feature, debugfs not mounted, or not root. +Check: `ls /sys/kernel/debug/ceph/*/reset/` + +**Multiple Ceph clients found:** +Use `--client-id` to select one. +List: `ls /sys/kernel/debug/ceph/` + +## Files + +| File | Role | +|------|------| +| `reset_stress.sh` | Single-client stress test runner | +| `validate_consistency.py` | Single-client post-run validator | +| `reset_corner_cases.sh` | Corner case harness (4 sequential tests) | +| `run_validation.sh` | One-shot validation harness | diff --git a/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh b/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh new file mode 100755 index 000000000000..a6dae84a616d --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh @@ -0,0 +1,646 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# CephFS client reset corner case tests. +# Runs a checklist of targeted tests that exercise specific reset +# code paths not covered by the stress tests. +# +# Requires: mounted CephFS, debugfs access (root), flock(1) utility. + +set -uo pipefail + +KSFT_SKIP=4 + +# kselftest auto-detect: when invoked with no arguments (e.g. by +# "make run_tests"), find a CephFS mount automatically or skip. +if [[ $# -eq 0 ]]; then + MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)" + if [[ -z "$MOUNT_POINT" ]]; then + echo "SKIP: No CephFS mount found and --mount-point not specified" + exit "$KSFT_SKIP" + fi + exec "$0" --mount-point "$MOUNT_POINT" +fi + +MOUNT_POINT="" +DEBUGFS_ROOT="/sys/kernel/debug/ceph" +DEBUGFS_CLIENT="" +TRIGGER_PATH="" +STATUS_PATH="" +TEMP_MNT="" + +PASS_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 +TOTAL=4 + +log() +{ + printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$1" +} + +result() +{ + local num="$1" + local name="$2" + local status="$3" + local detail="${4:-}" + + case "$status" in + PASS) PASS_COUNT=$((PASS_COUNT + 1)) ;; + FAIL) FAIL_COUNT=$((FAIL_COUNT + 1)) ;; + SKIP) SKIP_COUNT=$((SKIP_COUNT + 1)) ;; + esac + + if [[ -n "$detail" ]]; then + printf '[%d/%d] %-30s %s (%s)\n' "$num" "$TOTAL" "$name" "$status" "$detail" + else + printf '[%d/%d] %-30s %s\n' "$num" "$TOTAL" "$name" "$status" + fi +} + +read_status_field() +{ + local field="$1" + awk -F': ' -v key="$field" '$1 == key {print $2}' "$STATUS_PATH" 2>/dev/null +} + +wait_reset_done() +{ + local timeout="${1:-30}" + local elapsed=0 + + while [[ "$(read_status_field "phase")" != "idle" ]]; do + sleep 1 + elapsed=$((elapsed + 1)) + if [[ "$elapsed" -ge "$timeout" ]]; then + return 1 + fi + done + return 0 +} + +list_reset_clients() +{ + local entry + + for entry in "$DEBUGFS_ROOT"/*/; do + entry="$(basename "$entry")" + [[ -d "$DEBUGFS_ROOT/$entry/reset" ]] || continue + [[ -w "$DEBUGFS_ROOT/$entry/reset/trigger" ]] || continue + printf '%s\n' "$entry" + done +} + +wait_status_nonidle() +{ + local status_path="$1" + local timeout="${2:-10}" + local polls=$((timeout * 10)) + local phase + + while [[ "$polls" -gt 0 ]]; do + phase="$(awk -F': ' '$1 == "phase" {print $2}' "$status_path" 2>/dev/null)" + if [[ -n "$phase" && "$phase" != "idle" ]]; then + return 0 + fi + sleep 0.1 + polls=$((polls - 1)) + done + + return 1 +} + +discover_debugfs() +{ + local candidates=() + local entry + + if [[ -n "$DEBUGFS_CLIENT" ]]; then + if [[ ! -d "$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset" ]]; then + echo "SKIP: reset debugfs not found for $DEBUGFS_CLIENT" >&2 + exit "$KSFT_SKIP" + fi + return 0 + fi + + for entry in "$DEBUGFS_ROOT"/*/; do + entry="$(basename "$entry")" + [[ -d "$DEBUGFS_ROOT/$entry/reset" ]] || continue + [[ -w "$DEBUGFS_ROOT/$entry/reset/trigger" ]] || continue + candidates+=("$entry") + done + + if [[ ${#candidates[@]} -eq 0 ]]; then + echo "SKIP: No writable Ceph reset interface found under $DEBUGFS_ROOT" >&2 + exit "$KSFT_SKIP" + fi + + if [[ ${#candidates[@]} -gt 1 ]]; then + echo "SKIP: Multiple Ceph clients found: ${candidates[*]}. Use --client-id." >&2 + exit "$KSFT_SKIP" + fi + + DEBUGFS_CLIENT="${candidates[0]}" +} + +# --- Test 1: ebusy_rejection ------------------------------------------------ +# +# Trigger a reset while another is guaranteed in-flight. Creates +# dirty state so the first reset enters DRAINING (which takes +# measurable time), then polls until phase != idle and issues the +# second trigger. The second trigger must fail (the kernel returns +# -EBUSY), and only one reset must be counted in the accounting. + +test_ebusy_rejection() +{ + local num=1 + local name="ebusy_rejection" + local testfile="$MOUNT_POINT/.reset_corner_ebusy_$$" + local tc_before tc_after sc_before sc_after second_rc phase elapsed + + tc_before="$(read_status_field "trigger_count")" + sc_before="$(read_status_field "success_count")" + + # Create dirty state so the first reset enters DRAINING + echo "ebusy_dirty_data" > "$testfile" + sync "$testfile" + + python3 -c " +import os, sys +fd = os.open('$testfile', os.O_WRONLY | os.O_APPEND) +os.write(fd, b'dirty_for_ebusy_test\n') +sys.stdout.write('written') +" 2>/dev/null || { + result "$num" "$name" FAIL "dirty write failed" + rm -f "$testfile" + return + } + + # Trigger the first reset -- it will drain dirty state + echo "ebusy_first" > "$TRIGGER_PATH" 2>/dev/null || { + result "$num" "$name" FAIL "first trigger failed" + rm -f "$testfile" + return + } + + # Poll until phase is non-idle (quiescing or draining) + elapsed=0 + while true; do + phase="$(read_status_field "phase")" + if [[ "$phase" != "idle" ]]; then + break + fi + sleep 0.1 + elapsed=$((elapsed + 1)) + if [[ "$elapsed" -ge 50 ]]; then + result "$num" "$name" SKIP \ + "first reset completed before overlap could be tested" + rm -f "$testfile" 2>/dev/null + return + fi + done + + # Issue the second trigger -- should be rejected with EBUSY + second_rc=0 + echo "ebusy_second" > "$TRIGGER_PATH" 2>/dev/null && second_rc=0 || second_rc=$? + + if ! wait_reset_done 30; then + result "$num" "$name" FAIL "first reset never completed" + rm -f "$testfile" + return + fi + + tc_after="$(read_status_field "trigger_count")" + sc_after="$(read_status_field "success_count")" + + if [[ "$((tc_after - tc_before))" -ne 1 ]]; then + result "$num" "$name" FAIL "trigger_count +$((tc_after - tc_before)), expected +1" + rm -f "$testfile" + return + fi + + if [[ "$((sc_after - sc_before))" -ne 1 ]]; then + result "$num" "$name" FAIL "success_count +$((sc_after - sc_before)), expected +1" + rm -f "$testfile" + return + fi + + if [[ "$second_rc" -eq 0 ]]; then + result "$num" "$name" FAIL "second trigger did not return error" + rm -f "$testfile" + return + fi + + rm -f "$testfile" 2>/dev/null + result "$num" "$name" PASS +} + +# --- Test 2: dirty_caps_at_reset -------------------------------------------- +# +# Write to a file without fsync (dirty caps), trigger reset, then +# verify the file is not corrupt. Manual reset drains dirty caps +# before teardown (best-effort, 5s timeout). For a non-stuck cap +# the dirty write should be flushed during drain and persist. +# If the drain window is too short, only the synced first line +# persists -- that is acceptable (data loss is documented for +# unflushed writes). + +test_dirty_caps_at_reset() +{ + local num=2 + local name="dirty_caps_at_reset" + local testfile="$MOUNT_POINT/.reset_corner_dirty_caps_$$" + local content_after line_count sc_before sc_after le + + sc_before="$(read_status_field "success_count")" + + echo "line_1_before_dirty_write" > "$testfile" + sync "$testfile" + + python3 -c " +import os, sys +fd = os.open('$testfile', os.O_WRONLY | os.O_APPEND) +os.write(fd, b'line_2_dirty_no_fsync\n') +# deliberately no fsync -- leave caps dirty +sys.stdout.write('written') +" 2>/dev/null || { + result "$num" "$name" FAIL "dirty write failed" + rm -f "$testfile" + return + } + + echo "dirty_caps_test" > "$TRIGGER_PATH" 2>/dev/null || { + result "$num" "$name" FAIL "reset trigger failed" + rm -f "$testfile" + return + } + + if ! wait_reset_done 30; then + result "$num" "$name" FAIL "reset did not complete" + rm -f "$testfile" + return + fi + + sc_after="$(read_status_field "success_count")" + if [[ "$sc_after" -le "$sc_before" ]]; then + result "$num" "$name" FAIL "success_count did not increment (reset not exercised)" + rm -f "$testfile" + return + fi + + sync "$testfile" 2>/dev/null || true + content_after="$(cat "$testfile" 2>/dev/null)" || { + result "$num" "$name" FAIL "cannot read file after reset" + rm -f "$testfile" + return + } + + if [[ -z "$content_after" ]]; then + result "$num" "$name" FAIL "file is empty after reset" + rm -f "$testfile" + return + fi + + line_count="$(echo "$content_after" | wc -l)" + if [[ "$line_count" -lt 1 ]]; then + result "$num" "$name" FAIL "file has $line_count lines, expected >= 1" + rm -f "$testfile" + return + fi + + echo "$content_after" | head -1 | grep -q "line_1_before_dirty_write" || { + result "$num" "$name" FAIL "first line corrupted" + rm -f "$testfile" + return + } + + le="$(read_status_field "last_errno")" + if [[ "$le" != "0" ]]; then + result "$num" "$name" FAIL "last_errno=$le, expected 0" + rm -f "$testfile" + return + fi + + rm -f "$testfile" + result "$num" "$name" PASS "file intact ($line_count lines)" +} + +# --- Test 3: flock_after_reset ---------------------------------------------- +# +# Take an exclusive flock, trigger reset, verify stale lock state is +# marked with CEPH_I_ERROR_FILELOCK (same-client flock attempt returns +# EIO). After the original holder exits (releasing the local lock +# reference and clearing the error flag), a fresh lock can be acquired. +# +# The lock holder uses the fd-based flock form with exec, so killing +# $lock_pid closes the lock fd immediately (no orphaned child with an +# inherited fd copy that would prevent the VFS flock release). + +test_flock_after_reset() +{ + local num=3 + local name="flock_after_reset" + local testfile="$MOUNT_POINT/.reset_corner_flock_$$" + local lock_pid probe_rc sc_before sc_after + + sc_before="$(read_status_field "success_count")" + + echo "flock_test_content" > "$testfile" + sync "$testfile" + + # Hold lock via fd in a subshell; exec ensures killing $lock_pid + # closes the lock fd directly (no fork/child fd inheritance). + ( + exec 9<"$testfile" + flock --exclusive --nonblock 9 || exit 1 + exec sleep 300 + ) & + lock_pid=$! + sleep 0.5 + + if ! kill -0 "$lock_pid" 2>/dev/null; then + result "$num" "$name" FAIL "flock holder died immediately" + rm -f "$testfile" + return + fi + + echo "flock_after_reset_test" > "$TRIGGER_PATH" 2>/dev/null || { + kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null + result "$num" "$name" FAIL "reset trigger failed" + rm -f "$testfile" + return + } + + if ! wait_reset_done 30; then + kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null + result "$num" "$name" FAIL "reset did not complete" + rm -f "$testfile" + return + fi + + sc_after="$(read_status_field "success_count")" + if [[ "$sc_after" -le "$sc_before" ]]; then + kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null + result "$num" "$name" FAIL "success_count did not increment" + rm -f "$testfile" + return + fi + + # After teardown, CEPH_I_ERROR_FILELOCK is set on the inode. + # A same-client lock attempt should fail (EIO), NOT succeed. + probe_rc=0 + flock --exclusive --nonblock "$testfile" true 2>/dev/null && probe_rc=0 || probe_rc=$? + if [[ "$probe_rc" -eq 0 ]]; then + kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null + result "$num" "$name" FAIL \ + "same-client probe succeeded, expected EIO from stale lock state" + rm -f "$testfile" + return + fi + + # Kill the holder -- the exec'd sleep IS $lock_pid, so killing it + # closes fd 9 directly. VFS flock release fires ceph_fl_release_lock(), + # which decrements i_filelock_ref to 0 and clears CEPH_I_ERROR_FILELOCK. + kill "$lock_pid" 2>/dev/null + wait "$lock_pid" 2>/dev/null + + # After the holder exits, a fresh lock should be acquirable. + # The reset teardown sends SESSION_REQUEST_CLOSE so the MDS + # releases locks promptly, but retry briefly in case the + # message races with the connection close. + local attempt + probe_rc=1 + for attempt in 1 2 3 4 5; do + probe_rc=0 + flock --exclusive --nonblock "$testfile" true 2>/dev/null \ + && probe_rc=0 || probe_rc=$? + [[ "$probe_rc" -eq 0 ]] && break + sleep 1 + done + if [[ "$probe_rc" -ne 0 ]]; then + result "$num" "$name" FAIL \ + "cannot acquire fresh lock after holder exit (rc=$probe_rc, ${attempt} attempts)" + rm -f "$testfile" + return + fi + + # Verify file content survived + grep -q "flock_test_content" "$testfile" 2>/dev/null || { + result "$num" "$name" FAIL "file content corrupted after reset" + rm -f "$testfile" + return + } + + rm -f "$testfile" + result "$num" "$name" PASS "stale lock detected, fresh lock acquired after holder exit" +} + +# --- Test 4: unmount_during_reset ------------------------------------------- +# +# Mount a fresh CephFS, trigger reset, immediately unmount. The +# ceph_mdsc_destroy() path must wake blocked waiters with -ESHUTDOWN +# and not hang. + +test_unmount_during_reset() +{ + local num=4 + local name="unmount_during_reset" + local temp_mnt="/tmp/ceph_corner_mnt_$$" + local mount_opts="" + local mount_src="" + local temp_trigger="" + local temp_status="" + local temp_client="" + local temp_file="$temp_mnt/.reset_corner_umount_$$" + local phase="" + local trigger_ok=0 + local attempt + local -a new_clients=() + declare -A existing_clients=() + + mount_src="$(awk -v mp="$MOUNT_POINT" '$2 == mp && $3 == "ceph" {print $1; exit}' /proc/mounts 2>/dev/null)" + mount_opts="$(awk -v mp="$MOUNT_POINT" '$2 == mp && $3 == "ceph" {print $4; exit}' /proc/mounts 2>/dev/null)" + + if [[ -z "$mount_src" ]]; then + result "$num" "$name" SKIP "cannot determine mount source from /proc/mounts" + return + fi + + while IFS= read -r existing; do + [[ -n "$existing" ]] || continue + existing_clients["$existing"]=1 + done < <(list_reset_clients) + + mkdir -p "$temp_mnt" + + if ! mount -t ceph "$mount_src" "$temp_mnt" -o "$mount_opts" 2>/dev/null; then + result "$num" "$name" SKIP "cannot mount additional CephFS instance" + rmdir "$temp_mnt" 2>/dev/null + return + fi + + ls "$temp_mnt" > /dev/null 2>&1 + sync + sleep 1 + + for attempt in $(seq 1 50); do + new_clients=() + while IFS= read -r entry; do + [[ -n "$entry" ]] || continue + if [[ -n "${existing_clients[$entry]+x}" ]]; then + continue + fi + new_clients+=("$entry") + done < <(list_reset_clients) + + if [[ "${#new_clients[@]}" -eq 1 ]]; then + temp_client="${new_clients[0]}" + break + fi + + if [[ "${#new_clients[@]}" -gt 1 ]]; then + break + fi + + sleep 0.1 + done + + if [[ -z "$temp_client" ]]; then + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" SKIP "cannot identify debugfs client for temp mount" + return + fi + + if [[ "${#new_clients[@]}" -gt 1 ]]; then + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" SKIP "multiple new debugfs clients appeared" + return + fi + + temp_trigger="$DEBUGFS_ROOT/$temp_client/reset/trigger" + temp_status="$DEBUGFS_ROOT/$temp_client/reset/status" + + echo "umount_dirty_seed" > "$temp_file" 2>/dev/null || { + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" FAIL "cannot create dirty state on temp mount" + return + } + sync "$temp_file" + python3 -c " +import os, sys +fd = os.open('$temp_file', os.O_WRONLY | os.O_APPEND) +os.write(fd, b'dirty_for_umount_test\\n') +os.close(fd) +" 2>/dev/null || { + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" FAIL "cannot dirty temp mount for reset overlap" + return + } + + echo "unmount_test" > "$temp_trigger" 2>/dev/null && trigger_ok=1 || trigger_ok=0 + if [[ "$trigger_ok" -ne 1 ]]; then + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" FAIL "cannot trigger reset on temp mount" + return + fi + + if ! wait_status_nonidle "$temp_status" 10; then + phase="$(awk -F': ' '$1 == "phase" {print $2}' "$temp_status" 2>/dev/null)" + umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" FAIL \ + "reset never became active before umount (phase=${phase:-unknown})" + return + fi + + local umount_ok=0 + timeout 30 umount "$temp_mnt" 2>/dev/null && umount_ok=1 + + if [[ "$umount_ok" -ne 1 ]]; then + umount -l "$temp_mnt" 2>/dev/null || true + rmdir "$temp_mnt" 2>/dev/null + result "$num" "$name" FAIL "umount hung for >30s" + return + fi + + rmdir "$temp_mnt" 2>/dev/null + + ls "$MOUNT_POINT" > /dev/null 2>&1 || { + result "$num" "$name" FAIL "original mount unhealthy after test" + return + } + + result "$num" "$name" PASS +} + +# --- Main -------------------------------------------------------------------- + +usage() +{ + cat < [--client-id ] [--debugfs-root ] + +Runs targeted corner-case tests for the CephFS client reset feature. +Requires root (debugfs access) and a mounted CephFS filesystem. + +Options: + --mount-point PATH CephFS mount point (required) + --client-id ID Ceph debugfs client id (auto-detect if one client) + --debugfs-root PATH Debugfs ceph root (default: /sys/kernel/debug/ceph) + --help Show this message +EOF +} + +main() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --mount-point) MOUNT_POINT="$2"; shift 2 ;; + --client-id) DEBUGFS_CLIENT="$2"; shift 2 ;; + --debugfs-root) DEBUGFS_ROOT="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 2 ;; + esac + done + + if [[ -z "$MOUNT_POINT" ]]; then + echo "--mount-point is required" >&2 + usage + exit 2 + fi + + if [[ ! -d "$MOUNT_POINT" ]]; then + echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2 + exit "$KSFT_SKIP" + fi + + discover_debugfs + TRIGGER_PATH="$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset/trigger" + STATUS_PATH="$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset/status" + + log "CephFS client reset corner case tests" + log "Mount: $MOUNT_POINT" + log "Client: $DEBUGFS_CLIENT" + echo "" + + test_ebusy_rejection + test_dirty_caps_at_reset + test_flock_after_reset + test_unmount_during_reset + + echo "" + echo "Results: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped (of $TOTAL)" + + if [[ "$FAIL_COUNT" -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +main "$@" diff --git a/tools/testing/selftests/filesystems/ceph/reset_stress.sh b/tools/testing/selftests/filesystems/ceph/reset_stress.sh new file mode 100755 index 000000000000..c503c75a5f7a --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/reset_stress.sh @@ -0,0 +1,694 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# CephFS reset stress test: +# - Runs concurrent I/O and rename workloads +# - Triggers random client resets through debugfs +# - Validates consistency and recovery behavior + +set -euo pipefail + +KSFT_SKIP=4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# kselftest auto-detect: when invoked with no arguments (e.g. by +# "make run_tests"), find a CephFS mount automatically or skip. +if [[ $# -eq 0 ]]; then + MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)" + if [[ -z "$MOUNT_POINT" ]]; then + echo "SKIP: No CephFS mount found and --mount-point not specified" + exit "$KSFT_SKIP" + fi + exec "$0" --mount-point "$MOUNT_POINT" +fi + +PROFILE="moderate" +DURATION_SEC="" +COOLDOWN_SEC=20 +FILE_COUNT=64 +IO_WORKERS="" +RENAME_WORKERS="" +MOUNT_POINT="" +OUT_DIR="" +CLIENT_ID="" +DEBUGFS_ROOT="/sys/kernel/debug/ceph" +SLO_SECONDS=30 +EXPECT_RESET=1 +DMESG_CMD="" +SUDO="" + +RESET_MIN_SEC=5 +RESET_MAX_SEC=15 + +RUN_ID="$(date +%Y%m%d-%H%M%S)" +WORKLOAD_FLAG="" +RESET_FLAG="" +DATA_DIR="" + +IO_LOG="" +RENAME_LOG="" +RESET_LOG="" +STATUS_LOG="" +STATUS_BEFORE="" +STATUS_FINAL="" +DMESG_LOG="" +SUMMARY_LOG="" +REPORT_JSON="" + +RESET_PID=0 +STATUS_PID=0 +declare -a IO_WORKER_PIDS=() +declare -a RENAME_WORKER_PIDS=() + +usage() +{ + cat < [options] + +Required: + --mount-point PATH CephFS mount point to test under + +Options: + --profile NAME baseline|moderate|aggressive|soak (default: moderate) + --duration-sec N Override profile runtime in seconds + --cooldown-sec N Workload drain time after injector stop (default: 20) + --file-count N Number of logical files (default: 64) + --io-workers N Number of concurrent I/O workers (profile default) + --rename-workers N Number of concurrent rename workers (profile default) + --out-dir PATH Artifact directory (default: /tmp/ceph_reset_stress_) + --client-id ID Ceph debugfs client id; auto-detect if one client exists + --debugfs-root PATH Debugfs Ceph root (default: /sys/kernel/debug/ceph) + --slo-seconds N Max allowed post-reset stall window (default: 30) + --no-reset Disable reset injector (baseline mode helper) + --help Show this message + +Examples: + $0 --mount-point /mnt/cephfs --profile moderate + $0 --mount-point /mnt/cephfs --profile aggressive --duration-sec 300 + $0 --mount-point /mnt/cephfs --profile baseline --no-reset +EOF +} + +now_ms() +{ + date +%s%3N +} + +set_profile_defaults() +{ + case "$PROFILE" in + baseline) + RESET_MIN_SEC=0 + RESET_MAX_SEC=0 + EXPECT_RESET=0 + : "${DURATION_SEC:=600}" + : "${IO_WORKERS:=1}" + : "${RENAME_WORKERS:=1}" + ;; + moderate) + RESET_MIN_SEC=5 + RESET_MAX_SEC=15 + : "${DURATION_SEC:=900}" + : "${IO_WORKERS:=2}" + : "${RENAME_WORKERS:=1}" + ;; + aggressive) + RESET_MIN_SEC=1 + RESET_MAX_SEC=5 + : "${DURATION_SEC:=900}" + : "${IO_WORKERS:=4}" + : "${RENAME_WORKERS:=2}" + ;; + soak) + RESET_MIN_SEC=5 + RESET_MAX_SEC=15 + : "${DURATION_SEC:=3600}" + : "${IO_WORKERS:=2}" + : "${RENAME_WORKERS:=1}" + ;; + *) + echo "Unknown profile: $PROFILE" >&2 + exit 2 + ;; + esac +} + +log_summary() +{ + local msg="$1" + printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$msg" | tee -a "$SUMMARY_LOG" +} + +discover_client_id() +{ + local candidates=() + local entry + + if [[ -n "$CLIENT_ID" ]]; then + if ! $SUDO test -d "$DEBUGFS_ROOT/$CLIENT_ID/reset"; then + echo "SKIP: reset debugfs not found for client-id=$CLIENT_ID" >&2 + exit "$KSFT_SKIP" + fi + return 0 + fi + + if ! $SUDO test -d "$DEBUGFS_ROOT"; then + echo "SKIP: Debugfs root not found: $DEBUGFS_ROOT" >&2 + exit "$KSFT_SKIP" + fi + + while IFS= read -r entry; do + $SUDO test -d "$DEBUGFS_ROOT/$entry/reset" || continue + $SUDO test -w "$DEBUGFS_ROOT/$entry/reset/trigger" || continue + candidates+=("$entry") + done < <($SUDO ls -1 "$DEBUGFS_ROOT" 2>/dev/null || true) + + if [[ ${#candidates[@]} -eq 1 ]]; then + CLIENT_ID="${candidates[0]}" + return 0 + fi + + if [[ ${#candidates[@]} -eq 0 ]]; then + echo "SKIP: No writable Ceph reset interface found under $DEBUGFS_ROOT" >&2 + exit "$KSFT_SKIP" + fi + + echo "SKIP: Multiple Ceph clients found (${candidates[*]}). Use --client-id." >&2 + exit "$KSFT_SKIP" +} + +init_dataset() +{ + local i + mkdir -p "$DATA_DIR/A" "$DATA_DIR/B" + + for ((i = 0; i < FILE_COUNT; i++)); do + printf 'seed logical_id=%05d ts_ms=%s\n' "$i" "$(now_ms)" > "$DATA_DIR/A/file_$(printf '%05d' "$i")" + done +} + +io_worker() +{ + set +e + local worker_id="$1" + local seq=0 + local id + local relpath + local abspath + local payload + local hash + local ts + + while [[ -f "$WORKLOAD_FLAG" ]]; do + id="$(printf '%05d' $((RANDOM % FILE_COUNT)))" + if [[ -f "$DATA_DIR/A/file_$id" ]]; then + relpath="A/file_$id" + elif [[ -f "$DATA_DIR/B/file_$id" ]]; then + relpath="B/file_$id" + else + sleep 0.02 + continue + fi + + abspath="$DATA_DIR/$relpath" + alt_relpath="" + if [[ "$relpath" == A/* ]]; then + alt_relpath="B/file_$id" + else + alt_relpath="A/file_$id" + fi + alt_abspath="$DATA_DIR/$alt_relpath" + payload="worker=${worker_id} io_seq=${seq} id=${id} ts_ms=$(now_ms)" + result="$( + python3 - "$abspath" "$alt_abspath" "$payload" <<'PY' +import hashlib +import os +import sys + +path = sys.argv[1] +alt_path = sys.argv[2] +payload = sys.argv[3] + +try: + fd = os.open(path, os.O_RDWR | os.O_APPEND) + actual = path +except FileNotFoundError: + try: + fd = os.open(alt_path, os.O_RDWR | os.O_APPEND) + actual = alt_path + except FileNotFoundError: + sys.exit(1) + +try: + os.write(fd, (payload + "\n").encode()) + os.fsync(fd) + os.lseek(fd, 0, os.SEEK_SET) + digest = hashlib.sha256() + while True: + chunk = os.read(fd, 1 << 20) + if not chunk: + break + digest.update(chunk) + print(actual + " " + digest.hexdigest()) +finally: + os.close(fd) +PY + )" || { + sleep 0.02 + continue + } + + actual_abspath="${result%% *}" + hash="${result#* }" + if [[ "$actual_abspath" == "$alt_abspath" ]]; then + relpath="$alt_relpath" + fi + + ts="$(now_ms)" + printf '%s,%s,%s,%s,%s\n' "$ts" "$seq" "$id" "$relpath" "$hash" >> "$IO_LOG" + seq=$((seq + 1)) + sleep 0.02 + done +} + +rename_worker() +{ + set +e + local worker_id="$1" + local seq=0 + local id + local src_rel + local dst_rel + local rc + local ts + + while [[ -f "$WORKLOAD_FLAG" ]]; do + id="$(printf '%05d' $((RANDOM % FILE_COUNT)))" + + if [[ -f "$DATA_DIR/A/file_$id" ]]; then + src_rel="A/file_$id" + dst_rel="B/file_$id" + elif [[ -f "$DATA_DIR/B/file_$id" ]]; then + src_rel="B/file_$id" + dst_rel="A/file_$id" + else + sleep 0.02 + continue + fi + + ts="$(now_ms)" + if mv -T "$DATA_DIR/$src_rel" "$DATA_DIR/$dst_rel" 2>/dev/null; then + rc=0 + else + rc=$? + fi + printf '%s,%s,%s,%s,%s,%s,%s\n' "$ts" "$worker_id" "$seq" "$id" "$src_rel" "$dst_rel" "$rc" >> "$RENAME_LOG" + seq=$((seq + 1)) + sleep 0.02 + done +} + +random_sleep_seconds() +{ + local min_sec="$1" + local max_sec="$2" + local wait_sec + local span + + span=$((max_sec - min_sec + 1)) + wait_sec=$((min_sec + RANDOM % span)) + sleep "$wait_sec" +} + +reset_injector() +{ + set +e + local trigger_path="$1" + local seq=0 + local ts + local reason + local rc + + while [[ -f "$RESET_FLAG" ]]; do + random_sleep_seconds "$RESET_MIN_SEC" "$RESET_MAX_SEC" + [[ -f "$RESET_FLAG" ]] || break + + ts="$(now_ms)" + reason="stress_${seq}_${ts}" + if echo "$reason" | $SUDO tee "$trigger_path" > /dev/null 2>&1; then + rc=0 + else + rc=$? + fi + printf '%s,%s,%s,%s\n' "$ts" "$seq" "$reason" "$rc" >> "$RESET_LOG" + seq=$((seq + 1)) + done +} + +status_sampler() +{ + set +e + local status_path="$1" + local ts + local kv_line + + while [[ -f "$WORKLOAD_FLAG" || -f "$RESET_FLAG" ]]; do + ts="$(now_ms)" + if $SUDO test -r "$status_path"; then + kv_line="$($SUDO awk -F': ' 'NF>=2 {gsub(/ /, "", $1); gsub(/ /, "", $2); printf "%s=%s;", $1, $2}' "$status_path")" + printf '%s,%s\n' "$ts" "$kv_line" >> "$STATUS_LOG" + fi + sleep 1 + done +} + +stop_pid_with_timeout() +{ + local pid="$1" + local name="$2" + local timeout="$3" + local waited=0 + + if [[ "$pid" -le 0 ]]; then + return 0 + fi + + while kill -0 "$pid" 2>/dev/null; do + if (( waited >= timeout )); then + log_summary "Timeout waiting for $name (pid=$pid), sending SIGTERM/SIGKILL" + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + + wait "$pid" 2>/dev/null || true + return 0 +} + +detect_privileges() +{ + if [[ -r "$DEBUGFS_ROOT" ]]; then + SUDO="" + elif sudo -n true 2>/dev/null; then + SUDO="sudo" + else + echo "WARNING: $DEBUGFS_ROOT is not readable and passwordless sudo is not available" >&2 + echo "WARNING: reset injection, debugfs status checks, and dmesg capture will not work" >&2 + fi + + if $SUDO dmesg > /dev/null 2>&1; then + DMESG_CMD="$SUDO dmesg" + else + DMESG_CMD="" + echo "WARNING: dmesg is not accessible; kernel errors (hung tasks) will not be detected" >&2 + fi +} + +check_dmesg() +{ + local start_epoch="$1" + + if [[ -z "$DMESG_CMD" ]]; then + return 0 + fi + + if ! $DMESG_CMD --since "@$start_epoch" > "$DMESG_LOG" 2>/dev/null; then + if ! $DMESG_CMD > "$DMESG_LOG" 2>/dev/null; then + log_summary "WARNING: dmesg capture failed unexpectedly" + return 0 + fi + log_summary "dmesg --since unsupported; captured full dmesg" + fi + + if grep -qi "hung task" "$DMESG_LOG" 2>/dev/null; then + log_summary "ERROR: kernel log contains 'hung task' during test window" + return 1 + fi + + return 0 +} + +cleanup() +{ + rm -f "$WORKLOAD_FLAG" "$RESET_FLAG" + local pid + for pid in "${IO_WORKER_PIDS[@]}" "${RENAME_WORKER_PIDS[@]}" "$RESET_PID" "$STATUS_PID"; do + [[ "$pid" -gt 0 ]] 2>/dev/null && kill "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true +} + +parse_args() +{ + while [[ $# -gt 0 ]]; do + case "$1" in + --mount-point) + MOUNT_POINT="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --duration-sec) + DURATION_SEC="$2" + shift 2 + ;; + --cooldown-sec) + COOLDOWN_SEC="$2" + shift 2 + ;; + --file-count) + FILE_COUNT="$2" + shift 2 + ;; + --io-workers) + IO_WORKERS="$2" + shift 2 + ;; + --rename-workers) + RENAME_WORKERS="$2" + shift 2 + ;; + --out-dir) + OUT_DIR="$2" + shift 2 + ;; + --client-id) + CLIENT_ID="$2" + shift 2 + ;; + --debugfs-root) + DEBUGFS_ROOT="$2" + shift 2 + ;; + --slo-seconds) + SLO_SECONDS="$2" + shift 2 + ;; + --no-reset) + EXPECT_RESET=0 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage + exit 2 + ;; + esac + done +} + +main() +{ + local start_epoch + local trigger_path="" + local status_path="" + local final_rc=0 + local reset_enabled=0 + local i + + parse_args "$@" + + if [[ -z "$MOUNT_POINT" ]]; then + echo "--mount-point is required" >&2 + usage + exit 2 + fi + + if [[ ! -d "$MOUNT_POINT" ]]; then + echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2 + exit "$KSFT_SKIP" + fi + + if ! touch "$MOUNT_POINT/.ceph_reset_test_probe" 2>/dev/null; then + echo "SKIP: Mount point is not writable: $MOUNT_POINT" >&2 + exit "$KSFT_SKIP" + fi + rm -f "$MOUNT_POINT/.ceph_reset_test_probe" + + if ! command -v python3 > /dev/null 2>&1; then + echo "SKIP: python3 is required but not found in PATH" >&2 + exit "$KSFT_SKIP" + fi + + if ! stat -f -c '%T' "$MOUNT_POINT" 2>/dev/null | grep -qi ceph; then + echo "WARNING: $MOUNT_POINT does not appear to be a CephFS mount" >&2 + fi + + detect_privileges + + set_profile_defaults + if [[ "$EXPECT_RESET" -eq 0 ]]; then + PROFILE="baseline" + RESET_MIN_SEC=0 + RESET_MAX_SEC=0 + fi + + if ! [[ "$IO_WORKERS" =~ ^[0-9]+$ && "$RENAME_WORKERS" =~ ^[0-9]+$ ]]; then + echo "io-workers and rename-workers must be integers" >&2 + exit 2 + fi + + if [[ "$IO_WORKERS" -le 0 || "$RENAME_WORKERS" -le 0 ]]; then + echo "io-workers and rename-workers must be > 0" >&2 + exit 2 + fi + + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="/tmp/ceph_reset_stress_${RUN_ID}" + fi + mkdir -p "$OUT_DIR" + + WORKLOAD_FLAG="$OUT_DIR/workload.running" + RESET_FLAG="$OUT_DIR/reset.running" + + DATA_DIR="$MOUNT_POINT/ceph_reset_stress_${RUN_ID}" + mkdir -p "$DATA_DIR" + + IO_LOG="$OUT_DIR/io.log" + RENAME_LOG="$OUT_DIR/rename.log" + RESET_LOG="$OUT_DIR/reset.log" + STATUS_LOG="$OUT_DIR/status.log" + STATUS_BEFORE="$OUT_DIR/reset_status.before" + STATUS_FINAL="$OUT_DIR/reset_status.final" + DMESG_LOG="$OUT_DIR/dmesg.log" + SUMMARY_LOG="$OUT_DIR/summary.log" + REPORT_JSON="$OUT_DIR/validator_report.json" + + : > "$IO_LOG" + : > "$RENAME_LOG" + : > "$RESET_LOG" + : > "$STATUS_LOG" + : > "$SUMMARY_LOG" + + start_epoch="$(date +%s)" + + log_summary "Starting Ceph reset stress test" + log_summary "Profile=$PROFILE duration=${DURATION_SEC}s cooldown=${COOLDOWN_SEC}s file_count=${FILE_COUNT} io_workers=${IO_WORKERS} rename_workers=${RENAME_WORKERS}" + [[ -n "$SUDO" ]] && log_summary "Using sudo for privileged operations" + [[ -z "$DMESG_CMD" ]] && log_summary "WARNING: dmesg not available; hung task detection disabled" + log_summary "Artifacts=$OUT_DIR" + log_summary "Data dir=$DATA_DIR" + + init_dataset + + if [[ "$EXPECT_RESET" -eq 1 ]]; then + discover_client_id + trigger_path="$DEBUGFS_ROOT/$CLIENT_ID/reset/trigger" + status_path="$DEBUGFS_ROOT/$CLIENT_ID/reset/status" + if ! $SUDO test -w "$trigger_path"; then + echo "SKIP: Reset trigger is not writable: $trigger_path" >&2 + exit "$KSFT_SKIP" + fi + if ! $SUDO test -r "$status_path"; then + echo "SKIP: Reset status is not readable: $status_path" >&2 + exit "$KSFT_SKIP" + fi + $SUDO cat "$status_path" > "$STATUS_BEFORE" || true + reset_enabled=1 + log_summary "Using ceph client id: $CLIENT_ID" + fi + + trap cleanup EXIT INT TERM + + touch "$WORKLOAD_FLAG" + for ((i = 0; i < IO_WORKERS; i++)); do + io_worker "$i" & + IO_WORKER_PIDS+=("$!") + done + + for ((i = 0; i < RENAME_WORKERS; i++)); do + rename_worker "$i" & + RENAME_WORKER_PIDS+=("$!") + done + + if [[ "$reset_enabled" -eq 1 ]]; then + touch "$RESET_FLAG" + reset_injector "$trigger_path" & + RESET_PID=$! + + status_sampler "$status_path" & + STATUS_PID=$! + fi + + sleep "$DURATION_SEC" + + if [[ "$reset_enabled" -eq 1 ]]; then + rm -f "$RESET_FLAG" + stop_pid_with_timeout "$RESET_PID" "reset_injector" 20 || final_rc=1 + log_summary "Injector stopped; entering cooldown=${COOLDOWN_SEC}s" + fi + + sleep "$COOLDOWN_SEC" + + rm -f "$WORKLOAD_FLAG" + for i in "${!IO_WORKER_PIDS[@]}"; do + stop_pid_with_timeout "${IO_WORKER_PIDS[$i]}" "io_worker[$i]" 20 || final_rc=1 + done + for i in "${!RENAME_WORKER_PIDS[@]}"; do + stop_pid_with_timeout "${RENAME_WORKER_PIDS[$i]}" "rename_worker[$i]" 20 || final_rc=1 + done + + if [[ "$reset_enabled" -eq 1 ]]; then + stop_pid_with_timeout "$STATUS_PID" "status_sampler" 10 || final_rc=1 + $SUDO cat "$status_path" > "$STATUS_FINAL" || true + fi + + if ! check_dmesg "$start_epoch"; then + final_rc=1 + fi + + if ! python3 "$SCRIPT_DIR/validate_consistency.py" \ + --data-dir "$DATA_DIR" \ + --file-count "$FILE_COUNT" \ + --io-log "$IO_LOG" \ + --rename-log "$RENAME_LOG" \ + --reset-log "$RESET_LOG" \ + --status-final "$STATUS_FINAL" \ + --slo-seconds "$SLO_SECONDS" \ + --report-json "$REPORT_JSON" \ + $( [[ "$reset_enabled" -eq 1 ]] && echo "--expect-reset" ); then + final_rc=1 + fi + + if [[ "$final_rc" -eq 0 ]]; then + log_summary "PASS: stress run completed successfully" + else + log_summary "FAIL: stress run detected one or more failures" + fi + + log_summary "Artifacts available in: $OUT_DIR" + exit "$final_rc" +} + +main "$@" diff --git a/tools/testing/selftests/filesystems/ceph/run_validation.sh b/tools/testing/selftests/filesystems/ceph/run_validation.sh new file mode 100755 index 000000000000..5d521e4f9e9b --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/run_validation.sh @@ -0,0 +1,350 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# CephFS client reset - single-command validation. +# Runs all test stages in sequence with per-stage timeouts. +# If any stage hangs (filesystem stuck, process blocked), the +# timeout kills it and reports failure. +# +# Usage: +# sudo ./run_validation.sh --mount-point /mnt/mycephfs +# +# Expected output on success: +# +# === CephFS Client Reset Validation === +# [stage 1/5] baseline PASS (60s, no resets) +# [stage 2/5] corner_cases PASS (4/4 passed) +# [stage 3/5] moderate PASS (120s, resets every 5-15s) +# [stage 4/5] aggressive PASS (120s, resets every 1-5s) +# [stage 5/5] status_check PASS (phase=idle, last_errno=0) +# +# RESULT: 5/5 stages passed +# Artifacts: /tmp/ceph_reset_validation_ + +set -uo pipefail + +KSFT_SKIP=4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# kselftest auto-detect: when invoked with no arguments (e.g. by +# "make run_tests"), find a CephFS mount automatically or skip. +if [[ $# -eq 0 ]]; then + MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)" + if [[ -z "$MOUNT_POINT" ]]; then + echo "SKIP: No CephFS mount found and --mount-point not specified" + exit "$KSFT_SKIP" + fi + exec "$0" --mount-point "$MOUNT_POINT" +fi + +MOUNT_POINT="" +CLIENT_ID="" +declare -a CLIENT_ARGS=() +declare -a DEBUGFS_ARGS=() +RUN_ID="$(date +%Y%m%d-%H%M%S)" +OUT_DIR="/tmp/ceph_reset_validation_${RUN_ID}" +DEBUGFS_ROOT="/sys/kernel/debug/ceph" + +# Timeout margins: stage runtime + cooldown + validation + safety buffer +STAGE1_TIMEOUT=120 # 60s run + 20s cooldown + 40s buffer +STAGE2_TIMEOUT=300 # 4 corner cases, 30s each worst case + buffer +STAGE3_TIMEOUT=240 # 120s run + 20s cooldown + 100s buffer +STAGE4_TIMEOUT=240 # 120s run + 20s cooldown + 100s buffer +STAGE5_TIMEOUT=10 # just reading debugfs + +PASS=0 +FAIL=0 +TOTAL=5 + +usage() +{ + cat < [options] + +Required: + --mount-point PATH CephFS mount point + +Options: + --out-dir PATH Artifact directory (default: /tmp/ceph_reset_validation_) + --client-id ID Ceph debugfs client id (optional) + --debugfs-root PATH Debugfs Ceph root (default: /sys/kernel/debug/ceph) + --help Show this message +EOF +} + +stage_result() +{ + local num="$1" + local name="$2" + local status="$3" + local detail="$4" + + if [[ "$status" == "PASS" ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + fi + printf '[stage %d/%d] %-16s %s (%s)\n' "$num" "$TOTAL" "$name" "$status" "$detail" +} + +# Run a command with a timeout. Returns 0 on success, 1 on failure/timeout. +# Sets RUN_TIMED_OUT=1 if killed by timeout. +# +# The stage command runs in its own session/process group (via setsid). +# On timeout the entire process group is killed, not just the top-level +# script PID. This is required because stage scripts (reset_stress.sh, +# reset_corner_cases.sh) spawn child processes - I/O workers, rename +# workers, reset injectors, samplers - that would otherwise survive the +# timeout and bleed into later stages, invalidating results. +RUN_TIMED_OUT=0 + +run_with_timeout() +{ + local timeout_sec="$1" + local logfile="$2" + shift 2 + + RUN_TIMED_OUT=0 + + # Start the stage in its own session via setsid so all descendant + # processes share a process group that we can kill atomically. + # In a non-interactive script, background children are not process + # group leaders, so setsid(1) calls setsid(2) directly (no extra + # fork) and the PID we capture IS the group leader. + setsid "$@" > "$logfile" 2>&1 & + local pid=$! + + # Watchdog: on timeout, kill the entire process group + ( + sleep "$timeout_sec" + if kill -0 "$pid" 2>/dev/null; then + echo "TIMEOUT: stage exceeded ${timeout_sec}s, killing process group $pid" >> "$logfile" + kill -TERM -- -"$pid" 2>/dev/null + sleep 2 + kill -KILL -- -"$pid" 2>/dev/null + fi + ) & + local watchdog_pid=$! + + # Wait for the stage command + wait "$pid" 2>/dev/null + local rc=$? + + # Kill the watchdog if it's still running + kill "$watchdog_pid" 2>/dev/null + wait "$watchdog_pid" 2>/dev/null + + # Check if it was killed by timeout + if grep -q "^TIMEOUT:" "$logfile" 2>/dev/null; then + RUN_TIMED_OUT=1 + return 1 + fi + + return "$rc" +} + +find_status_path() +{ + local entry + + if [[ -n "$CLIENT_ID" ]]; then + if [[ -r "$DEBUGFS_ROOT/$CLIENT_ID/reset/status" ]]; then + echo "$DEBUGFS_ROOT/$CLIENT_ID/reset/status" + return 0 + fi + return 1 + fi + + for entry in "$DEBUGFS_ROOT"/*/; do + if [[ -r "${entry}reset/status" ]]; then + echo "${entry}reset/status" + return 0 + fi + done + return 1 +} + +read_status_field() +{ + local status_path="$1" + local field="$2" + awk -F': ' -v key="$field" '$1 == key {print $2}' "$status_path" 2>/dev/null +} + +# --- Parse arguments ------------------------------------------------------- + +while [[ $# -gt 0 ]]; do + case "$1" in + --mount-point) MOUNT_POINT="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --client-id) CLIENT_ID="$2"; shift 2 ;; + --debugfs-root) DEBUGFS_ROOT="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 2 ;; + esac +done + +if [[ -z "$MOUNT_POINT" ]]; then + echo "SKIP: --mount-point is required" >&2 + usage + exit "$KSFT_SKIP" +fi + +if [[ ! -d "$MOUNT_POINT" ]]; then + echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2 + exit "$KSFT_SKIP" +fi + +# Auto-detect client id when not specified, so all stages (including +# stage 5 status check) use the same client consistently. +if [[ -z "$CLIENT_ID" ]]; then + candidates=() + for entry in "$DEBUGFS_ROOT"/*/; do + name="$(basename "$entry")" + if [[ -r "${entry}reset/status" ]]; then + candidates+=("$name") + fi + done + if [[ ${#candidates[@]} -eq 1 ]]; then + CLIENT_ID="${candidates[0]}" + elif [[ ${#candidates[@]} -gt 1 ]]; then + echo "SKIP: Multiple Ceph clients found (${candidates[*]}). Use --client-id." >&2 + exit "$KSFT_SKIP" + fi +fi + +if [[ -n "$CLIENT_ID" ]]; then + CLIENT_ARGS=(--client-id "$CLIENT_ID") +fi +DEBUGFS_ARGS=(--debugfs-root "$DEBUGFS_ROOT") + +# Quick sanity: can we write to the mount? +if ! touch "$MOUNT_POINT/.validation_probe_$$" 2>/dev/null; then + echo "SKIP: Mount point is not writable: $MOUNT_POINT" >&2 + exit "$KSFT_SKIP" +fi +rm -f "$MOUNT_POINT/.validation_probe_$$" + +mkdir -p "$OUT_DIR" + +echo "" +echo "=== CephFS Client Reset Validation ===" +echo "" + +# --- Stage 1: Baseline (no resets) ----------------------------------------- + +stage1_out="$OUT_DIR/stage1_baseline" +if run_with_timeout "$STAGE1_TIMEOUT" "$stage1_out.log" \ + "$SCRIPT_DIR/reset_stress.sh" \ + --mount-point "$MOUNT_POINT" \ + --profile baseline \ + --no-reset \ + --duration-sec 60 \ + "${CLIENT_ARGS[@]}" \ + "${DEBUGFS_ARGS[@]}" \ + --out-dir "$stage1_out"; then + stage_result 1 "baseline" "PASS" "60s, no resets" +elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then + stage_result 1 "baseline" "FAIL" "HUNG: killed after ${STAGE1_TIMEOUT}s" +else + stage_result 1 "baseline" "FAIL" "see $stage1_out.log" +fi + +# --- Stage 2: Corner cases ------------------------------------------------- + +stage2_out="$OUT_DIR/stage2_corner_cases" +mkdir -p "$stage2_out" +if run_with_timeout "$STAGE2_TIMEOUT" "$stage2_out/output.log" \ + "$SCRIPT_DIR/reset_corner_cases.sh" \ + "${CLIENT_ARGS[@]}" \ + "${DEBUGFS_ARGS[@]}" \ + --mount-point "$MOUNT_POINT"; then + pass_line=$(grep -Eo '[0-9]+ passed, [0-9]+ failed, [0-9]+ skipped' "$stage2_out/output.log" | tail -1) + stage_result 2 "corner_cases" "PASS" "${pass_line:-all tests passed}" +elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then + stage_result 2 "corner_cases" "FAIL" "HUNG: killed after ${STAGE2_TIMEOUT}s" +else + fail_line=$(grep -c 'FAIL' "$stage2_out/output.log" 2>/dev/null || echo "?") + stage_result 2 "corner_cases" "FAIL" "${fail_line} failures, see $stage2_out/output.log" +fi + +# --- Stage 3: Moderate resets ----------------------------------------------- + +stage3_out="$OUT_DIR/stage3_moderate" +if run_with_timeout "$STAGE3_TIMEOUT" "$stage3_out.log" \ + "$SCRIPT_DIR/reset_stress.sh" \ + --mount-point "$MOUNT_POINT" \ + --profile moderate \ + --duration-sec 120 \ + "${CLIENT_ARGS[@]}" \ + "${DEBUGFS_ARGS[@]}" \ + --out-dir "$stage3_out"; then + stage_result 3 "moderate" "PASS" "120s, resets every 5-15s" +elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then + stage_result 3 "moderate" "FAIL" "HUNG: killed after ${STAGE3_TIMEOUT}s" +else + stage_result 3 "moderate" "FAIL" "see $stage3_out.log" +fi + +# --- Stage 4: Aggressive resets --------------------------------------------- + +stage4_out="$OUT_DIR/stage4_aggressive" +if run_with_timeout "$STAGE4_TIMEOUT" "$stage4_out.log" \ + "$SCRIPT_DIR/reset_stress.sh" \ + --mount-point "$MOUNT_POINT" \ + --profile aggressive \ + --duration-sec 120 \ + "${CLIENT_ARGS[@]}" \ + "${DEBUGFS_ARGS[@]}" \ + --out-dir "$stage4_out"; then + stage_result 4 "aggressive" "PASS" "120s, resets every 1-5s" +elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then + stage_result 4 "aggressive" "FAIL" "HUNG: killed after ${STAGE4_TIMEOUT}s" +else + stage_result 4 "aggressive" "FAIL" "see $stage4_out.log" +fi + +# --- Stage 5: Post-run status check ---------------------------------------- + +status_path="" +if status_path=$(find_status_path); then + phase=$(read_status_field "$status_path" "phase") + last_errno=$(read_status_field "$status_path" "last_errno") + failure_count=$(read_status_field "$status_path" "failure_count") + drain_timed_out=$(read_status_field "$status_path" "drain_timed_out") + sessions_reset=$(read_status_field "$status_path" "sessions_reset") + blocked=$(read_status_field "$status_path" "blocked_requests") + + # Save full status + cat "$status_path" > "$OUT_DIR/final_status.txt" 2>/dev/null + + errors="" + [[ "$phase" != "idle" ]] && errors="${errors}phase=$phase " + [[ "$last_errno" != "0" ]] && errors="${errors}last_errno=$last_errno " + [[ "$failure_count" != "0" && -n "$failure_count" ]] && errors="${errors}failure_count=$failure_count " + [[ "$blocked" != "0" ]] && errors="${errors}blocked_requests=$blocked " + + if [[ -z "$errors" ]]; then + detail="phase=$phase, last_errno=$last_errno, failure_count=${failure_count:-0}" + [[ "$drain_timed_out" == "yes" ]] && detail="$detail, drain_timed_out=yes" + [[ -n "$sessions_reset" ]] && detail="$detail, sessions_reset=$sessions_reset" + stage_result 5 "status_check" "PASS" "$detail" + else + stage_result 5 "status_check" "FAIL" "$errors" + fi +else + stage_result 5 "status_check" "FAIL" "cannot read reset/status" +fi + +# --- Summary ---------------------------------------------------------------- + +echo "" +if [[ "$FAIL" -eq 0 ]]; then + echo "RESULT: $PASS/$TOTAL stages passed" +else + echo "RESULT: $PASS/$TOTAL stages passed, $FAIL FAILED" +fi +echo "Artifacts: $OUT_DIR" +echo "" + +exit "$FAIL" diff --git a/tools/testing/selftests/filesystems/ceph/settings b/tools/testing/selftests/filesystems/ceph/settings new file mode 100644 index 000000000000..79b65bdf05db --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/settings @@ -0,0 +1 @@ +timeout=1200 diff --git a/tools/testing/selftests/filesystems/ceph/validate_consistency.py b/tools/testing/selftests/filesystems/ceph/validate_consistency.py new file mode 100755 index 000000000000..c230a59bdb3a --- /dev/null +++ b/tools/testing/selftests/filesystems/ceph/validate_consistency.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import argparse +import bisect +import hashlib +import json +import os +from pathlib import Path + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1 << 20) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def parse_io_log(path: Path): + records = [] + if not path.exists(): + return records + with path.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, 1): + line = line.strip() + if not line: + continue + parts = line.split(",") + if len(parts) != 5: + raise ValueError(f"io log line {line_no}: expected 5 columns, got {len(parts)}") + ts_ms, seq, logical_id, relpath, digest = parts + records.append( + { + "ts_ms": int(ts_ms), + "seq": int(seq), + "logical_id": int(logical_id), + "relpath": relpath, + "digest": digest, + } + ) + return records + + +def parse_rename_log(path: Path): + records = [] + if not path.exists(): + return records + with path.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, 1): + line = line.strip() + if not line: + continue + parts = line.split(",") + if len(parts) == 6: + ts_ms, seq, logical_id, src_rel, dst_rel, rc = parts + elif len(parts) == 7: + ts_ms, worker_id, seq, logical_id, src_rel, dst_rel, rc = parts + _ = worker_id # worker id is informational only + else: + raise ValueError( + f"rename log line {line_no}: expected 6 or 7 columns, got {len(parts)}" + ) + records.append( + { + "ts_ms": int(ts_ms), + "seq": int(seq), + "logical_id": int(logical_id), + "src_rel": src_rel, + "dst_rel": dst_rel, + "rc": int(rc), + } + ) + return records + + +def parse_reset_log(path: Path): + records = [] + if not path.exists(): + return records + with path.open("r", encoding="utf-8") as handle: + for line_no, line in enumerate(handle, 1): + line = line.strip() + if not line: + continue + parts = line.split(",") + if len(parts) != 4: + raise ValueError(f"reset log line {line_no}: expected 4 columns, got {len(parts)}") + ts_ms, seq, reason, rc = parts + records.append( + { + "ts_ms": int(ts_ms), + "seq": int(seq), + "reason": reason, + "rc": int(rc), + } + ) + return records + + +def parse_status_file(path: Path): + status = {} + if not path.exists(): + return status + with path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line or ":" not in line: + continue + key, value = line.split(":", 1) + status[key.strip()] = value.strip() + return status + + +def to_int(value: str, default: int = 0): + try: + return int(value) + except Exception: + return default + + +def validate_namespace(data_dir: Path, file_count: int, issues): + actual_locations = {} + actual_paths = {} + for logical_id in range(file_count): + name = f"file_{logical_id:05d}" + found = [] + for subdir in ("A", "B"): + candidate = data_dir / subdir / name + if candidate.exists(): + found.append((subdir, candidate)) + if len(found) != 1: + issues.append( + f"namespace invariant failed for logical_id={logical_id:05d}: expected exactly one file in A/B, found {len(found)}" + ) + continue + actual_locations[logical_id] = found[0][0] + actual_paths[logical_id] = found[0][1] + return actual_locations, actual_paths + + +def validate_rename_invariant(rename_records, actual_locations, issues): + expected_locations = {} + for rec in rename_records: + if rec["rc"] != 0: + continue + dst = rec["dst_rel"] + if "/" not in dst: + continue + expected_locations[rec["logical_id"]] = dst.split("/", 1)[0] + + for logical_id, expected in expected_locations.items(): + actual = actual_locations.get(logical_id) + if actual is None: + continue + if actual != expected: + issues.append( + f"rename invariant failed for logical_id={logical_id:05d}: expected location={expected}, actual={actual}" + ) + + +def validate_data_invariant(io_records, actual_paths, issues): + expected_hash = {} + for rec in io_records: + digest = rec["digest"] + if not digest: + continue + expected_hash[rec["logical_id"]] = digest + + for logical_id, digest in expected_hash.items(): + path = actual_paths.get(logical_id) + if path is None: + continue + actual_digest = sha256_file(path) + if digest != actual_digest: + issues.append( + f"data invariant failed for logical_id={logical_id:05d}: expected digest={digest}, actual digest={actual_digest}" + ) + + +def validate_reset_and_slo(args, reset_records, io_records, rename_records, status, issues): + if not args.expect_reset: + return + + successful_reset_times = [rec["ts_ms"] for rec in reset_records if rec["rc"] == 0] + if not successful_reset_times: + issues.append("expected reset activity but no successful reset trigger was observed") + + phase = status.get("phase") + blocked_requests = to_int(status.get("blocked_requests", "0"), default=-1) + last_errno = to_int(status.get("last_errno", "0"), default=1) + failure_count = to_int(status.get("failure_count", "0"), default=-1) + + if phase is None: + issues.append("missing final reset status file or phase field") + elif phase.lower() != "idle": + issues.append(f"recovery invariant failed: phase={phase}, expected idle") + + if blocked_requests != 0: + issues.append(f"recovery invariant failed: blocked_requests={blocked_requests}, expected 0") + if last_errno != 0: + issues.append(f"recovery invariant failed: last_errno={last_errno}, expected 0") + if failure_count > 0: + issues.append( + f"recovery invariant failed: failure_count={failure_count}, " + "one or more resets failed during the run" + ) + + op_times = [rec["ts_ms"] for rec in io_records] + op_times.extend(rec["ts_ms"] for rec in rename_records if rec["rc"] == 0) + op_times.sort() + + if successful_reset_times and not op_times: + issues.append("recovery SLO failed: no workload completion events were recorded") + return + + slo_ms = args.slo_seconds * 1000 + for ts in successful_reset_times: + idx = bisect.bisect_left(op_times, ts) + if idx >= len(op_times): + issues.append(f"recovery SLO failed: no operation completion observed after reset at ts_ms={ts}") + continue + delta = op_times[idx] - ts + if delta > slo_ms: + issues.append( + f"recovery SLO failed: first post-reset completion at {delta}ms exceeds threshold {slo_ms}ms (reset ts_ms={ts})" + ) + + +def main(): + parser = argparse.ArgumentParser(description="Validate Ceph reset stress artifacts") + parser.add_argument("--data-dir", required=True) + parser.add_argument("--file-count", required=True, type=int) + parser.add_argument("--io-log", required=True) + parser.add_argument("--rename-log", required=True) + parser.add_argument("--reset-log", required=True) + parser.add_argument("--status-final", required=False, default="") + parser.add_argument("--slo-seconds", required=False, type=int, default=30) + parser.add_argument("--expect-reset", action="store_true") + parser.add_argument("--report-json", required=False, default="") + args = parser.parse_args() + + data_dir = Path(args.data_dir) + io_log = Path(args.io_log) + rename_log = Path(args.rename_log) + reset_log = Path(args.reset_log) + status_final = Path(args.status_final) if args.status_final else Path("__missing_status__") + + issues = [] + + if not data_dir.exists(): + issues.append(f"data directory is missing: {data_dir}") + + try: + io_records = parse_io_log(io_log) + rename_records = parse_rename_log(rename_log) + reset_records = parse_reset_log(reset_log) + except Exception as exc: + issues.append(f"log parsing failed: {exc}") + io_records = [] + rename_records = [] + reset_records = [] + + status = parse_status_file(status_final) + + actual_locations, actual_paths = validate_namespace(data_dir, args.file_count, issues) + validate_rename_invariant(rename_records, actual_locations, issues) + validate_data_invariant(io_records, actual_paths, issues) + validate_reset_and_slo(args, reset_records, io_records, rename_records, status, issues) + + report = { + "file_count": args.file_count, + "io_records": len(io_records), + "rename_records": len(rename_records), + "reset_records": len(reset_records), + "expect_reset": args.expect_reset, + "issues": issues, + } + + if args.report_json: + report_path = Path(args.report_json) + report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + + if issues: + print("FAIL: consistency validation found issues") + for issue in issues: + print(f" - {issue}") + raise SystemExit(1) + + print("PASS: consistency validation succeeded") + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/filesystems/idmapped_tmpfile.c b/tools/testing/selftests/filesystems/idmapped_tmpfile.c new file mode 100644 index 000000000000..bc411ab8281e --- /dev/null +++ b/tools/testing/selftests/filesystems/idmapped_tmpfile.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kselftest_harness.h" +#include "wrappers.h" +#include "utils.h" + +/* + * The test mount maps caller-visible ids [0, MAP_RANGE) onto the on-disk range + * [MAP_HOST, MAP_HOST + MAP_RANGE). An id outside [0, MAP_RANGE) therefore has + * no mapping in the mount and is not representable in the filesystem. + */ +#define MAP_HOST 10000 +#define MAP_RANGE 10000 +#define UNMAPPED 50000 + +#ifndef MOUNT_ATTR_IDMAP +#define MOUNT_ATTR_IDMAP 0x00100000 +#endif + +#ifndef __NR_mount_setattr +#define __NR_mount_setattr 442 +#endif + +static inline int sys_mount_setattr(int dfd, const char *path, + unsigned int flags, + struct mount_attr *attr, size_t size) +{ + return syscall(__NR_mount_setattr, dfd, path, flags, attr, size); +} + +/* + * Clone @path into a detached mount idmapped so that caller-visible ids + * [0, MAP_RANGE) map onto the on-disk ids [MAP_HOST, MAP_HOST + MAP_RANGE). + * Returns the mount fd, or -1 if idmapped mounts are not available. + */ +static int idmapped_clone(const char *path) +{ + struct mount_attr attr = { + .attr_set = MOUNT_ATTR_IDMAP, + }; + int fd_tree, userns_fd, ret; + + fd_tree = sys_open_tree(AT_FDCWD, path, + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + if (fd_tree < 0) + return -1; + + userns_fd = get_userns_fd(MAP_HOST, 0, MAP_RANGE); + if (userns_fd < 0) { + close(fd_tree); + return -1; + } + + attr.userns_fd = userns_fd; + ret = sys_mount_setattr(fd_tree, "", AT_EMPTY_PATH, &attr, sizeof(attr)); + close(userns_fd); + if (ret) { + close(fd_tree); + return -1; + } + + return fd_tree; +} + +FIXTURE(idmapped_tmpfile) { + char dir[64]; /* non-idmapped path to the layer directory */ +}; + +FIXTURE_SETUP(idmapped_tmpfile) +{ + /* Private mount namespace so test mounts need no cleanup. */ + ASSERT_EQ(unshare(CLONE_NEWNS), 0); + ASSERT_EQ(sys_mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL), 0); + ASSERT_EQ(sys_mount("tmpfs", "/tmp", "tmpfs", 0, NULL), 0); + + snprintf(self->dir, sizeof(self->dir), "/tmp/d"); + ASSERT_EQ(mkdir(self->dir, 0777), 0); + /* World-writable so an unmapped caller still passes permission(). */ + ASSERT_EQ(chmod(self->dir, 0777), 0); +} + +FIXTURE_TEARDOWN(idmapped_tmpfile) +{ +} + +/* + * A caller whose fsuid/fsgid have no mapping in the idmapped mount must not be + * able to create an O_TMPFILE. Without the check in vfs_tmpfile() the inode + * would be created owned by (uid_t)-1 and could then be linked into the + * namespace. + */ +TEST_F(idmapped_tmpfile, unmapped_caller_is_refused) +{ + int mfd, fd; + + mfd = idmapped_clone(self->dir); + if (mfd < 0) + SKIP(return, "idmapped mounts not supported"); + + /* Become a caller outside the mount's [0, MAP_RANGE) range. */ + setfsgid(UNMAPPED); + setfsuid(UNMAPPED); + ASSERT_EQ(setfsuid(-1), UNMAPPED); + + fd = openat(mfd, ".", O_TMPFILE | O_WRONLY, 0644); + ASSERT_LT(fd, 0); + EXPECT_EQ(errno, EOVERFLOW); + if (fd >= 0) + close(fd); + + EXPECT_EQ(close(mfd), 0); +} + +/* + * A mapped caller can create an O_TMPFILE and link it into the namespace; the + * ownership round-trips through the mount idmap. This is what makes refusing + * the unmapped case above necessary in the first place. + */ +TEST_F(idmapped_tmpfile, mapped_caller_creates_and_links) +{ + char path[PATH_MAX]; + struct stat st; + int mfd, fd; + + mfd = idmapped_clone(self->dir); + if (mfd < 0) + SKIP(return, "idmapped mounts not supported"); + + /* Caller is uid/gid 0, which maps to MAP_HOST through the mount. */ + fd = openat(mfd, ".", O_TMPFILE | O_RDWR, 0600); + ASSERT_GE(fd, 0); + + ASSERT_EQ(fstat(fd, &st), 0); + EXPECT_EQ(st.st_uid, 0); + EXPECT_EQ(st.st_gid, 0); + + /* The tmpfile is linkable: splice it into the directory. */ + ASSERT_EQ(linkat(fd, "", mfd, "linked", AT_EMPTY_PATH), 0); + EXPECT_EQ(close(fd), 0); + + ASSERT_EQ(fstatat(mfd, "linked", &st, 0), 0); + EXPECT_EQ(st.st_uid, 0); + EXPECT_EQ(st.st_gid, 0); + + /* On the underlying, non-idmapped tmpfs it is stored as MAP_HOST. */ + snprintf(path, sizeof(path), "%s/linked", self->dir); + ASSERT_EQ(stat(path, &st), 0); + EXPECT_EQ(st.st_uid, MAP_HOST); + EXPECT_EQ(st.st_gid, MAP_HOST); + + EXPECT_EQ(close(mfd), 0); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/filesystems/overlayfs/.gitignore b/tools/testing/selftests/filesystems/overlayfs/.gitignore index e23a18c8b37f..077f7a128168 100644 --- a/tools/testing/selftests/filesystems/overlayfs/.gitignore +++ b/tools/testing/selftests/filesystems/overlayfs/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only dev_in_maps set_layers_via_fds +idmapped_mounts diff --git a/tools/testing/selftests/filesystems/overlayfs/Makefile b/tools/testing/selftests/filesystems/overlayfs/Makefile index d3ad4a77db9b..b3185f684add 100644 --- a/tools/testing/selftests/filesystems/overlayfs/Makefile +++ b/tools/testing/selftests/filesystems/overlayfs/Makefile @@ -8,7 +8,9 @@ LOCAL_HDRS += ../wrappers.h log.h TEST_GEN_PROGS := dev_in_maps TEST_GEN_PROGS += set_layers_via_fds +TEST_GEN_PROGS += idmapped_mounts include ../../lib.mk $(OUTPUT)/set_layers_via_fds: ../utils.c +$(OUTPUT)/idmapped_mounts: ../utils.c diff --git a/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c b/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c new file mode 100644 index 000000000000..44a75839f4ed --- /dev/null +++ b/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kselftest_harness.h" +#include "../wrappers.h" +#include "../utils.h" + +/* + * An idmapping that maps the mount-visible id range [0, ID_RANGE) onto the + * host/overlay-final id range [ID_HOST, ID_HOST + ID_RANGE). Through such an + * idmapped overlay mount, an overlay-final id of ID_HOST + n is reported as n, + * and an id of n requested through the mount is stored as ID_HOST + n. + */ +#define ID_NS 0 +#define ID_HOST 10000 +#define ID_RANGE 10000 + +/* + * For the composition test the lower layer's on-disk ids live in a + * separate range and are mapped by an idmapped lower layer onto the + * overlay-final range [ID_HOST, ID_HOST + ID_RANGE). + */ +#define LAYER_HOST 20000 + +#ifndef MOUNT_ATTR_IDMAP +#define MOUNT_ATTR_IDMAP 0x00100000 +#endif + +#ifndef __NR_mount_setattr +#define __NR_mount_setattr 442 +#endif + +static inline int sys_mount_setattr(int dfd, const char *path, + unsigned int flags, + struct mount_attr *attr, size_t size) +{ + return syscall(__NR_mount_setattr, dfd, path, flags, attr, size); +} + +static bool ovl_supported(void) +{ + int fd = sys_fsopen("overlay", 0); + + if (fd < 0) + return false; + close(fd); + return true; +} + +/* base/{l,u,w} owned by ID_HOST so they map to ID_NS through the idmap. */ +static int setup_layers(const char *base) +{ + static const char *sub[] = { "", "/l", "/u", "/w" }; + char path[PATH_MAX]; + + for (size_t i = 0; i < ARRAY_SIZE(sub); i++) { + snprintf(path, sizeof(path), "%s%s", base, sub[i]); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + if (i && chown(path, ID_HOST, ID_HOST)) + return -1; + } + return 0; +} + +static int ovl_mount(const char *base, bool nfs_export) +{ + char lower[PATH_MAX], upper[PATH_MAX], work[PATH_MAX]; + int fsfd, ovl; + + snprintf(lower, sizeof(lower), "%s/l", base); + snprintf(upper, sizeof(upper), "%s/u", base); + snprintf(work, sizeof(work), "%s/w", base); + + fsfd = sys_fsopen("overlay", 0); + if (fsfd < 0) + return -1; + + if (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "source", "test", 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "lowerdir", lower, 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", upper, 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", work, 0)) + goto err; + if (nfs_export && + (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "index", "on", 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "nfs_export", "on", 0))) + goto err; + if (sys_fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0)) + goto err; + + ovl = sys_fsmount(fsfd, 0, 0); + close(fsfd); + return ovl; +err: + close(fsfd); + return -1; +} + +/* Idmap the (still detached, not yet visible) overlay mount @mfd. */ +static int ovl_idmap(int mfd) +{ + struct mount_attr attr = { + .attr_set = MOUNT_ATTR_IDMAP, + }; + int ret, userns_fd; + + /* + * get_userns_fd(fs_id, mount_id, range): a file whose filesystem id + * is fs_id + n is shown through the idmapped mount as mount_id + n. + * Here the overlay-final (fs side) range is [ID_HOST, ..) and the + * caller-visible (mount side) range is [ID_NS, ..). + */ + userns_fd = get_userns_fd(ID_HOST, ID_NS, ID_RANGE); + if (userns_fd < 0) + return -1; + + attr.userns_fd = userns_fd; + ret = sys_mount_setattr(mfd, "", AT_EMPTY_PATH, &attr, sizeof(attr)); + close(userns_fd); + return ret; +} + +/* Clone @path into a detached, idmapped mount usable as an overlay layer. */ +static int idmapped_layer_fd(const char *path, int nsid, int hostid, int range) +{ + struct mount_attr attr = { + .attr_set = MOUNT_ATTR_IDMAP, + }; + int fd_tree, userns_fd; + + fd_tree = sys_open_tree(AT_FDCWD, path, + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + if (fd_tree < 0) + return -1; + userns_fd = get_userns_fd(nsid, hostid, range); + if (userns_fd < 0) { + close(fd_tree); + return -1; + } + attr.userns_fd = userns_fd; + if (sys_mount_setattr(fd_tree, "", AT_EMPTY_PATH, &attr, + sizeof(attr))) { + close(userns_fd); + close(fd_tree); + return -1; + } + close(userns_fd); + return fd_tree; +} + +/* Overlay with a layer passed by fd (idmapped) plus a plain upper/work. */ +static int ovl_mount_lower_fd(const char *upper, const char *work, int fd_lower) +{ + int fsfd, ovl; + + fsfd = sys_fsopen("overlay", 0); + if (fsfd < 0) + return -1; + + if (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "source", "test", 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", upper, 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", work, 0) || + sys_fsconfig(fsfd, FSCONFIG_SET_FD, "lowerdir+", NULL, fd_lower) || + sys_fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0)) + goto err; + + ovl = sys_fsmount(fsfd, 0, 0); + close(fsfd); + return ovl; +err: + close(fsfd); + return -1; +} + +/* + * Mount an overlay inside user namespace @u1 (so the overlay sb's s_user_ns is + * not the initial namespace) and idmap that overlay mount with @u2. Runs in a + * child that joins @u1; returns 0 on success. + */ +static int userns_overlay_child(int u1) +{ + struct mount_attr attr = { + .attr_set = MOUNT_ATTR_IDMAP, + }; + struct stat st; + int ovl, u2; + + /* Become root in the overlay sb's user namespace u1. */ + if (!switch_userns(u1, 0, 0, false)) + return fprintf(stderr, "userns: switch_userns: %m\n"), -1; + if (unshare(CLONE_NEWNS) || + sys_mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL)) + return fprintf(stderr, "userns: unshare/slave: %m\n"), -1; + if (sys_mount("tmpfs", "/tmp", "tmpfs", 0, NULL)) + return fprintf(stderr, "userns: mount tmpfs: %m\n"), -1; + if (setup_layers("/tmp/ovl")) + return fprintf(stderr, "userns: setup_layers: %m\n"), -1; + if (mknod("/tmp/ovl/l/file", S_IFREG | 0644, 0) || + chown("/tmp/ovl/l/file", ID_HOST + 5, ID_HOST + 5)) + return fprintf(stderr, "userns: lower file: %m\n"), -1; + + ovl = ovl_mount("/tmp/ovl", false); + if (ovl < 0) + return fprintf(stderr, "userns: ovl_mount: %m\n"), -1; + + /* + * mount_setattr() requires CAP_SYS_ADMIN over the idmap user + * namespace, so it must be a child of u1. Create it now, from + * inside u1. + */ + u2 = get_userns_fd(ID_HOST, ID_NS, ID_RANGE); + if (u2 < 0) + return fprintf(stderr, "userns: get_userns_fd: %m\n"), -1; + attr.userns_fd = u2; + if (sys_mount_setattr(ovl, "", AT_EMPTY_PATH, &attr, sizeof(attr))) + return fprintf(stderr, "userns: mount_setattr: %m\n"), -1; + close(u2); + + if (fstatat(ovl, "file", &st, 0)) + return fprintf(stderr, "userns: fstatat: %m\n"), -1; + if (st.st_uid != ID_NS + 5 || st.st_gid != ID_NS + 5) { + fprintf(stderr, "userns: got %u:%u expected %u:%u\n", + st.st_uid, st.st_gid, ID_NS + 5, ID_NS + 5); + return -1; + } + return 0; +} + +FIXTURE(idmapped_overlay) { + char base[64]; +}; + +FIXTURE_SETUP(idmapped_overlay) +{ + /* Private mount namespace so test mounts need no cleanup. */ + ASSERT_EQ(unshare(CLONE_NEWNS), 0); + ASSERT_EQ(sys_mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL), 0); + + /* tmpfs for the layers so we can chown them to arbitrary ids. */ + ASSERT_EQ(sys_mount("tmpfs", "/tmp", "tmpfs", 0, NULL), 0); + + snprintf(self->base, sizeof(self->base), "/tmp/ovl"); + ASSERT_EQ(setup_layers(self->base), 0); +} + +FIXTURE_TEARDOWN(idmapped_overlay) +{ +} + +/* A file owned by ID_HOST + 5 is reported as ID_NS + 5 through the idmap. */ +TEST_F(idmapped_overlay, getattr) +{ + char path[PATH_MAX]; + struct stat st; + int ovl; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + snprintf(path, sizeof(path), "%s/l/file", self->base); + ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0); + ASSERT_EQ(chown(path, ID_HOST + 5, ID_HOST + 5), 0); + + ovl = ovl_mount(self->base, false); + ASSERT_GE(ovl, 0); + ASSERT_EQ(ovl_idmap(ovl), 0); + + ASSERT_EQ(fstatat(ovl, "file", &st, 0), 0); + EXPECT_EQ(st.st_uid, ID_NS + 5); + EXPECT_EQ(st.st_gid, ID_NS + 5); + + EXPECT_EQ(close(ovl), 0); +} + +/* + * Every creation path initializes the new owner through the mount idmap: + * created as caller id ID_NS, stored on the upper layer as overlay-final + * ID_HOST. Covers ovl_create() (regular file), ovl_mkdir(), ovl_mknod() + * and ovl_symlink() (which share ovl_create_object()), plus the separate + * ovl_tmpfile() path. + */ +TEST_F(idmapped_overlay, create) +{ + static const char *names[] = { "reg", "dir", "fifo", "lnk" }; + char path[PATH_MAX]; + struct stat st; + int ovl, fd; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + ovl = ovl_mount(self->base, false); + ASSERT_GE(ovl, 0); + ASSERT_EQ(ovl_idmap(ovl), 0); + + /* One object per creation operation, all as caller id ID_NS. */ + fd = openat(ovl, "reg", O_CREAT | O_WRONLY | O_EXCL, 0644); + ASSERT_GE(fd, 0); + EXPECT_EQ(close(fd), 0); + ASSERT_EQ(mkdirat(ovl, "dir", 0755), 0); + ASSERT_EQ(mknodat(ovl, "fifo", S_IFIFO | 0644, 0), 0); + ASSERT_EQ(symlinkat("target", ovl, "lnk"), 0); + + for (size_t i = 0; i < ARRAY_SIZE(names); i++) { + /* Reported as ID_NS through the idmapped mount ... */ + ASSERT_EQ(fstatat(ovl, names[i], &st, AT_SYMLINK_NOFOLLOW), 0); + EXPECT_EQ(st.st_uid, ID_NS); + EXPECT_EQ(st.st_gid, ID_NS); + /* ... and stored as ID_HOST on the upper layer. */ + snprintf(path, sizeof(path), "%s/u/%s", self->base, names[i]); + ASSERT_EQ(lstat(path, &st), 0); + EXPECT_EQ(st.st_uid, ID_HOST); + EXPECT_EQ(st.st_gid, ID_HOST); + } + + /* O_TMPFILE goes through the separate ovl_tmpfile() path. */ + fd = openat(ovl, ".", O_TMPFILE | O_WRONLY, 0644); + ASSERT_GE(fd, 0); + /* Inside the mount: caller id ID_NS. */ + ASSERT_EQ(fstat(fd, &st), 0); + EXPECT_EQ(st.st_uid, ID_NS); + EXPECT_EQ(st.st_gid, ID_NS); + /* Link it in so the upper backing file can be inspected too. */ + ASSERT_EQ(linkat(fd, "", ovl, "tmp", AT_EMPTY_PATH), 0); + EXPECT_EQ(close(fd), 0); + snprintf(path, sizeof(path), "%s/u/tmp", self->base); + ASSERT_EQ(lstat(path, &st), 0); + EXPECT_EQ(st.st_uid, ID_HOST); + EXPECT_EQ(st.st_gid, ID_HOST); + + EXPECT_EQ(close(ovl), 0); +} + +/* chown through the idmapped mount round-trips: ID_NS + 5 <-> ID_HOST + 5. */ +TEST_F(idmapped_overlay, chown) +{ + char path[PATH_MAX]; + struct stat st; + int ovl, fd; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + ovl = ovl_mount(self->base, false); + ASSERT_GE(ovl, 0); + ASSERT_EQ(ovl_idmap(ovl), 0); + + fd = openat(ovl, "f", O_CREAT | O_WRONLY | O_EXCL, 0644); + ASSERT_GE(fd, 0); + EXPECT_EQ(close(fd), 0); + + ASSERT_EQ(fchownat(ovl, "f", ID_NS + 5, ID_NS + 5, 0), 0); + + ASSERT_EQ(fstatat(ovl, "f", &st, 0), 0); + EXPECT_EQ(st.st_uid, ID_NS + 5); + EXPECT_EQ(st.st_gid, ID_NS + 5); + + snprintf(path, sizeof(path), "%s/u/f", self->base); + ASSERT_EQ(stat(path, &st), 0); + EXPECT_EQ(st.st_uid, ID_HOST + 5); + EXPECT_EQ(st.st_gid, ID_HOST + 5); + + EXPECT_EQ(close(ovl), 0); +} + +/* + * Composition: an idmapped lower layer underneath an idmapped overlay mount. + * An on-disk id is mapped by the layer idmap into the overlay-final range and + * then by the mount idmap into the caller's range: + * + * on-disk LAYER_HOST+7 --layer--> ID_HOST+7 --mount--> ID_NS+7 + */ +TEST_F(idmapped_overlay, composition) +{ + char lower[PATH_MAX], upper[PATH_MAX], work[PATH_MAX], path[PATH_MAX]; + struct stat st; + int ovl, fd_lower; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + snprintf(lower, sizeof(lower), "%s/l", self->base); + snprintf(upper, sizeof(upper), "%s/u", self->base); + snprintf(work, sizeof(work), "%s/w", self->base); + + /* Put the lower layer's ids in the on-disk [LAYER_HOST, ..) range. */ + ASSERT_EQ(chown(lower, LAYER_HOST, LAYER_HOST), 0); + snprintf(path, sizeof(path), "%s/l/file", self->base); + ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0); + ASSERT_EQ(chown(path, LAYER_HOST + 7, LAYER_HOST + 7), 0); + + /* Idmapped lower: on-disk LAYER_HOST <-> overlay-final ID_HOST. */ + fd_lower = idmapped_layer_fd(lower, LAYER_HOST, ID_HOST, ID_RANGE); + ASSERT_GE(fd_lower, 0); + + ovl = ovl_mount_lower_fd(upper, work, fd_lower); + ASSERT_GE(ovl, 0); + EXPECT_EQ(close(fd_lower), 0); + + /* Idmap the overlay mount: overlay-final ID_HOST <-> caller ID_NS. */ + ASSERT_EQ(ovl_idmap(ovl), 0); + + ASSERT_EQ(fstatat(ovl, "file", &st, 0), 0); + EXPECT_EQ(st.st_uid, ID_NS + 7); + EXPECT_EQ(st.st_gid, ID_NS + 7); + + EXPECT_EQ(close(ovl), 0); +} + +/* An idmapped overlay mount whose sb lives inside a user namespace. */ +TEST_F(idmapped_overlay, userns) +{ + int u1; + pid_t pid; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + /* u1 backs the overlay sb: identity-mapped, but not the init ns. */ + u1 = get_userns_fd(0, 0, 65536); + if (u1 < 0) + SKIP(return, "user namespaces not available"); + + pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + int ret = userns_overlay_child(u1); + + _exit(ret ? EXIT_FAILURE : EXIT_SUCCESS); + } + EXPECT_EQ(wait_for_pid(pid), 0); + + EXPECT_EQ(close(u1), 0); +} + +/* + * An nfs_export overlay can be idmapped, and decodable file handles round-trip + * through the idmapped mount with correctly mapped ownership. Overlay file + * handles encode object identity, not ownership, so the mount idmap does not + * affect them; it only maps the owner reported once a handle is reopened. + */ +TEST_F(idmapped_overlay, nfs_export_handles) +{ + char path[PATH_MAX], mnt[128]; + union { + struct file_handle fh; + char buf[sizeof(struct file_handle) + MAX_HANDLE_SZ]; + } fhu; + struct file_handle *fh = &fhu.fh; + struct stat st; + int ovl, mfd, fd, mount_id; + + if (!ovl_supported()) + SKIP(return, "overlayfs not supported"); + + snprintf(path, sizeof(path), "%s/l/file", self->base); + ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0); + ASSERT_EQ(chown(path, ID_HOST + 7, ID_HOST + 7), 0); + + /* nfs_export=on gives decodable overlay file handles. */ + ovl = ovl_mount(self->base, true); + if (ovl < 0) + SKIP(return, "overlayfs nfs_export not supported"); + ASSERT_EQ(ovl_idmap(ovl), 0); + + /* Attach the idmapped mount so handles can be resolved against it. */ + snprintf(mnt, sizeof(mnt), "%s/mnt", self->base); + ASSERT_EQ(mkdir(mnt, 0755), 0); + ASSERT_EQ(sys_move_mount(ovl, "", AT_FDCWD, mnt, + MOVE_MOUNT_F_EMPTY_PATH), 0); + + snprintf(path, sizeof(path), "%s/file", mnt); + fh->handle_bytes = MAX_HANDLE_SZ; + ASSERT_EQ(name_to_handle_at(AT_FDCWD, path, fh, &mount_id, 0), 0); + + mfd = open(mnt, O_RDONLY | O_DIRECTORY); + ASSERT_GE(mfd, 0); + fd = open_by_handle_at(mfd, fh, O_RDONLY); + EXPECT_EQ(close(mfd), 0); + ASSERT_GE(fd, 0); + + ASSERT_EQ(fstat(fd, &st), 0); + EXPECT_EQ(st.st_uid, ID_NS + 7); + EXPECT_EQ(st.st_gid, ID_NS + 7); + + EXPECT_EQ(close(fd), 0); + EXPECT_EQ(close(ovl), 0); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c index 3c0b93183348..7a293544233d 100644 --- a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c +++ b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c @@ -624,7 +624,7 @@ TEST_F(set_layers_via_fds, set_layers_via_detached_mount_fds) ASSERT_EQ(sys_move_mount(fd_tmpfs, "", -EBADF, "/set_layers_via_fds_tmpfs", MOVE_MOUNT_F_EMPTY_PATH), 0); - fd_tmp = open_tree(fd_tmpfs, "u", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + fd_tmp = sys_open_tree(fd_tmpfs, "u", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(fd_tmp, 0); layer_fds[0] = openat(fd_tmp, "upper", O_CLOEXEC | O_DIRECTORY | O_PATH); @@ -633,25 +633,25 @@ TEST_F(set_layers_via_fds, set_layers_via_detached_mount_fds) layer_fds[1] = openat(fd_tmp, "work", O_CLOEXEC | O_DIRECTORY | O_PATH); ASSERT_GE(layer_fds[1], 0); - layer_fds[2] = open_tree(fd_tmpfs, "l1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[2] = sys_open_tree(fd_tmpfs, "l1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[2], 0); - layer_fds[3] = open_tree(fd_tmpfs, "l2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[3] = sys_open_tree(fd_tmpfs, "l2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[3], 0); - layer_fds[4] = open_tree(fd_tmpfs, "l3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[4] = sys_open_tree(fd_tmpfs, "l3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[4], 0); - layer_fds[5] = open_tree(fd_tmpfs, "l4", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[5] = sys_open_tree(fd_tmpfs, "l4", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[5], 0); - layer_fds[6] = open_tree(fd_tmpfs, "d1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[6] = sys_open_tree(fd_tmpfs, "d1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[6], 0); - layer_fds[7] = open_tree(fd_tmpfs, "d2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[7] = sys_open_tree(fd_tmpfs, "d2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[7], 0); - layer_fds[8] = open_tree(fd_tmpfs, "d3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + layer_fds[8] = sys_open_tree(fd_tmpfs, "d3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); ASSERT_GE(layer_fds[8], 0); ASSERT_EQ(close(fd_tmpfs), 0); diff --git a/tools/testing/selftests/filesystems/ustat_test.c b/tools/testing/selftests/filesystems/ustat_test.c new file mode 100644 index 000000000000..d429fd18d779 --- /dev/null +++ b/tools/testing/selftests/filesystems/ustat_test.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test ustat(2): looking up superblocks by device number. + * + * ustat() resolves a device number to a mounted superblock via + * user_get_super(). Check that the device number of a mounted tmpfs (an + * anonymous device) resolves, that it stops resolving once the filesystem + * is unmounted and that bogus device numbers report EINVAL. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../kselftest_harness.h" + +/* struct ustat is not exported through UAPI, mirror include/linux/types.h. */ +struct ustat_buf { + int f_tfree; + unsigned long f_tinode; + char f_fname[6]; + char f_fpack[6]; + /* slack in case an architecture lays the struct out differently */ + char pad[64]; +}; + +#ifdef __NR_ustat + +/* + * The kernel decodes @dev with new_decode_dev(), which matches the low 32 + * bits of the st_dev encoding stat(2) returns for any major below 4096. + */ +static int sys_ustat(unsigned int dev, struct ustat_buf *buf) +{ + return syscall(__NR_ustat, dev, buf); +} + +static int write_string(const char *path, const char *string) +{ + ssize_t len = strlen(string); + int fd; + + fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + if (write(fd, string, len) != len) { + close(fd); + return -1; + } + return close(fd); +} + +/* Enter namespaces in which mounting a tmpfs instance is allowed. */ +static int setup_namespaces(void) +{ + uid_t uid = getuid(); + gid_t gid = getgid(); + char map[64]; + + if (unshare(CLONE_NEWNS | (uid ? CLONE_NEWUSER : 0))) + return -1; + + if (uid) { + if (write_string("/proc/self/setgroups", "deny")) + return -1; + snprintf(map, sizeof(map), "0 %d 1", uid); + if (write_string("/proc/self/uid_map", map)) + return -1; + snprintf(map, sizeof(map), "0 %d 1", gid); + if (write_string("/proc/self/gid_map", map)) + return -1; + } + + return mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL); +} + +TEST(resolves_mounted_superblock) +{ + char dir[] = "/tmp/ustat_test.XXXXXX"; + struct ustat_buf ub; + struct stat st; + + ASSERT_NE(NULL, mkdtemp(dir)); + + if (setup_namespaces()) { + rmdir(dir); + SKIP(return, "cannot set up namespaces: %s", strerror(errno)); + } + + ASSERT_EQ(0, mount("ustat_test", dir, "tmpfs", 0, NULL)); + ASSERT_EQ(0, stat(dir, &st)); + + memset(&ub, 0xff, sizeof(ub)); + ASSERT_EQ(0, sys_ustat(st.st_dev, &ub)) + TH_LOG("ustat(%u): %s", (unsigned int)st.st_dev, + strerror(errno)); + + ASSERT_EQ(0, umount(dir)); + + /* The unmount removed the superblock, the device is gone. */ + ASSERT_EQ(-1, sys_ustat(st.st_dev, &ub)); + ASSERT_EQ(EINVAL, errno); + + rmdir(dir); +} + +TEST(bogus_device_numbers) +{ + struct ustat_buf ub; + + ASSERT_EQ(-1, sys_ustat(0, &ub)); + ASSERT_EQ(EINVAL, errno); + + /* major 4095, minor 1048575: nothing plausible lives there */ + ASSERT_EQ(-1, sys_ustat((0xfffu << 8) | 0xffu | (0xfff00u << 12), &ub)); + ASSERT_EQ(EINVAL, errno); +} + +#else /* !__NR_ustat */ + +TEST(unsupported) +{ + SKIP(return, "ustat(2) is not available on this architecture"); +} + +#endif /* __NR_ustat */ + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc b/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc index e71cc3ad0bdf..6d00d3c0f493 100644 --- a/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc +++ b/tools/testing/selftests/ftrace/test.d/00basic/test_ownership.tc @@ -6,7 +6,7 @@ original_group=`stat -c "%g" .` original_owner=`stat -c "%u" .` -local mount_point=$(get_mount_point) +mount_point=$(get_mount_point) mount_options=$(get_mnt_options "$mount_point") diff --git a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc index 8e905d4fe6dd..f985ff391463 100644 --- a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc +++ b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc @@ -36,15 +36,23 @@ make_str() { data=`printf -- 'X%.0s' $(seq $cnt)` - printf "${val}${data}" + # Return escape-sequence text (e.g. "\003\000..."); the caller + # converts to binary. Shell command substitution strips NUL bytes, + # so the binary form cannot survive being captured into a variable. + printf '%s' "${val}${data}" } write_buffer() { id=$1 size=$2 - # write the string into the raw marker - make_str $id $size > trace_marker_raw + str=`make_str $id $size` + len=`printf "$str" | wc -c` + # Pipe through dd to ensure a single atomic write() syscall + # on architectures with 64K pages, where shell's printf builtin + # uses stdio buffering which may split the output into multiple + # writes. + printf "$str" | dd of=trace_marker_raw bs=$len iflag=fullblock } diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc new file mode 100644 index 000000000000..bf71368c31a4 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc @@ -0,0 +1,51 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: BTF event with typecast and percpu access +# requires: dynamic_events "this_cpu_read()":README "[(structname[,field])][->field[->field|.field...]]":README + +# Check if the sample module is loaded +if ! lsmod | grep -q trace_events_sample; then + modprobe trace-events-sample || exit_unresolved +fi + +echo 0 > events/enable +echo > dynamic_events + +# The sample_timer_cb(struct timer_list *t) is called. +# We want to check (STRUCT,FIELD)VAR typecast and this_cpu_read() access. +# (foo_timer_data,timer)t converts t to struct foo_timer_data * using container_of. +# data->counter is a per-cpu pointer to int. +# this_cpu_read(data->counter) should give the value of the counter. + +echo 'f:mysample/myevent sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events + +echo 1 > events/mysample/myevent/enable +echo 1 > events/sample-trace/foo_timer_fn/enable + +sleep 2 + +echo 0 > events/mysample/myevent/enable +echo 0 > events/sample-trace/foo_timer_fn/enable + +# Compare the values. +MATCH=0 +while read line; do + if echo $line | grep -q "foo_timer_fn:"; then + NAME=`echo $line | sed 's/.*name=\([^ ]*\) .*/\1/'` + COUNT=`echo $line | sed 's/.*count=\([^ ]*\).*/\1/'` + if grep -q "myevent:.*name=\"${NAME}\" count=$COUNT" trace; then + MATCH=$((MATCH+1)) + fi + fi +done < trace + +if [ $MATCH -eq 0 ]; then + echo "No matching events found" + exit_fail +fi + +# Clean up +echo 0 > events/mysample/myevent/enable +echo 0 > events/sample-trace/foo_timer_fn/enable +echo > dynamic_events +clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc b/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc new file mode 100644 index 000000000000..dd5552727054 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc @@ -0,0 +1,103 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: BTF typecast and percpu access syntax validation +# requires: dynamic_events "this_cpu_read()":README "[(structname[,field])][->field[->field|.field...]]":README + +KPROBES= +FPROBES= + +if grep -qF "p[:[/][]] []" README ; then + KPROBES=yes +fi +if grep -qF "f[:[/][]] [%return] []" README ; then + FPROBES=yes +fi + +if [ -z "$KPROBES" -a -z "$FPROBES" ] ; then + exit_unsupported +fi + +echo 0 > events/enable +echo > dynamic_events + +# Load trace-events-sample module if available to have per-CPU counter structure defined +if ! lsmod | grep -q trace_events_sample; then + modprobe trace-events-sample || exit_unresolved +fi + +if [ "$FPROBES" ] ; then + # 1. Test basic typecast on fprobe + echo 'f:fpevent1 vfs_read name=(file)file->f_path.dentry->d_name.name:string' >> dynamic_events + # 2. Test parenthesized typecast target on fprobe + echo 'f:fpevent2 vfs_read name=(file)(file)->f_path.dentry->d_name.name:string' >> dynamic_events + # 3. Test nested typecasts on fprobe + echo 'f:fpevent3 vfs_read name=(dentry)((file)file->f_path.dentry)->d_name.name:string' >> dynamic_events + # 4. Test container_of-style typecast with field option on fprobe + echo 'f:fpevent4 vfs_read name=(file,f_path)file->f_mode' >> dynamic_events + # 5. Test typecast on return value on fprobe + echo 'f:fpevent5 vfs_read%return name=(file)$retval->f_path.dentry->d_name.name:string' >> dynamic_events + # 6. Test $current variable support on fprobe + echo 'f:fpevent6 vfs_read pid=$current->pid' >> dynamic_events + echo 'f:fpevent7 vfs_read pid=(task_struct)$current->pid' >> dynamic_events + echo 'f:fpevent8 vfs_read pid=(task_struct,group_leader)$current->pid' >> dynamic_events + + # Test this_cpu_read and this_cpu_ptr on fprobe + echo 'f:fpevent9 sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events + echo 'f:fpevent10 sample_timer_cb ptr=this_cpu_ptr((foo_timer_data,timer)t->counter)' >> dynamic_events +fi + +if [ "$KPROBES" ] ; then + # 7. Test basic typecast on kprobe + echo 'p:kpevent1 vfs_read name=(file)file->f_path.dentry->d_name.name:string' >> dynamic_events + # 8. Test parenthesized typecast target on kprobe + echo 'p:kpevent2 vfs_read name=(file)(file)->f_path.dentry->d_name.name:string' >> dynamic_events + # 9. Test nested typecasts on kprobe + echo 'p:kpevent3 vfs_read name=(dentry)((file)file->f_path.dentry)->d_name.name:string' >> dynamic_events + # 10. Test container_of-style typecast with field option on kprobe + echo 'p:kpevent4 vfs_read name=(file,f_path)file->f_mode' >> dynamic_events + # 11. Test typecast on return value on kretprobe + echo 'r:kpevent5 vfs_read name=(file)$retval->f_path.dentry->d_name.name:string' >> dynamic_events + # 12. Test $current variable support on kprobe + echo 'p:kpevent6 vfs_read pid=$current->pid' >> dynamic_events + echo 'p:kpevent7 vfs_read pid=(task_struct)$current->pid' >> dynamic_events + echo 'p:kpevent8 vfs_read pid=(task_struct,group_leader)$current->pid' >> dynamic_events + + # Test this_cpu_read and this_cpu_ptr on kprobe + echo 'p:kpevent9 sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events + echo 'p:kpevent10 sample_timer_cb ptr=this_cpu_ptr((foo_timer_data,timer)t->counter)' >> dynamic_events +fi + +# Verify the events exist in dynamic_events +if [ "$FPROBES" ] ; then + grep -q "fpevent1 " dynamic_events + grep -q "fpevent2 " dynamic_events + grep -q "fpevent3 " dynamic_events + grep -q "fpevent4 " dynamic_events + grep -q "fpevent5 " dynamic_events + grep -q "fpevent6 " dynamic_events + grep -q "fpevent7 " dynamic_events + grep -q "fpevent8 " dynamic_events + if lsmod | grep -q trace_events_sample; then + grep -q "fpevent9 " dynamic_events + grep -q "fpevent10 " dynamic_events + fi +fi + +if [ "$KPROBES" ] ; then + grep -q "kpevent1 " dynamic_events + grep -q "kpevent2 " dynamic_events + grep -q "kpevent3 " dynamic_events + grep -q "kpevent4 " dynamic_events + grep -q "kpevent5 " dynamic_events + grep -q "kpevent6 " dynamic_events + grep -q "kpevent7 " dynamic_events + grep -q "kpevent8 " dynamic_events + if lsmod | grep -q trace_events_sample; then + grep -q "kpevent9 " dynamic_events + grep -q "kpevent10 " dynamic_events + fi +fi + +# Clean up +echo > dynamic_events +clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc index 2a680c086047..c2e3f9d19f13 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc @@ -10,7 +10,7 @@ check_error() { # command-with-error-pos-by-^ check_error 'e ^a.' # NO_EVENT_INFO check_error 'e ^.b' # NO_EVENT_INFO check_error 'e ^a.b' # BAD_ATTACH_EVENT -check_error 'e syscalls/sys_enter_openat ^foo' # BAD_ATTACH_ARG +check_error 'e syscalls/sys_enter_openat ^foo' # NO_EVENT_FIELD check_error 'e:^/bar syscalls/sys_enter_openat' # NO_GROUP_NAME check_error 'e:^12345678901234567890123456789012345678901234567890123456789012345/bar syscalls/sys_enter_openat' # GROUP_TOO_LONG @@ -19,11 +19,19 @@ check_error 'e:^ syscalls/sys_enter_openat' # NO_EVENT_NAME check_error 'e:foo/^12345678901234567890123456789012345678901234567890123456789012345 syscalls/sys_enter_openat' # EVENT_TOO_LONG check_error 'e:foo/^bar.1 syscalls/sys_enter_openat' # BAD_EVENT_NAME -check_error 'e:foo/bar syscalls/sys_enter_openat arg=^dfd' # BAD_FETCH_ARG check_error 'e:foo/bar syscalls/sys_enter_openat arg=^$foo' # BAD_ATTACH_ARG +check_error 'e:foo/bar syscalls/sys_enter_openat arg=^COMM' # NO_EVENT_FIELD +if grep -q "\$current.*" README; then + check_error 'e:foo/bar syscalls/sys_enter_openat arg=^current' # NO_EVENT_FIELD +fi + if grep -q '\..*\[if \]' README; then check_error 'e:foo/bar syscalls/sys_enter_openat if ^' # NO_EP_FILTER fi +if grep -q 'this_cpu_read()' README; then + check_error 'e:foo/bar syscalls/sys_enter_openat arg=^this_cpu_read(file)' # NOSUP_PERCPU +fi + exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc index fee479295e2f..e9d7e6919c7f 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc @@ -112,6 +112,18 @@ check_error 'f vfs_read%return $retval->^foo' # NO_PTR_STRCT check_error 'f vfs_read file->^foo' # NO_BTF_FIELD check_error 'f vfs_read file^-.foo' # BAD_HYPHEN check_error 'f vfs_read ^file:string' # BAD_TYPE4STR +if grep -qF "[(structname" README ; then +check_error 'f vfs_read arg1=(task_struct)file^' # TYPECAST_REQ_FIELD +check_error 'f vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a' # TOO_MANY_NESTED +check_error 'f vfs_read arg1=(task_struct,^in_execve)file->comm' # TYPECAST_NOT_ALIGNED +check_error 'f vfs_read arg1=(task_struct,^foo_bar)file->pid' # NO_BTF_FIELD +check_error 'f vfs_read arg1=(^task_struct1234)file->pid' # NO_PTR_STRCT +check_error 'f vfs_read arg1=(task_struct,se^->group_node)file->comm' # TYPECAST_BAD_ARROW +check_error 'f vfs_read arg1=(task_struct,^->pid)file->comm' # NO_BTF_FIELD +check_error 'f vfs_read arg1=(task_struct,^.pid)file->comm' # NO_BTF_FIELD +check_error 'f vfs_read arg1=(task_struct,^.)file->comm' # NO_BTF_FIELD +check_error 'f vfs_read arg1=(task_struct)^@symbol+10->comm' # TYPECAST_SYM_OFFSET +fi fi else diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc index 8f1c58f0c239..21ce8414459f 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc @@ -115,6 +115,18 @@ check_error 'p vfs_read+20 ^$arg*' # NOFENTRY_ARGS check_error 'p vfs_read ^hoge' # NO_BTFARG check_error 'p kfree ^$arg10' # NO_BTFARG (exceed the number of parameters) check_error 'r kfree ^$retval' # NO_RETVAL +if grep -qF "[(structname" README ; then +check_error 'p vfs_read arg1=(task_struct)file^' # TYPECAST_REQ_FIELD +check_error 'p vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a' # TOO_MANY_NESTED +check_error 'p vfs_read arg1=(task_struct,^in_execve)file->comm' # TYPECAST_NOT_ALIGNED +check_error 'p vfs_read arg1=(task_struct,^foo_bar)file->pid' # NO_BTF_FIELD +check_error 'p vfs_read arg1=(^task_struct1234)file->pid' # NO_PTR_STRCT +check_error 'p vfs_read arg1=(task_struct,se^->group_node)file->comm' # TYPECAST_BAD_ARROW +check_error 'p vfs_read arg1=(task_struct,^->pid)file->comm' # NO_BTF_FIELD +check_error 'p vfs_read arg1=(task_struct,^.pid)file->comm' # NO_BTF_FIELD +check_error 'p vfs_read arg1=(task_struct,^.)file->comm' # NO_BTF_FIELD +check_error 'p vfs_read arg1=(task_struct)^@symbol+10->comm' # TYPECAST_SYM_OFFSET +fi else check_error 'p vfs_read ^$arg*' # NOSUP_BTFARG fi diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc index c817158b99db..e12dc967ec76 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc @@ -28,4 +28,9 @@ if grep -q ".*symstr.*" README; then check_error 'p /bin/sh:10 $stack0:^symstr' # BAD_TYPE fi +# $current is not supported by uprobe +if grep -q "\$current.*" README; then +check_error 'p /bin/sh:10 ^$current:u8' # BAD_VAR +fi + exit 0 diff --git a/tools/testing/selftests/hid/Makefile b/tools/testing/selftests/hid/Makefile index 96071b4800e8..2f423de83147 100644 --- a/tools/testing/selftests/hid/Makefile +++ b/tools/testing/selftests/hid/Makefile @@ -24,7 +24,7 @@ CXX ?= $(CROSS_COMPILE)g++ HOSTPKG_CONFIG := pkg-config -CFLAGS += -g -O0 -rdynamic -Wall -Werror -I$(OUTPUT) +CFLAGS += -g -O0 -rdynamic -Wall -Werror -I$(OUTPUT) $(KHDR_INCLUDES) CFLAGS += -I$(OUTPUT)/tools/include LDLIBS += -lelf -lz -lrt -lpthread diff --git a/tools/testing/selftests/hid/hid_bpf.c b/tools/testing/selftests/hid/hid_bpf.c index 1e979fb3542b..b851339308c2 100644 --- a/tools/testing/selftests/hid/hid_bpf.c +++ b/tools/testing/selftests/hid/hid_bpf.c @@ -86,6 +86,20 @@ static void load_programs(const struct test_program programs[], self->skel = hid__open(); ASSERT_OK_PTR(self->skel) TEARDOWN_LOG("Error while calling hid__open"); + /* + * Disable all struct_ops maps by default so libbpf does not autoload + * programs referenced by maps that are unrelated to the current test. + */ + bpf_object__for_each_map(iter_map, *self->skel->skeleton->obj) { + if (bpf_map__type(iter_map) == BPF_MAP_TYPE_STRUCT_OPS) { + err = bpf_map__set_autocreate(iter_map, false); + ASSERT_OK(err) TH_LOG("can not disable struct_ops map '%s'", + bpf_map__name(iter_map)); + } + + bpf_map__set_autoattach(iter_map, false); + } + for (int i = 0; i < progs_count; i++) { struct bpf_program *prog; struct bpf_map *map; @@ -102,6 +116,10 @@ static void load_programs(const struct test_program programs[], ASSERT_OK_PTR(map) TH_LOG("can not find struct_ops by name '%s'", programs[i].name + 4); + err = bpf_map__set_autocreate(map, true); + ASSERT_OK(err) TH_LOG("can not enable struct_ops map '%s'", + programs[i].name + 4); + /* hid_id is the first field of struct hid_bpf_ops */ ops_hid_id = bpf_map__initial_value(map, NULL); ASSERT_OK_PTR(ops_hid_id) TH_LOG("unable to retrieve struct_ops data"); @@ -109,13 +127,6 @@ static void load_programs(const struct test_program programs[], *ops_hid_id = self->hid.hid_id; } - /* we disable the auto-attach feature of all maps because we - * only want the tested one to be manually attached in the next - * call to bpf_map__attach_struct_ops() - */ - bpf_object__for_each_map(iter_map, *self->skel->skeleton->obj) - bpf_map__set_autoattach(iter_map, false); - err = hid__load(self->skel); ASSERT_OK(err) TH_LOG("hid_skel_load failed: %d", err); @@ -887,6 +898,17 @@ TEST_F(hid_bpf, test_rdesc_fixup) ASSERT_EQ(rpt_desc.value[4], 0x42); } +TEST_F(hid_bpf, test_rdesc_fixup_get_data_overflow) +{ + const struct test_program progs[] = { + { .name = "hid_rdesc_fixup_get_data_overflow" }, + }; + + LOAD_PROGRAMS(progs); + + ASSERT_EQ(self->skel->bss->get_data_overflow_check, 1); +} + static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) { diff --git a/tools/testing/selftests/hid/progs/hid.c b/tools/testing/selftests/hid/progs/hid.c index 5ecc845ef792..b21fbb13c926 100644 --- a/tools/testing/selftests/hid/progs/hid.c +++ b/tools/testing/selftests/hid/progs/hid.c @@ -13,6 +13,7 @@ struct attach_prog_args { __u64 callback_check = 52; __u64 callback2_check = 52; +__u64 get_data_overflow_check; SEC("?struct_ops/hid_device_event") int BPF_PROG(hid_first_event, struct hid_bpf_ctx *hid_ctx, enum hid_report_type type) @@ -240,6 +241,20 @@ struct hid_bpf_ops rdesc_fixup = { .hid_rdesc_fixup = (void *)hid_rdesc_fixup, }; +SEC("?struct_ops.s/hid_rdesc_fixup") +int BPF_PROG(hid_rdesc_fixup_get_data_overflow, struct hid_bpf_ctx *hid_ctx) +{ + if (!hid_bpf_get_data(hid_ctx, 2 /* offset */, ~0ULL /* size */)) + get_data_overflow_check = 1; + + return 0; +} + +SEC(".struct_ops.link") +struct hid_bpf_ops rdesc_fixup_get_data_overflow = { + .hid_rdesc_fixup = (void *)hid_rdesc_fixup_get_data_overflow, +}; + SEC("?struct_ops/hid_device_event") int BPF_PROG(hid_test_insert1, struct hid_bpf_ctx *hid_ctx, enum hid_report_type type) { diff --git a/tools/testing/selftests/hid/tests/test_multitouch.py b/tools/testing/selftests/hid/tests/test_multitouch.py index fa4fb2054bd4..7897340118b4 100644 --- a/tools/testing/selftests/hid/tests/test_multitouch.py +++ b/tools/testing/selftests/hid/tests/test_multitouch.py @@ -513,6 +513,79 @@ def match_evdev_rule(self, application, evdev): return absinfo is not None and absinfo.resolution == 3 +class MinWin8TSParallelBigContactMax(Digitizer): + """A parallel Win8 touchscreen that advertises a ContactCountMaximum much + larger than the number of contacts it actually reports. + + Such firmware makes the driver allocate that many input slots (up to 255) + while the input report only carries a few contacts. This is what used to + drive the per-slot bit operations on mt_io_flags out of bounds. The number + of contacts a HID report can describe is limited by the descriptor size, + so a large ContactCountMaximum can only be expressed this way, decoupled + from the number of finger collections.""" + + def __init__(self, n_fingers=5, contact_max=250): + self.phys_max = 120, 90 + rdesc_finger_str = f""" + Usage Page (Digitizers) + Usage (Finger) + Collection (Logical) + Report Size (1) + Report Count (1) + Logical Minimum (0) + Logical Maximum (1) + Usage (Tip Switch) + Input (Data,Var,Abs) + Report Size (7) + Logical Maximum (127) + Input (Cnst,Var,Abs) + Report Size (8) + Logical Maximum (255) + Usage (Contact Id) + Input (Data,Var,Abs) + Report Size (16) + Unit Exponent (-1) + Unit (SILinear: cm) + Logical Maximum (4095) + Physical Minimum (0) + Physical Maximum ({self.phys_max[0]}) + Usage Page (Generic Desktop) + Usage (X) + Input (Data,Var,Abs) + Physical Maximum ({self.phys_max[1]}) + Usage (Y) + Input (Data,Var,Abs) + End Collection +""" + rdesc_str = f""" + Usage Page (Digitizers) + Usage (Touch Screen) + Collection (Application) + Report ID (1) + {rdesc_finger_str * n_fingers} + Unit Exponent (-4) + Unit (SILinear: s) + Logical Maximum (65535) + Physical Maximum (65535) + Usage Page (Digitizers) + Usage (Scan Time) + Input (Data,Var,Abs) + Report Size (8) + Logical Maximum (255) + Usage (Contact Count) + Input (Data,Var,Abs) + Report ID (2) + Logical Maximum ({contact_max}) + Usage (Contact Max) + Feature (Data,Var,Abs) + End Collection + {Digitizer.msCertificationBlob(68)} +""" + super().__init__( + f"uhid test parallel big contact max {contact_max}", rdesc_str + ) + + class BaseTest: class TestMultitouch(base.BaseTestCase.TestUhid): kernel_modules = [KERNEL_MODULE] @@ -1735,6 +1808,47 @@ def create_device(self): return MinWin8TSParallel(10) +class TestMinWin8TSParallelBigContactMax(base.BaseTestCase.TestUhid): + """Regression test for the out-of-bounds bit operations on + struct mt_device.mt_io_flags. + + A Win8 touchscreen may advertise a ContactCountMaximum much larger than + the number of contacts it reports. The driver used to keep the per-slot + active state in the bits of a single unsigned long while indexing + set_bit()/clear_bit() by the slot number, so such a device drove those bit + operations out of bounds. The sticky-fingers release timer made it fatal: + mt_release_contacts() cleared one bit per slot, overwrote the adjacent + struct mt_device members and panicked the kernel. + + Send a single contact, let the 100ms sticky-fingers timer release it, and + check that the kernel reports the release cleanly instead of crashing.""" + + kernel_modules = [KERNEL_MODULE] + + def create_device(self): + return MinWin8TSParallelBigContactMax() + + def test_sticky_fingers_release_big_contact_max(self): + uhdev = self.uhdev + evdev = uhdev.get_evdev() + + assert evdev.num_slots == uhdev.max_contacts + + t0 = Touch(1, 5, 10) + r = uhdev.event([t0]) + events = uhdev.next_sync_events() + self.debug_reports(r, uhdev, events) + assert evdev.slots[0][libevdev.EV_ABS.ABS_MT_TRACKING_ID] == 0 + + # do not release the contact; the sticky-fingers timer must do it + # after 100ms, which is where the out-of-bounds release used to hit + time.sleep(0.2) + events = uhdev.next_sync_events() + self.debug_reports(r, uhdev, events) + assert libevdev.InputEvent(libevdev.EV_KEY.BTN_TOUCH, 0) in events + assert evdev.slots[0][libevdev.EV_ABS.ABS_MT_TRACKING_ID] == -1 + + class TestMinWin8TSHybrid(BaseTest.TestWin8Multitouch): def create_device(self): return MinWin8TSHybrid() diff --git a/tools/testing/selftests/kho/vmtest.sh b/tools/testing/selftests/kho/vmtest.sh index 49fdac8e8b15..0014bd76e88d 100755 --- a/tools/testing/selftests/kho/vmtest.sh +++ b/tools/testing/selftests/kho/vmtest.sh @@ -59,10 +59,14 @@ function build_kernel() { tee "$kconfig" > "$kho_config" < #include /* alt_stfle_fac_list[16] + stfle_fac_list[16] */ @@ -19,6 +20,11 @@ extern u64 stfl_doublewords[NB_STFL_DOUBLEWORDS]; extern bool stfle_flag; +static inline bool clear_bit_inv(unsigned long nr, unsigned long *ptr) +{ + return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr); +} + static inline bool test_bit_inv(unsigned long nr, const unsigned long *ptr) { return test_bit(nr ^ (BITS_PER_LONG - 1), ptr); diff --git a/tools/testing/selftests/kvm/include/x86/pmu.h b/tools/testing/selftests/kvm/include/x86/pmu.h index 98537cc8840d..608ed83d7c6a 100644 --- a/tools/testing/selftests/kvm/include/x86/pmu.h +++ b/tools/testing/selftests/kvm/include/x86/pmu.h @@ -38,6 +38,12 @@ #define ARCH_PERFMON_EVENTSEL_INV BIT_ULL(23) #define ARCH_PERFMON_EVENTSEL_CMASK GENMASK_ULL(31, 24) +/* + * These are AMD-specific bits. + */ +#define AMD64_EVENTSEL_GUESTONLY BIT_ULL(40) +#define AMD64_EVENTSEL_HOSTONLY BIT_ULL(41) + /* RDPMC control flags, Intel only. */ #define INTEL_RDPMC_METRICS BIT_ULL(29) #define INTEL_RDPMC_FIXED BIT_ULL(30) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 513e4a1075fa..7d3a27bc0d84 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1226,6 +1226,8 @@ struct idt_entry { void vm_install_exception_handler(struct kvm_vm *vm, int vector, void (*handler)(struct ex_regs *)); +gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); + /* * Exception fixup morphs #DE to an arbitrary magic vector so that '0' can be * used to signal "no expcetion". @@ -1392,6 +1394,14 @@ static inline bool kvm_is_pmu_enabled(void) return get_kvm_param_bool("enable_pmu"); } +static inline bool kvm_is_mediated_pmu_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("enable_mediated_pmu"); + + return get_kvm_amd_param_bool("enable_mediated_pmu"); +} + static inline bool kvm_is_forced_emulation_enabled(void) { return !!get_kvm_param_integer("force_emulation_prefix"); diff --git a/tools/testing/selftests/kvm/include/x86/svm_util.h b/tools/testing/selftests/kvm/include/x86/svm_util.h index 6c013eb838be..c201c30485e7 100644 --- a/tools/testing/selftests/kvm/include/x86/svm_util.h +++ b/tools/testing/selftests/kvm/include/x86/svm_util.h @@ -28,6 +28,9 @@ struct svm_test_data { void *msr_hva; u64 msr_gpa; + /* Stack */ + void *stack; /* gva */ + /* NPT */ u64 ncr3_gpa; }; @@ -57,7 +60,7 @@ static inline void vmmcall(void) ) struct svm_test_data *vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva); -void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp); +void generic_svm_setup(struct svm_test_data *svm, void *guest_rip); void run_guest(struct vmcb *vmcb, u64 vmcb_gpa); static inline bool kvm_cpu_has_npt(void) diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 90fffaf91595..4bcfd60e3aec 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -524,6 +524,8 @@ struct vmx_pages { u64 apic_access_gpa; void *apic_access; + void *stack; + u64 eptp_gpa; }; @@ -552,7 +554,7 @@ union vmx_ctrl_msr { struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); bool prepare_for_vmx_operation(struct vmx_pages *vmx); -void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp); +void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); bool load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index 61cf952cd2dc..e19e8b5a09c5 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -30,21 +30,15 @@ __asm__( " ud2;" ); -#define L2_GUEST_STACK_SIZE 64 - static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - unsigned long *rsp; - GUEST_ASSERT(vmx->vmcs_gpa); GUEST_ASSERT(prepare_for_vmx_operation(vmx)); GUEST_ASSERT(load_vmcs(vmx)); GUEST_ASSERT(ept_1g_pages_supported()); - rsp = &l2_guest_stack[L2_GUEST_STACK_SIZE - 1]; - *rsp = vcpu_id; - prepare_vmcs(vmx, memstress_l2_guest_entry, rsp); + *(u64 *)vmx->stack = vcpu_id; + prepare_vmcs(vmx, memstress_l2_guest_entry); GUEST_ASSERT(!vmlaunch()); GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); @@ -53,13 +47,8 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) static void l1_svm_code(struct svm_test_data *svm, u64 vcpu_id) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - unsigned long *rsp; - - - rsp = &l2_guest_stack[L2_GUEST_STACK_SIZE - 1]; - *rsp = vcpu_id; - generic_svm_setup(svm, memstress_l2_guest_entry, rsp); + *(u64 *)svm->stack = vcpu_id; + generic_svm_setup(svm, memstress_l2_guest_entry); run_guest(svm->vmcb, svm->vmcb_gpa); GUEST_ASSERT_EQ(svm->vmcb->control.exit_code, SVM_EXIT_VMMCALL); diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index 4ca48de7a926..ef56dcefe011 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -778,6 +778,30 @@ void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) REPORT_GUEST_ASSERT(uc); } +gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages) +{ + int size = nr_pages * getpagesize(); + gva_t stack_gva; + + stack_gva = __vm_alloc(vm, size, DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA); + stack_gva += size; + + /* + * Align stack to match calling sequence requirements in section "The + * Stack Frame" of the System V ABI AMD64 Architecture Processor + * Supplement, which requires the value (%rsp + 8) to be a multiple of + * 16 when control is transferred to the function entry point. + * + * If this code is ever used to launch a vCPU with 32-bit entry point it + * may need to subtract 4 bytes instead of 8 bytes. + */ + TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE), + "__vm_alloc() did not provide a page-aligned address"); + stack_gva -= 8; + + return stack_gva; +} + void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) { int r; @@ -820,27 +844,8 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) { struct kvm_mp_state mp_state; struct kvm_regs regs; - gva_t stack_gva; struct kvm_vcpu *vcpu; - stack_gva = __vm_alloc(vm, DEFAULT_STACK_PGS * getpagesize(), - DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA); - - stack_gva += DEFAULT_STACK_PGS * getpagesize(); - - /* - * Align stack to match calling sequence requirements in section "The - * Stack Frame" of the System V ABI AMD64 Architecture Processor - * Supplement, which requires the value (%rsp + 8) to be a multiple of - * 16 when control is transferred to the function entry point. - * - * If this code is ever used to launch a vCPU with 32-bit entry point it - * may need to subtract 4 bytes instead of 8 bytes. - */ - TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE), - "__vm_alloc() did not provide a page-aligned address"); - stack_gva -= 8; - vcpu = __vm_vcpu_add(vm, vcpu_id); vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid()); vcpu_init_sregs(vm, vcpu); @@ -849,7 +854,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) /* Setup guest general purpose registers */ vcpu_regs_get(vcpu, ®s); regs.rflags = regs.rflags | X86_EFLAGS_FIXED; - regs.rsp = stack_gva; + regs.rsp = vm_alloc_stack(vm, DEFAULT_STACK_PGS); vcpu_regs_set(vcpu, ®s); /* Setup the MP state */ diff --git a/tools/testing/selftests/kvm/lib/x86/svm.c b/tools/testing/selftests/kvm/lib/x86/svm.c index 3b01605ab016..1445b890986f 100644 --- a/tools/testing/selftests/kvm/lib/x86/svm.c +++ b/tools/testing/selftests/kvm/lib/x86/svm.c @@ -46,6 +46,8 @@ vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva) svm->msr_gpa = addr_gva2gpa(vm, (uintptr_t)svm->msr); memset(svm->msr_hva, 0, getpagesize()); + svm->stack = (void *)vm_alloc_stack(vm, 1); + if (vm->stage2_mmu.pgd_created) svm->ncr3_gpa = vm->stage2_mmu.pgd; @@ -81,7 +83,7 @@ void vm_enable_npt(struct kvm_vm *vm) tdp_mmu_init(vm, vm->mmu.pgtable_levels, &pte_masks); } -void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp) +void generic_svm_setup(struct svm_test_data *svm, void *guest_rip) { struct vmcb *vmcb = svm->vmcb; u64 vmcb_gpa = svm->vmcb_gpa; @@ -122,7 +124,7 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r ctrl->msrpm_base_pa = svm->msr_gpa; vmcb->save.rip = (u64)guest_rip; - vmcb->save.rsp = (u64)guest_rsp; + vmcb->save.rsp = (u64)svm->stack; guest_regs.rdi = (u64)svm; if (svm->ncr3_gpa) { diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 7c10ba6e6fb4..cd09c9de4485 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -116,6 +116,8 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva) vmx->vmwrite_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmwrite); memset(vmx->vmwrite_hva, 0, getpagesize()); + vmx->stack = (void *)vm_alloc_stack(vm, 1); + if (vm->stage2_mmu.pgd_created) vmx->eptp_gpa = vm->stage2_mmu.pgd; @@ -366,11 +368,11 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP)); } -void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp) +void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) { init_vmcs_control_fields(vmx); init_vmcs_host_state(); - init_vmcs_guest_state(guest_rip, guest_rsp); + init_vmcs_guest_state(guest_rip, vmx->stack); } bool kvm_cpu_has_ept(void) diff --git a/tools/testing/selftests/kvm/s390/cmma_test.c b/tools/testing/selftests/kvm/s390/cmma_test.c index e39a724fe860..15d81b2ed7ad 100644 --- a/tools/testing/selftests/kvm/s390/cmma_test.c +++ b/tools/testing/selftests/kvm/s390/cmma_test.c @@ -34,16 +34,22 @@ static char cmma_value_buf[MAIN_PAGE_COUNT + TEST_DATA_PAGE_COUNT]; /** * Dirty CMMA attributes of exactly one page in the TEST_DATA memslot, * so use_cmma goes on and the CMMA related ioctls do something. + * Touch the page at offset 1M inside TEST_DATA to make sure its page + * tables are allocated in the host. */ static void guest_do_one_essa(void) { asm volatile( /* load TEST_DATA_START_GFN into r1 */ + " xgr 1,1\n" " llilf 1,%[start_gfn]\n" /* calculate the address from the gfn */ " sllg 1,1,12(0)\n" /* set the first page in TEST_DATA memslot to STABLE */ " .insn rrf,0xb9ab0000,2,1,1,0\n" + " agfi 1,0x100000\n" + /* also touch the first page of the second MB of TEST_DATA */ + " .insn rrf,0xb9ab0000,2,1,1,0\n" /* hypercall */ " diag 0,0,0x501\n" "0: j 0b" diff --git a/tools/testing/selftests/kvm/s390/user_operexec.c b/tools/testing/selftests/kvm/s390/user_operexec.c index 714906c1d12a..b24c1f9dbbe8 100644 --- a/tools/testing/selftests/kvm/s390/user_operexec.c +++ b/tools/testing/selftests/kvm/s390/user_operexec.c @@ -6,6 +6,7 @@ * Authors: * Janosch Frank */ +#include "facility.h" #include "kselftest.h" #include "kvm_util.h" #include "test_util.h" @@ -109,6 +110,111 @@ static void test_user_operexec_combined(void) kvm_vm_free(vm); } +static struct kvm_vm *create_vm_without_sthyi(void) +{ + struct kvm_s390_vm_cpu_processor info; + struct kvm_vm *vm; + + vm = vm_create(1); + + kvm_device_attr_get(vm->fd, KVM_S390_VM_CPU_MODEL, + KVM_S390_VM_CPU_PROCESSOR, &info); + + clear_bit_inv(74, (unsigned long *)&info.fac_list); + kvm_device_attr_set(vm->fd, KVM_S390_VM_CPU_MODEL, + KVM_S390_VM_CPU_PROCESSOR, &info); + + return vm; +} + +static void test_user_instr0_no_stfle_74(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int rc; + + vm = create_vm_without_sthyi(); + + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0); + TEST_ASSERT_EQ(0, rc); + + vcpu = vm_vcpu_add(vm, 0, guest_code_instr0); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0000); + + kvm_vm_free(vm); +} + +static void test_user_operexec_no_stfle_74(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int rc; + + vm = create_vm_without_sthyi(); + + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0); + TEST_ASSERT_EQ(0, rc); + + vcpu = vm_vcpu_add(vm, 0, guest_code_user_operexec); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807); + + kvm_vm_free(vm); +} + +static void test_instr0_combined_no_stfle_74(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int rc; + + vm = create_vm_without_sthyi(); + + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0); + TEST_ASSERT_EQ(0, rc); + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0); + TEST_ASSERT_EQ(0, rc); + + vcpu = vm_vcpu_add(vm, 0, guest_code_instr0); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0000); + + kvm_vm_free(vm); +} + +static void test_operexec_combined_no_stfle_74(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int rc; + + vm = create_vm_without_sthyi(); + + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0); + TEST_ASSERT_EQ(0, rc); + rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0); + TEST_ASSERT_EQ(0, rc); + + vcpu = vm_vcpu_add(vm, 0, guest_code_user_operexec); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC); + TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807); + + kvm_vm_free(vm); +} + /* * Run all tests above. * @@ -122,6 +228,10 @@ static struct testdef { { "instr0", test_user_instr0 }, { "operexec", test_user_operexec }, { "operexec_combined", test_user_operexec_combined}, + { "instr0_no_stfle_74", test_user_instr0_no_stfle_74 }, + { "instr0_combined_no_stfle_74", test_instr0_combined_no_stfle_74 }, + { "operexec_combined_no_stfle_74", test_operexec_combined_no_stfle_74 }, + { "operexec_no_stfle_74", test_user_operexec_no_stfle_74 }, }; int main(int argc, char *argv[]) diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index c91660103137..845cb685f174 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -54,8 +54,6 @@ static void guest_read_aperf_mperf(void) GUEST_SYNC2(rdmsr(MSR_IA32_APERF), rdmsr(MSR_IA32_MPERF)); } -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code(void) { guest_read_aperf_mperf(); @@ -64,21 +62,18 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; - generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); run_guest(vmcb, svm->vmcb_gpa); } static void l1_vmx_code(struct vmx_pages *vmx) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); GUEST_ASSERT_EQ(load_vmcs(vmx), true); - prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, NULL); /* * Enable MSR bitmaps (the bitmap itself is allocated, zeroed, and set diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index 5b3aef109cfc..77ce87c41a86 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c @@ -52,8 +52,6 @@ static void l2_guest_code(void) static void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; /* Set up Hyper-V enlightenments and eVMCS */ wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID); @@ -62,8 +60,7 @@ static void guest_code(struct vmx_pages *vmx_pages, GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_evmcs(hv_pages)); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); GUEST_ASSERT(!vmlaunch()); diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index c7fa114aee20..1bda2cd3f739 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -78,9 +78,6 @@ void l2_guest_code(void) void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, gpa_t hv_hcall_page_gpa) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID); wrmsr(HV_X64_MSR_HYPERCALL, hv_hcall_page_gpa); @@ -100,8 +97,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, GUEST_SYNC(4); GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); GUEST_SYNC(5); GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c index 7a62f6a9d606..1f74b0fa9b83 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c +++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c @@ -18,8 +18,6 @@ #include "svm_util.h" #include "hyperv.h" -#define L2_GUEST_STACK_SIZE 256 - /* Exit to L1 from L2 with RDMSR instruction */ static inline void rdmsr_from_l2(u32 msr) { @@ -69,7 +67,6 @@ static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm, struct hyperv_test_pages *hv_pages, gpa_t pgs_gpa) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; struct hv_vmcb_enlightenments *hve = &vmcb->control.hv_enlightenments; @@ -81,8 +78,7 @@ static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm, GUEST_ASSERT(svm->vmcb_gpa); /* Prepare for L2 execution. */ - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); /* L2 TLB flush setup */ hve->partition_assist_page = hv_pages->partition_assist_gpa; diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 52014a3210c8..25a182be00a9 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -26,8 +26,6 @@ static void guest_generate_buslocks(void) atomic_inc(val); } -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code(void) { guest_generate_buslocks(); @@ -36,21 +34,18 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; - generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); run_guest(vmcb, svm->vmcb_gpa); } static void l1_vmx_code(struct vmx_pages *vmx) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); GUEST_ASSERT_EQ(load_vmcs(vmx), true); - prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, NULL); GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); GUEST_ASSERT(!vmlaunch()); diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index 761fec293408..b974cfb347d6 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c @@ -21,8 +21,6 @@ enum { PORT_L0_EXIT = 0x2000, }; -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code(void) { /* Exit to L0 */ @@ -32,14 +30,11 @@ static void l2_guest_code(void) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); GUEST_ASSERT(!vmlaunch()); GUEST_ASSERT(0); @@ -47,11 +42,8 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - /* Prepare the VMCB for L2 execution. */ - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); run_guest(svm->vmcb, svm->vmcb_gpa); GUEST_ASSERT(0); diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 0e67cce83570..26b474bf1353 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -40,8 +40,6 @@ #define TEST_HVA(vm, idx) addr_gpa2hva(vm, TEST_GPA(idx)) -#define L2_GUEST_STACK_SIZE 64 - /* Use the page offset bits to communicate the access+fault type. */ #define TEST_SYNC_READ_FAULT BIT(0) #define TEST_SYNC_WRITE_FAULT BIT(1) @@ -92,7 +90,6 @@ static void l2_guest_code_tdp_disabled(void) void l1_vmx_code(struct vmx_pages *vmx) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; void *l2_rip; GUEST_ASSERT(vmx->vmcs_gpa); @@ -104,7 +101,7 @@ void l1_vmx_code(struct vmx_pages *vmx) else l2_rip = l2_guest_code_tdp_disabled; - prepare_vmcs(vmx, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, l2_rip); GUEST_SYNC(TEST_SYNC_NO_FAULT); GUEST_ASSERT(!vmlaunch()); @@ -115,7 +112,6 @@ void l1_vmx_code(struct vmx_pages *vmx) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; void *l2_rip; if (svm->ncr3_gpa) @@ -123,7 +119,7 @@ static void l1_svm_code(struct svm_test_data *svm) else l2_rip = l2_guest_code_tdp_disabled; - generic_svm_setup(svm, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_rip); GUEST_SYNC(TEST_SYNC_NO_FAULT); run_guest(svm->vmcb, svm->vmcb_gpa); diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index fb7dcbe53ac7..e08c6b0697e5 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -57,7 +57,7 @@ static void guest_code(void *test_data) struct svm_test_data *svm = test_data; struct vmcb *vmcb = svm->vmcb; - generic_svm_setup(svm, NULL, NULL); + generic_svm_setup(svm, NULL); vmcb->save.idtr.limit = 0; vmcb->save.rip = (u64)l2_guest_code; @@ -69,7 +69,7 @@ static void guest_code(void *test_data) GUEST_ASSERT(prepare_for_vmx_operation(test_data)); GUEST_ASSERT(load_vmcs(test_data)); - prepare_vmcs(test_data, NULL, NULL); + prepare_vmcs(test_data, NULL); GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0)); GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0)); diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index 186e980aa8ee..aeec3121c8e8 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -5,8 +5,6 @@ #include "vmx.h" #include "svm_util.h" -#define L2_GUEST_STACK_SIZE 256 - /* * Arbitrary, never shoved into KVM/hardware, just need to avoid conflict with * the "real" exceptions used, #SS/#GP/#DF (12/13/8). @@ -91,9 +89,8 @@ static void svm_run_l2(struct svm_test_data *svm, void *l2_code, int vector, static void l1_svm_code(struct svm_test_data *svm) { struct vmcb_control_area *ctrl = &svm->vmcb->control; - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, NULL); svm->vmcb->save.idtr.limit = 0; ctrl->intercept |= BIT_ULL(INTERCEPT_SHUTDOWN); @@ -128,13 +125,11 @@ static void vmx_run_l2(void *l2_code, int vector, u32 error_code) static void l1_vmx_code(struct vmx_pages *vmx) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); GUEST_ASSERT_EQ(load_vmcs(vmx), true); - prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, NULL); GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); /* diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 11fd2467d823..8c2ba9674558 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -11,8 +11,6 @@ #include "kselftest.h" -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code(void) { vmcall(); @@ -20,11 +18,9 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; uintptr_t save_cr3; - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); /* Try to run L2 with invalid CR3 and make sure it fails */ save_cr3 = svm->vmcb->save.cr3; @@ -42,14 +38,12 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; uintptr_t save_cr3; GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); /* Try to run L2 with invalid CR3 and make sure it fails */ save_cr3 = vmreadz(GUEST_CR3); diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index fa95568f55ff..2e04563790ff 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -9,8 +9,6 @@ #include "svm_util.h" #include "vmx.h" -#define L2_GUEST_STACK_SIZE 64 - enum test_type { TEST_FINAL_PAGE_UNMAPPED, /* Final data page not present */ TEST_PT_PAGE_UNMAPPED, /* Page table page not present */ @@ -54,14 +52,13 @@ static void l2_guest_code_ins(void) static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, u64 test_type) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u64 exit_qual; GUEST_ASSERT(vmx->vmcs_gpa); GUEST_ASSERT(prepare_for_vmx_operation(vmx)); GUEST_ASSERT(load_vmcs(vmx)); - prepare_vmcs(vmx, l2_entry, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, l2_entry); GUEST_ASSERT(!vmlaunch()); @@ -120,12 +117,10 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, static void l1_svm_code(struct svm_test_data *svm, u64 expected_fault_gpa, u64 test_type) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; u64 exit_info_1; - generic_svm_setup(svm, l2_entry, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_entry); run_guest(vmcb, svm->vmcb_gpa); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index f0e4adac4751..cb79d7b9619c 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -34,8 +34,6 @@ #define TSC_ADJUST_VALUE (1ll << 32) #define TSC_OFFSET_VALUE -(1ll << 48) -#define L2_GUEST_STACK_SIZE 64 - enum { PORT_ABORT = 0x1000, PORT_REPORT, @@ -75,8 +73,6 @@ static void l2_guest_code(void) static void l1_guest_code(void *data) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - /* Set TSC from L1 and make sure TSC_ADJUST is updated correctly */ GUEST_ASSERT(rdtsc() < TSC_ADJUST_VALUE); wrmsr(MSR_IA32_TSC, rdtsc() - TSC_ADJUST_VALUE); @@ -93,8 +89,7 @@ static void l1_guest_code(void *data) GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); @@ -105,8 +100,7 @@ static void l1_guest_code(void *data) } else { struct svm_test_data *svm = data; - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); svm->vmcb->control.tsc_offset = TSC_OFFSET_VALUE; run_guest(svm->vmcb, svm->vmcb_gpa); diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 190e93af20a1..18f765835bf4 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -22,8 +22,6 @@ #define TSC_OFFSET_L2 ((u64)-33125236320908) #define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) -#define L2_GUEST_STACK_SIZE 64 - enum { USLEEP, UCHECK_L1, UCHECK_L2 }; #define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) #define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq) @@ -82,13 +80,10 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - /* check that L1's frequency looks alright before launching L2 */ check_tsc_freq(UCHECK_L1); - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); /* enable TSC scaling for L2 */ wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32); @@ -105,7 +100,6 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u32 control; /* check that L1's frequency looks alright before launching L2 */ @@ -115,7 +109,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(load_vmcs(vmx_pages)); /* prepare the VMCS for L2 execution */ - prepare_vmcs(vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); /* enable TSC offsetting and TSC scaling for L2 */ control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); diff --git a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c index 85d3f4cc76f3..a130759f39a1 100644 --- a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c +++ b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c @@ -28,8 +28,6 @@ #define TEST_VMCB_L2_GPA TEST_VMCB_L1_GPA(0) -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code_vmsave(void) { asm volatile("vmsave %0" : : "a"(TEST_VMCB_L2_GPA) : "memory"); @@ -70,10 +68,8 @@ static void l2_guest_code_vmcb1(void) static void l1_guest_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - /* Each test case initializes the guest RIP below */ - generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, NULL); /* Set VMSAVE/VMLOAD intercepts and make sure they work with.. */ svm->vmcb->control.intercept |= (BIT_ULL(INTERCEPT_VMSAVE) | diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c index 740051167dbd..e2542f4ced60 100644 --- a/tools/testing/selftests/kvm/x86/smm_test.c +++ b/tools/testing/selftests/kvm/x86/smm_test.c @@ -63,8 +63,6 @@ static void l2_guest_code(void) static void guest_code(void *arg) { - #define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u64 apicbase = rdmsr(MSR_IA32_APICBASE); struct svm_test_data *svm = arg; struct vmx_pages *vmx_pages = arg; @@ -81,13 +79,11 @@ static void guest_code(void *arg) if (arg) { if (this_cpu_has(X86_FEATURE_SVM)) { - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); } else { GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); } sync_with_host(5); diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 409c6cc9f921..4a1056a6cb8d 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -19,8 +19,6 @@ #include "vmx.h" #include "svm_util.h" -#define L2_GUEST_STACK_SIZE 256 - void svm_l2_guest_code(void) { GUEST_SYNC(4); @@ -35,13 +33,11 @@ void svm_l2_guest_code(void) static void svm_l1_guest_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; GUEST_ASSERT(svm->vmcb_gpa); /* Prepare for L2 execution. */ - generic_svm_setup(svm, svm_l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, svm_l2_guest_code); vmcb->control.int_ctl |= (V_GIF_ENABLE_MASK | V_GIF_MASK); @@ -78,8 +74,6 @@ void vmx_l2_guest_code(void) static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT(vmx_pages->vmcs_gpa); GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_SYNC(3); @@ -89,8 +83,7 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) GUEST_SYNC(4); GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - prepare_vmcs(vmx_pages, vmx_l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, vmx_l2_guest_code); GUEST_SYNC(5); GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); diff --git a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c index d3cc5e4f7883..7b1f4a4818bd 100644 --- a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c +++ b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c @@ -54,15 +54,12 @@ static void l2_guest_code(struct svm_test_data *svm) static void l1_guest_code(struct svm_test_data *svm) { - #define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; x2apic_enable(); /* Prepare for L2 execution. */ - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); /* No virtual interrupt masking */ vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; diff --git a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c index 7fbfaa054c95..77c6ce9f4507 100644 --- a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c +++ b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c @@ -9,8 +9,6 @@ #include "svm_util.h" -#define L2_GUEST_STACK_SIZE 64 - #define DO_BRANCH() do { asm volatile("jmp 1f\n 1: nop"); } while (0) struct lbr_branch { @@ -55,7 +53,6 @@ static void l2_guest_code(struct svm_test_data *svm) static void l1_guest_code(struct svm_test_data *svm, bool nested_lbrv) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; struct lbr_branch l1_branch; @@ -65,8 +62,7 @@ static void l1_guest_code(struct svm_test_data *svm, bool nested_lbrv) CHECK_BRANCH_MSRS(&l1_branch); /* Run L2, which will also do the same */ - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); if (nested_lbrv) vmcb->control.misc_ctl2 = SVM_MISC2_ENABLE_V_LBR; diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c index 6a89eaffc657..6bc301207cbc 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c @@ -8,8 +8,6 @@ #include "kselftest.h" -#define L2_GUEST_STACK_SIZE 64 - static void l2_guest_code(void) { unsigned long efer = rdmsr(MSR_EFER); @@ -24,10 +22,7 @@ static void l2_guest_code(void) static void l1_guest_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); run_guest(svm->vmcb, svm->vmcb_gpa); /* Unreachable, L1 should be shutdown */ diff --git a/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c b/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c index 92da8ff34da1..14ec9d6ad195 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c @@ -25,8 +25,6 @@ #include "processor.h" #include "svm_util.h" -#define L2_GUEST_STACK_SIZE 256 - #define PAT_DEFAULT 0x0007040600070406ULL #define L1_PAT_VALUE 0x0007040600070404ULL /* Change PA0 to WT */ #define L2_VMCB12_PAT 0x0606060606060606ULL /* All WB */ @@ -59,14 +57,13 @@ static void l2_guest_code(void) static void l1_guest_code(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; int i; wrmsr(MSR_IA32_CR_PAT, L1_PAT_VALUE); GUEST_ASSERT_EQ(rdmsr(MSR_IA32_CR_PAT), L1_PAT_VALUE); - generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); vmcb->save.g_pat = L2_VMCB12_PAT; vmcb->control.intercept &= ~(1ULL << INTERCEPT_MSR_PROT); @@ -94,11 +91,10 @@ static void l1_guest_code(struct svm_test_data *svm) static void l1_guest_code_invalid_gpat(struct svm_test_data *svm) { - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; /* VMRUN should fail without running L2 */ - generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, NULL); vmcb->save.g_pat = INVALID_PAT_VALUE; run_guest(vmcb, svm->vmcb_gpa); diff --git a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c index c6ea3d609a62..2a4a216954bb 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c @@ -19,12 +19,9 @@ static void l2_guest_code(struct svm_test_data *svm) static void l1_guest_code(struct svm_test_data *svm, struct idt_entry *idt) { - #define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); vmcb->control.intercept &= ~(BIT(INTERCEPT_SHUTDOWN)); diff --git a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c index f72f11d4c4f8..0b640d09d194 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c @@ -78,17 +78,13 @@ static void l2_guest_code_nmi(void) static void l1_guest_code(struct svm_test_data *svm, u64 is_nmi, u64 idt_alt) { - #define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; if (is_nmi) x2apic_enable(); /* Prepare for L2 execution. */ - generic_svm_setup(svm, - is_nmi ? l2_guest_code_nmi : l2_guest_code_int, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, is_nmi ? l2_guest_code_nmi : l2_guest_code_int); vmcb->control.intercept_exceptions |= BIT(PF_VECTOR) | BIT(UD_VECTOR); vmcb->control.intercept |= BIT(INTERCEPT_NMI) | BIT(INTERCEPT_HLT); diff --git a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c index a4935ce2fb99..b3f45035745f 100644 --- a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c +++ b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c @@ -9,14 +9,9 @@ #include "kvm_test_harness.h" #include "test_util.h" - -#define L2_GUEST_STACK_SIZE 64 - #define SYNC_GP 101 #define SYNC_L2_STARTED 102 -static unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - static void guest_gp_handler(struct ex_regs *regs) { GUEST_SYNC(SYNC_GP); @@ -30,28 +25,28 @@ static void l2_code(void) static void l1_vmrun(struct svm_test_data *svm, gpa_t gpa) { - generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_code); asm volatile ("vmrun %[gpa]" : : [gpa] "a" (gpa) : "memory"); } static void l1_vmload(struct svm_test_data *svm, gpa_t gpa) { - generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_code); asm volatile ("vmload %[gpa]" : : [gpa] "a" (gpa) : "memory"); } static void l1_vmsave(struct svm_test_data *svm, gpa_t gpa) { - generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_code); asm volatile ("vmsave %[gpa]" : : [gpa] "a" (gpa) : "memory"); } static void l1_vmexit(struct svm_test_data *svm, gpa_t gpa) { - generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_code); run_guest(svm->vmcb, svm->vmcb_gpa); GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL); diff --git a/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c b/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c new file mode 100644 index 000000000000..c5b5cd788d93 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM nested SVM PMU Host-Only/Guest-Only test + * + * Copyright (C) 2026, Google LLC. + * + * Test that KVM correctly virtualizes the AMD PMU Host-Only (bit 41) and + * Guest-Only (bit 40) event selector bits across all SVM state + * transitions. + * + * Programs 4 PMCs simultaneously with all combinations of Host-Only and + * Guest-Only bits, then verifies correct counting behavior with different + * combinations of EFER.SVME and host/guest mode -- as well as event filtering. + */ +#include +#include +#include +#include + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "svm_util.h" +#include "pmu.h" + +#define EVENTSEL_RETIRED_INSNS (ARCH_PERFMON_EVENTSEL_OS | \ + ARCH_PERFMON_EVENTSEL_USR | \ + ARCH_PERFMON_EVENTSEL_ENABLE | \ + AMD_ZEN_INSTRUCTIONS_RETIRED) + +/* PMC configurations: index corresponds to Host-Only | Guest-Only bits */ +#define PMC_NONE 0 /* Neither bit set */ +#define PMC_G 1 /* Guest-Only bit set */ +#define PMC_H 2 /* Host-Only bit set */ +#define PMC_HG 3 /* Both bits set */ +#define NR_PMCS 4 + +#define LOOP_INSNS 1000 + +static __always_inline void run_instruction_loop(void) +{ + unsigned int i; + + for (i = 0; i < LOOP_INSNS; i++) + __asm__ __volatile__("nop"); +} + +static __always_inline void read_counters(uint64_t *counts) +{ + int i; + + for (i = 0; i < NR_PMCS; i++) + counts[i] = rdmsr(MSR_F15H_PERF_CTR + 2 * i); +} + +static __always_inline void run_and_measure(uint64_t *deltas) +{ + uint64_t before[NR_PMCS], after[NR_PMCS]; + int i; + + read_counters(before); + run_instruction_loop(); + read_counters(after); + + for (i = 0; i < NR_PMCS; i++) + deltas[i] = after[i] - before[i]; +} + +static void assert_pmc_counts(uint64_t *deltas, unsigned int expected_counting) +{ + int i; + + for (i = 0; i < NR_PMCS; i++) { + if (expected_counting & BIT(i)) + GUEST_ASSERT_NE(deltas[i], 0); + else + GUEST_ASSERT_EQ(deltas[i], 0); + } +} + +static uint64_t l2_deltas[NR_PMCS]; + +static void l2_guest_code(void) +{ + run_and_measure(l2_deltas); + vmmcall(); +} + +static void l1_guest_code(struct svm_test_data *svm) +{ + struct vmcb *vmcb = svm->vmcb; + uint64_t deltas[NR_PMCS]; + uint64_t eventsel; + int i; + + /* Program 4 PMCs with all combinations of Host-Only/Guest-Only bits */ + for (i = 0; i < NR_PMCS; i++) { + eventsel = EVENTSEL_RETIRED_INSNS; + if (i & PMC_G) + eventsel |= AMD64_EVENTSEL_GUESTONLY; + if (i & PMC_H) + eventsel |= AMD64_EVENTSEL_HOSTONLY; + wrmsr(MSR_F15H_PERF_CTL + 2 * i, eventsel); + wrmsr(MSR_F15H_PERF_CTR + 2 * i, 0); + } + + /* Step 1: SVME=0 - Only the counter with neither bits set counts */ + wrmsr(MSR_EFER, rdmsr(MSR_EFER) & ~EFER_SVME); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE)); + + /* Step 2: Set SVME=1 - In L1 "host mode"; Guest-Only stops */ + wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_SVME); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG)); + + /* Step 3: VMRUN to L2 - In "guest mode"; Host-Only stops */ + generic_svm_setup(svm, l2_guest_code); + vmcb->control.intercept &= ~(1ULL << INTERCEPT_MSR_PROT); + + run_guest(vmcb, svm->vmcb_gpa); + + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + assert_pmc_counts(l2_deltas, BIT(PMC_NONE) | BIT(PMC_G) | BIT(PMC_HG)); + + /* Step 4: After VMEXIT to L1 - Back in "host mode"; Guest-Only stops */ + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG)); + + /* Step 5: Set KVM_PMU_EVENT_DENY - all counters stop */ + GUEST_SYNC(KVM_PMU_EVENT_DENY); + run_and_measure(deltas); + assert_pmc_counts(deltas, 0); + + /* Step 6: Set KVM_PMU_EVENT_ALLOW - back to all except Guest-only */ + GUEST_SYNC(KVM_PMU_EVENT_ALLOW); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG)); + + /* Step 7: Clear Host-Only for PMC_HG - counter stops in "host mode" */ + eventsel = rdmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG); + wrmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG, eventsel & ~AMD64_EVENTSEL_HOSTONLY); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H)); + + /* Step 8: Restore Host-Only for PMC_HG - counter counts again */ + wrmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG, eventsel); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG)); + + /* Step 9: Clear SVME - Only the counter with neither bits set counts */ + wrmsr(MSR_EFER, rdmsr(MSR_EFER) & ~EFER_SVME); + run_and_measure(deltas); + assert_pmc_counts(deltas, BIT(PMC_NONE)); + + GUEST_DONE(); +} + +static struct kvm_pmu_event_filter *alloc_event_filter(u64 event) +{ + struct kvm_pmu_event_filter *filter; + + filter = malloc(sizeof(*filter) + sizeof(event)); + TEST_ASSERT(filter != NULL, "Filter allocation failed"); + + memset(filter, 0, sizeof(*filter)); + memcpy(filter->events, &event, sizeof(event)); + filter->nevents = 1; + filter->action = KVM_PMU_EVENT_ALLOW; + + return filter; +} + +int main(int argc, char *argv[]) +{ + struct kvm_pmu_event_filter *filter; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + gva_t svm_gva; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_is_pmu_enabled()); + TEST_REQUIRE(kvm_is_mediated_pmu_enabled()); + + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + filter = alloc_event_filter(AMD_ZEN_INSTRUCTIONS_RETIRED); + + for (;;) { + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + goto done; + case UCALL_DONE: + goto done; + case UCALL_SYNC: + filter->action = uc.args[1]; + vm_ioctl(vm, KVM_SET_PMU_EVENT_FILTER, filter); + break; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + goto done; + } + } +done: + kvm_vm_free(vm); + return 0; +} diff --git a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c index b1887242f3b8..7c57fb7e6422 100644 --- a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c +++ b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c @@ -19,13 +19,10 @@ static void l2_guest_code(struct svm_test_data *svm) static void l1_guest_code(struct svm_test_data *svm) { - #define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; struct vmcb *vmcb = svm->vmcb; /* Prepare for L2 execution. */ - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); run_guest(vmcb, svm->vmcb_gpa); diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index f1c488e0d497..0d83516f4bd0 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -21,9 +21,6 @@ static void l2_guest_code(void) : : [port] "d" (ARBITRARY_IO_PORT) : "rax"); } -#define L2_GUEST_STACK_SIZE 64 -unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - void l1_guest_code_vmx(struct vmx_pages *vmx) { @@ -31,8 +28,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) GUEST_ASSERT(prepare_for_vmx_operation(vmx)); GUEST_ASSERT(load_vmcs(vmx)); - prepare_vmcs(vmx, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx, l2_guest_code); GUEST_ASSERT(!vmlaunch()); /* L2 should triple fault after a triple fault event injected. */ @@ -44,8 +40,7 @@ void l1_guest_code_svm(struct svm_test_data *svm) { struct vmcb *vmcb = svm->vmcb; - generic_svm_setup(svm, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + generic_svm_setup(svm, l2_guest_code); /* don't intercept shutdown to test the case of SVM allowing to do so */ vmcb->control.intercept &= ~(BIT(INTERCEPT_SHUTDOWN)); diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 1720113eae79..463f73aa9159 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -36,16 +36,13 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u32 control; GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index 80a4fd1e5bbb..f9b88a6f6113 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -31,15 +31,13 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u32 control; GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ - prepare_vmcs(vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index a2eaceed9ad5..6d88c54f69fa 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -25,15 +25,11 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); /* Prepare the VMCS for L2 execution. */ - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); /* * L2 must be run without unrestricted guest, verify that the selftests diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index f13dee317383..75073efa926d 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -27,8 +27,6 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u64 guest_cr4; gpa_t pml5_pa, pml4_pa; u64 *pml5; @@ -42,8 +40,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); GUEST_ASSERT(load_vmcs(vmx_pages)); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); /* * Set up L2 with a 4-level page table by pointing its CR3 to diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index 1b7b6ba23de7..eb8021c33cd4 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -66,8 +66,6 @@ void l2_guest_code(void) void l1_guest_code(struct vmx_pages *vmx_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; u64 l1_vmx_pt_start; u64 l1_vmx_pt_finish; u64 l1_tsc_deadline, l2_tsc_deadline; @@ -77,8 +75,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(load_vmcs(vmx_pages)); GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - prepare_vmcs(vmx_pages, l2_guest_code, - &l2_guest_stack[L2_GUEST_STACK_SIZE]); + prepare_vmcs(vmx_pages, l2_guest_code); /* * Check for Preemption timer support diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c index 2ed1f76b7a8b..2e4dc5025b04 100644 --- a/tools/testing/selftests/landlock/net_test.c +++ b/tools/testing/selftests/landlock/net_test.c @@ -1281,6 +1281,100 @@ TEST_F(protocol, connect_unspec) EXPECT_EQ(0, close(bind_fd)); } +TEST_F(protocol, tcp_fastopen) +{ + const bool restricted = variant->sandbox == TCP_SANDBOX && + variant->prot.type == SOCK_STREAM && + (variant->prot.protocol == IPPROTO_TCP || variant->prot.protocol == IPPROTO_IP) && + (variant->prot.domain == AF_INET || variant->prot.domain == AF_INET6); + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + int bind_fd, client_fd, status; + char buf; + pid_t child; + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + EXPECT_EQ(0, bind_variant(bind_fd, &self->srv0)); + if (self->srv0.protocol.type == SOCK_STREAM) + EXPECT_EQ(0, listen(bind_fd, backlog)); + + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + int connect_fd, ret; + + /* Closes listening socket for the child. */ + EXPECT_EQ(0, close(bind_fd)); + + connect_fd = socket_variant(&self->srv0); + ASSERT_LE(0, connect_fd); + + if (variant->sandbox == TCP_SANDBOX) { + const int ruleset_fd = landlock_create_ruleset( + &ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + /* Fast Open with no address. */ + ret = sendto_variant(connect_fd, NULL, NULL, 0, MSG_FASTOPEN); + if (self->srv0.protocol.domain == AF_UNIX) { + EXPECT_EQ(-ENOTCONN, ret); + } else if (self->srv0.protocol.type == SOCK_DGRAM) { + EXPECT_EQ(-EDESTADDRREQ, ret); + } else { + EXPECT_EQ(-EINVAL, ret); + } + + /* Fast Open to a denied address. */ + ret = sendto_variant(connect_fd, &self->srv0, "A", 1, MSG_FASTOPEN); + if (restricted) { + EXPECT_EQ(-EACCES, ret); + } else if (self->srv0.protocol.domain == AF_UNIX && + self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-EOPNOTSUPP, ret); + } else { + EXPECT_EQ(0, ret); + } + + EXPECT_EQ(0, close(connect_fd)); + _exit(_metadata->exit_code); + return; + } + + client_fd = bind_fd; + if (!restricted && self->srv0.protocol.type == SOCK_STREAM && + self->srv0.protocol.domain != AF_UNIX) { + client_fd = accept(bind_fd, NULL, 0); + ASSERT_LE(0, client_fd); + } + + if (restricted) { + EXPECT_EQ(-1, read(client_fd, &buf, 1)); + EXPECT_EQ(ENOTCONN, errno); + } else if (self->srv0.protocol.domain == AF_UNIX && + self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-1, read(client_fd, &buf, 1)); + EXPECT_EQ(EINVAL, errno); + } else { + EXPECT_EQ(1, read(client_fd, &buf, 1)); + EXPECT_EQ('A', buf); + } + + EXPECT_EQ(child, waitpid(child, &status, 0)); + EXPECT_EQ(1, WIFEXITED(status)); + EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status)); + + if (client_fd != bind_fd) + EXPECT_LE(0, close(client_fd)); + + EXPECT_EQ(0, close(bind_fd)); +} + TEST_F(protocol, sendmsg_stream) { int srv0_fd, tmp_fd, client_fd, res; diff --git a/tools/testing/selftests/liveupdate/Makefile b/tools/testing/selftests/liveupdate/Makefile index 080754787ede..30689d22cb02 100644 --- a/tools/testing/selftests/liveupdate/Makefile +++ b/tools/testing/selftests/liveupdate/Makefile @@ -6,6 +6,8 @@ TEST_GEN_PROGS += liveupdate TEST_GEN_PROGS_EXTENDED += luo_kexec_simple TEST_GEN_PROGS_EXTENDED += luo_multi_session +TEST_GEN_PROGS_EXTENDED += luo_stress_sessions +TEST_GEN_PROGS_EXTENDED += luo_stress_files TEST_FILES += do_kexec.sh diff --git a/tools/testing/selftests/liveupdate/liveupdate.c b/tools/testing/selftests/liveupdate/liveupdate.c index 37c808fbe1e9..502fb3567e38 100644 --- a/tools/testing/selftests/liveupdate/liveupdate.c +++ b/tools/testing/selftests/liveupdate/liveupdate.c @@ -26,6 +26,7 @@ #include +#include "luo_test_utils.h" #include "../kselftest.h" #include "../kselftest_harness.h" @@ -102,6 +103,22 @@ static int create_session(int lu_fd, const char *name) return args.fd; } +/* Helper function to get a session name via ioctl. */ +static int get_session_name(int session_fd, char *name, size_t name_len) +{ + struct liveupdate_session_get_name args = {}; + + args.size = sizeof(args); + + if (ioctl(session_fd, LIVEUPDATE_SESSION_GET_NAME, &args)) + return -errno; + + strncpy(name, (char *)args.name, name_len - 1); + name[name_len - 1] = '\0'; + + return 0; +} + /* * Test Case: Create Duplicate Session * @@ -386,4 +403,175 @@ TEST_F(liveupdate_device, prevent_double_preservation) ASSERT_EQ(close(session_fd2), 0); } +/* + * Test Case: Create Session with No Null Termination + * + * Verifies that filling the entire 64-byte name field with non-null characters + * (no '\0' terminator) is rejected by the kernel with EINVAL. + */ +TEST_F(liveupdate_device, create_session_no_null_termination) +{ + struct liveupdate_ioctl_create_session args = {}; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + /* Fill entire name field with 'X', no null terminator */ + args.size = sizeof(args); + memset(args.name, 'X', sizeof(args.name)); + + EXPECT_LT(ioctl(self->fd1, LIVEUPDATE_IOCTL_CREATE_SESSION, &args), 0); + EXPECT_EQ(errno, EINVAL); +} + +/* + * Test Case: Create Session with Empty Name + * + * Verifies that creating a session with an empty string name fails + * with EINVAL. + */ +TEST_F(liveupdate_device, create_session_empty_name) +{ + int session_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, ""); + EXPECT_EQ(session_fd, -EINVAL); +} + +/* + * Test Case: Get Session Name + * + * Verifies that the full session name can be retrieved from a session file + * descriptor via ioctl. + */ +TEST_F(liveupdate_device, get_session_name) +{ + char name_buf[LIVEUPDATE_SESSION_NAME_LENGTH] = {}; + const char *session_name = "get-name-test-session"; + int session_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, session_name); + ASSERT_GE(session_fd, 0); + + ASSERT_EQ(get_session_name(session_fd, name_buf, sizeof(name_buf)), 0); + ASSERT_STREQ(name_buf, session_name); + + ASSERT_EQ(close(session_fd), 0); +} + +/* + * Test Case: Get Session Name at Maximum Length + * + * Verifies that a session name using the full LIVEUPDATE_SESSION_NAME_LENGTH + * (minus the null terminator) can be correctly retrieved. + */ +TEST_F(liveupdate_device, get_session_name_max_length) +{ + char name_buf[LIVEUPDATE_SESSION_NAME_LENGTH] = {}; + char long_name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int session_fd; + + memset(long_name, 'A', sizeof(long_name) - 1); + long_name[sizeof(long_name) - 1] = '\0'; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, long_name); + ASSERT_GE(session_fd, 0); + + ASSERT_EQ(get_session_name(session_fd, name_buf, sizeof(name_buf)), 0); + ASSERT_STREQ(name_buf, long_name); + + ASSERT_EQ(close(session_fd), 0); +} + +/* + * Test Case: Manage Many Sessions + * + * Verifies that a large number of sessions can be created and then + * destroyed during normal system operation. This specifically tests the + * dynamic block allocation and reuse logic for session metadata management + * without preserving any files. + */ +TEST_F(liveupdate_device, preserve_many_sessions) +{ +#define MANY_SESSIONS 2000 + int session_fds[MANY_SESSIONS]; + int ret, i; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + ret = luo_ensure_nofile_limit(MANY_SESSIONS); + if (ret == -EPERM) + SKIP(return, "Insufficient privileges to set RLIMIT_NOFILE"); + ASSERT_EQ(ret, 0); + + for (i = 0; i < MANY_SESSIONS; i++) { + char name[64]; + + snprintf(name, sizeof(name), "many-session-%d", i); + session_fds[i] = create_session(self->fd1, name); + ASSERT_GE(session_fds[i], 0); + } + + for (i = 0; i < MANY_SESSIONS; i++) + ASSERT_EQ(close(session_fds[i]), 0); +} + +/* + * Test Case: Preserve Many Files + * + * Verifies that a large number of files can be preserved in a single session + * and then destroyed during normal system operation. This tests the dynamic + * block allocation and management for outgoing files. + */ +TEST_F(liveupdate_device, preserve_many_files) +{ +#define MANY_FILES 500 + int mem_fds[MANY_FILES]; + int session_fd, ret, i; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, "many-files-test"); + ASSERT_GE(session_fd, 0); + + ret = luo_ensure_nofile_limit(MANY_FILES + 10); + if (ret == -EPERM) + SKIP(return, "Insufficient privileges to set RLIMIT_NOFILE"); + ASSERT_EQ(ret, 0); + + for (i = 0; i < MANY_FILES; i++) { + mem_fds[i] = memfd_create("test-memfd", 0); + ASSERT_GE(mem_fds[i], 0); + ASSERT_EQ(preserve_fd(session_fd, mem_fds[i], i), 0); + } + + for (i = 0; i < MANY_FILES; i++) + ASSERT_EQ(close(mem_fds[i]), 0); + + ASSERT_EQ(close(session_fd), 0); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/liveupdate/luo_stress_files.c b/tools/testing/selftests/liveupdate/luo_stress_files.c new file mode 100644 index 000000000000..0cdf9cd4bac7 --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_stress_files.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + * + * Validate that LUO can handle a large number of files per session across + * a kexec reboot. + */ + +#include +#include +#include "luo_test_utils.h" + +#define NUM_FILES 500 +#define STATE_SESSION_NAME "kexec_many_files_state" +#define STATE_MEMFD_TOKEN 9999 +#define TEST_SESSION_NAME "many_files_session" + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int session_fd, i; + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating test session '%s'...\n", TEST_SESSION_NAME); + session_fd = luo_create_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_create_session"); + + ksft_print_msg("[STAGE 1] Preserving %d files...\n", NUM_FILES); + for (i = 0; i < NUM_FILES; i++) { + char data[64]; + + snprintf(data, sizeof(data), "file-data-%d", i); + if (create_and_preserve_memfd(session_fd, i, data) < 0) + fail_exit("create_and_preserve_memfd for index %d", i); + } + + ksft_print_msg("[STAGE 1] Successfully preserved %d files.\n", NUM_FILES); + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int session_fd; + int i, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) { + fail_exit("Expected stage 2, but state file contains %d", + stage); + } + + ksft_print_msg("[STAGE 2] Retrieving test session '%s'...\n", TEST_SESSION_NAME); + session_fd = luo_retrieve_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_retrieve_session"); + + ksft_print_msg("[STAGE 2] Verifying %d files...\n", NUM_FILES); + for (i = 0; i < NUM_FILES; i++) { + char data[64]; + int fd; + + snprintf(data, sizeof(data), "file-data-%d", i); + fd = restore_and_verify_memfd(session_fd, i, data); + if (fd < 0) + fail_exit("restore_and_verify_memfd for index %d", i); + close(fd); + } + + ksft_print_msg("[STAGE 2] Finishing test session...\n"); + if (luo_session_finish(session_fd) < 0) + fail_exit("luo_session_finish for test session"); + close(session_fd); + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- MANY-FILES KEXEC TEST PASSED (%d files) ---\n", + NUM_FILES); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_stress_sessions.c b/tools/testing/selftests/liveupdate/luo_stress_sessions.c new file mode 100644 index 000000000000..f201b1839d1d --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_stress_sessions.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + * + * Validate that LUO can handle a large number of sessions across a kexec + * reboot. + */ + +#include +#include +#include "luo_test_utils.h" + +#define NUM_SESSIONS 2000 +#define STATE_SESSION_NAME "kexec_many_state" +#define STATE_MEMFD_TOKEN 999 + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int ret, i; + + ksft_print_msg("[STAGE 1] Increasing ulimit for open files...\n"); + ret = luo_ensure_nofile_limit(NUM_SESSIONS); + if (ret == -EPERM) + ksft_exit_skip("Insufficient privileges to set RLIMIT_NOFILE\n"); + if (ret < 0) + ksft_exit_fail_msg("luo_ensure_nofile_limit failed: %s\n", strerror(-ret)); + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating %d sessions...\n", NUM_SESSIONS); + + for (i = 0; i < NUM_SESSIONS; i++) { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int s_fd; + + snprintf(name, sizeof(name), "many-test-%d", i); + s_fd = luo_create_session(luo_fd, name); + if (s_fd < 0) { + fail_exit("luo_create_session for '%s' at index %d", + name, i); + } + } + + ksft_print_msg("[STAGE 1] Successfully created %d sessions.\n", + NUM_SESSIONS); + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int i, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) { + fail_exit("Expected stage 2, but state file contains %d", + stage); + } + + ksft_print_msg("[STAGE 2] Retrieving and finishing %d sessions...\n", + NUM_SESSIONS); + + for (i = 0; i < NUM_SESSIONS; i++) { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int s_fd; + + snprintf(name, sizeof(name), "many-test-%d", i); + s_fd = luo_retrieve_session(luo_fd, name); + if (s_fd < 0) { + fail_exit("luo_retrieve_session for '%s' at index %d", + name, i); + } + + if (luo_session_finish(s_fd) < 0) { + fail_exit("luo_session_finish for '%s' at index %d", + name, i); + } + close(s_fd); + } + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- MANY-SESSIONS KEXEC TEST PASSED (%d sessions) ---\n", + NUM_SESSIONS); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.c b/tools/testing/selftests/liveupdate/luo_test_utils.c index 3c8721c505df..333a3530051b 100644 --- a/tools/testing/selftests/liveupdate/luo_test_utils.c +++ b/tools/testing/selftests/liveupdate/luo_test_utils.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,29 @@ int luo_open_device(void) return open(LUO_DEVICE, O_RDWR); } +int luo_ensure_nofile_limit(long min_limit) +{ + struct rlimit hl; + + /* Allow to extra files to be used by test itself */ + min_limit += 32; + + if (getrlimit(RLIMIT_NOFILE, &hl) < 0) + return -errno; + + if (hl.rlim_cur >= min_limit) + return 0; + + hl.rlim_cur = min_limit; + if (hl.rlim_cur > hl.rlim_max) + hl.rlim_max = hl.rlim_cur; + + if (setrlimit(RLIMIT_NOFILE, &hl) < 0) + return -errno; + + return 0; +} + int luo_create_session(int luo_fd, const char *name) { struct liveupdate_ioctl_create_session arg = { .size = sizeof(arg) }; diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.h b/tools/testing/selftests/liveupdate/luo_test_utils.h index 90099bf49577..6a0d85386613 100644 --- a/tools/testing/selftests/liveupdate/luo_test_utils.h +++ b/tools/testing/selftests/liveupdate/luo_test_utils.h @@ -26,6 +26,8 @@ int luo_create_session(int luo_fd, const char *name); int luo_retrieve_session(int luo_fd, const char *name); int luo_session_finish(int session_fd); +int luo_ensure_nofile_limit(long min_limit); + int create_and_preserve_memfd(int session_fd, int token, const char *data); int restore_and_verify_memfd(int session_fd, int token, const char *expected_data); diff --git a/tools/testing/selftests/mm/.gitignore b/tools/testing/selftests/mm/.gitignore index b0c30c5ee9e3..9ccd9e1447e6 100644 --- a/tools/testing/selftests/mm/.gitignore +++ b/tools/testing/selftests/mm/.gitignore @@ -4,6 +4,10 @@ hugepage-mmap hugepage-mremap hugepage-shm hugepage-vmemmap +hugetlb-mmap +hugetlb-mremap +hugetlb-shm +hugetlb-vmemmap hugetlb-madvise hugetlb-read-hwpoison hugetlb-soft-offline diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 41053fdaad88..ed321ae709da 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -61,16 +61,15 @@ TEST_GEN_FILES += gup_longterm TEST_GEN_FILES += gup_test TEST_GEN_FILES += hmm-tests TEST_GEN_FILES += hugetlb-madvise +TEST_GEN_FILES += hugetlb-mmap +TEST_GEN_FILES += hugetlb-mremap TEST_GEN_FILES += hugetlb-read-hwpoison +TEST_GEN_FILES += hugetlb-shm TEST_GEN_FILES += hugetlb-soft-offline -TEST_GEN_FILES += hugepage-mmap -TEST_GEN_FILES += hugepage-mremap -TEST_GEN_FILES += hugepage-shm -TEST_GEN_FILES += hugepage-vmemmap +TEST_GEN_FILES += hugetlb-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate TEST_GEN_FILES += map_fixed_noreplace -TEST_GEN_FILES += map_hugetlb TEST_GEN_FILES += map_populate ifneq (,$(filter $(ARCH),arm64 riscv riscv64 x86 x86_64 loongarch32 loongarch64)) TEST_GEN_FILES += memfd_secret @@ -175,6 +174,10 @@ TEST_PROGS += ksft_userfaultfd.sh TEST_PROGS += ksft_vma_merge.sh TEST_PROGS += ksft_vmalloc.sh +# Destructive: every successful run panics the kernel. Installed and +# kept executable, but not run from a default kselftest invocation. +TEST_PROGS_EXTENDED += hwpoison-panic.sh + TEST_FILES := test_vmalloc.sh TEST_FILES += test_hmm.sh TEST_FILES += va_high_addr_switch.sh @@ -188,8 +191,8 @@ TEST_FILES += write_hugetlb_memory.sh include ../lib.mk -$(TEST_GEN_PROGS): vm_util.c thp_settings.c -$(TEST_GEN_FILES): vm_util.c thp_settings.c +$(TEST_GEN_PROGS): vm_util.c hugepage_settings.c +$(TEST_GEN_FILES): vm_util.c hugepage_settings.c $(OUTPUT)/uffd-stress: uffd-common.c $(OUTPUT)/uffd-unit-tests: uffd-common.c diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index 44f4e703deb9..a1cfd3a349db 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -17,6 +17,7 @@ if ! command -v killall >/dev/null 2>&1; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM fault_limit_file=limit_in_bytes reservation_limit_file=rsvd.limit_in_bytes @@ -70,7 +71,6 @@ function cleanup() { if [[ -e $cgroup_path/hugetlb_cgroup_test2 ]]; then rmdir $cgroup_path/hugetlb_cgroup_test2 fi - echo 0 >/proc/sys/vm/nr_hugepages echo CLEANUP DONE } @@ -94,6 +94,15 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + # For 1GB hugepages + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + # For 2MB hugepages + UNIT="MB" + MB_DISPLAY=$MB +fi function setup_cgroup() { local name="$1" @@ -103,11 +112,12 @@ function setup_cgroup() { mkdir $cgroup_path/$name echo writing cgroup limit: "$cgroup_limit" - echo "$cgroup_limit" >$cgroup_path/$name/hugetlb.${MB}MB.$fault_limit_file + echo "$cgroup_limit" > \ + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$fault_limit_file echo writing reservation limit: "$reservation_limit" echo "$reservation_limit" > \ - $cgroup_path/$name/hugetlb.${MB}MB.$reservation_limit_file + $cgroup_path/$name/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_limit_file if [ -e "$cgroup_path/$name/cpuset.cpus" ]; then echo 0 >$cgroup_path/$name/cpuset.cpus @@ -142,7 +152,7 @@ function wait_for_file_value() { function wait_for_hugetlb_memory_to_get_depleted() { local cgroup="$1" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "0" } @@ -150,7 +160,7 @@ function wait_for_hugetlb_memory_to_get_depleted() { function wait_for_hugetlb_memory_to_get_reserved() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file" wait_for_file_value "$path" "$size" } @@ -158,7 +168,7 @@ function wait_for_hugetlb_memory_to_get_reserved() { function wait_for_hugetlb_memory_to_get_written() { local cgroup="$1" local size="$2" - local path="$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file" + local path="$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file" wait_for_file_value "$path" "$size" } @@ -180,8 +190,8 @@ function write_hugetlbfs_and_get_usage() { hugetlb_difference=0 reserved_difference=0 - local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$fault_usage_file - local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB}MB.$reservation_usage_file + local hugetlb_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local reserved_usage=$cgroup_path/$cgroup/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local hugetlb_before=$(cat $hugetlb_usage) local reserved_before=$(cat $reserved_usage) @@ -312,8 +322,10 @@ function run_test() { cleanup_hugetlb_memory "hugetlb_cgroup_test" - local final_hugetlb=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$fault_usage_file) - local final_reservation=$(cat $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB}MB.$reservation_usage_file) + local final_hugetlb=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file) + local final_reservation=$(cat \ + $cgroup_path/hugetlb_cgroup_test/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file) echo $hugetlb_difference echo $reserved_difference @@ -369,10 +381,14 @@ function run_multiple_cgroup_test() { reservation_failed1=$reservation_failed oom_killed1=$oom_killed - local cgroup1_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$fault_usage_file - local cgroup1_reservation_usage=$cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB}MB.$reservation_usage_file - local cgroup2_hugetlb_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$fault_usage_file - local cgroup2_reservation_usage=$cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB}MB.$reservation_usage_file + local cgroup1_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup1_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test1/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file + local cgroup2_hugetlb_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$fault_usage_file + local cgroup2_reservation_usage=\ + $cgroup_path/hugetlb_cgroup_test2/hugetlb.${MB_DISPLAY}${UNIT}.$reservation_usage_file local usage_before_second_write=$(cat $cgroup1_hugetlb_usage) local reservation_usage_before_second_write=$(cat $cgroup1_reservation_usage) @@ -599,4 +615,3 @@ if [[ $do_umount ]]; then rmdir $cgroup_path fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 30209c40b697..5b582588e015 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -17,6 +17,7 @@ #include #include "kselftest.h" +#include "hugepage_settings.h" #define MAP_SIZE_MB 100 #define MAP_SIZE (MAP_SIZE_MB * 1024 * 1024) @@ -82,124 +83,44 @@ int prereq(void) return -1; } -int check_compaction(unsigned long mem_free, unsigned long hugepage_size, - unsigned long initial_nr_hugepages) +int check_compaction(unsigned long mem_free, unsigned long hugepage_size) { - unsigned long nr_hugepages_ul; - int fd, ret = -1; + unsigned long nr_hugepages; int compaction_index = 0; - char nr_hugepages[20] = {0}; - char init_nr_hugepages[24] = {0}; - char target_nr_hugepages[24] = {0}; - int slen; - - snprintf(init_nr_hugepages, sizeof(init_nr_hugepages), - "%lu", initial_nr_hugepages); + int ret = -1; /* We want to test with 80% of available memory. Else, OOM killer comes in to play */ mem_free = mem_free * 0.8; - fd = open("/proc/sys/vm/nr_hugepages", O_RDWR | O_NONBLOCK); - if (fd < 0) { - ksft_print_msg("Failed to open /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - ret = -1; - goto out; - } - /* * Request huge pages for about half of the free memory. The Kernel * will allocate as much as it can, and we expect it will get at least 1/3 */ - nr_hugepages_ul = mem_free / hugepage_size / 2; - snprintf(target_nr_hugepages, sizeof(target_nr_hugepages), - "%lu", nr_hugepages_ul); - - slen = strlen(target_nr_hugepages); - if (write(fd, target_nr_hugepages, slen) != slen) { - ksft_print_msg("Failed to write %lu to /proc/sys/vm/nr_hugepages: %s\n", - nr_hugepages_ul, strerror(errno)); - goto close_fd; - } - - lseek(fd, 0, SEEK_SET); - - if (read(fd, nr_hugepages, sizeof(nr_hugepages)) <= 0) { - ksft_print_msg("Failed to re-read from /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } + nr_hugepages = mem_free / hugepage_size / 2; + hugetlb_set_nr_default_pages(nr_hugepages); /* We should have been able to request at least 1/3 rd of the memory in huge pages */ - nr_hugepages_ul = strtoul(nr_hugepages, NULL, 10); - if (!nr_hugepages_ul) { + nr_hugepages = hugetlb_nr_default_pages(); + if (!nr_hugepages) { ksft_print_msg("ERROR: No memory is available as huge pages\n"); - goto close_fd; - } - compaction_index = mem_free/(nr_hugepages_ul * hugepage_size); - - lseek(fd, 0, SEEK_SET); - - if (write(fd, init_nr_hugepages, strlen(init_nr_hugepages)) - != strlen(init_nr_hugepages)) { - ksft_print_msg("Failed to write value to /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; + goto out; } + compaction_index = mem_free/(nr_hugepages * hugepage_size); - ksft_print_msg("Number of huge pages allocated = %lu\n", - nr_hugepages_ul); + ksft_print_msg("Number of huge pages allocated = %lu\n", nr_hugepages); if (compaction_index > 3) { ksft_print_msg("ERROR: Less than 1/%d of memory is available\n" "as huge pages\n", compaction_index); - goto close_fd; - } - - ret = 0; - - close_fd: - close(fd); - out: - ksft_test_result(ret == 0, "check_compaction\n"); - return ret; -} - -int set_zero_hugepages(unsigned long *initial_nr_hugepages) -{ - int fd, ret = -1; - char nr_hugepages[20] = {0}; - - fd = open("/proc/sys/vm/nr_hugepages", O_RDWR | O_NONBLOCK); - if (fd < 0) { - ksft_print_msg("Failed to open /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); goto out; } - if (read(fd, nr_hugepages, sizeof(nr_hugepages)) <= 0) { - ksft_print_msg("Failed to read from /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } - - lseek(fd, 0, SEEK_SET); - - /* Start with the initial condition of 0 huge pages */ - if (write(fd, "0", sizeof(char)) != sizeof(char)) { - ksft_print_msg("Failed to write 0 to /proc/sys/vm/nr_hugepages: %s\n", - strerror(errno)); - goto close_fd; - } - *initial_nr_hugepages = strtoul(nr_hugepages, NULL, 10); ret = 0; - close_fd: - close(fd); - out: + ksft_test_result(ret == 0, "check_compaction\n"); return ret; } @@ -212,18 +133,17 @@ int main(int argc, char **argv) unsigned long mem_free = 0; unsigned long hugepage_size = 0; long mem_fragmentable_MB = 0; - unsigned long initial_nr_hugepages; ksft_print_header(); if (prereq() || geteuid()) ksft_exit_skip("Prerequisites unsatisfied\n"); - ksft_set_plan(1); - /* Start the test without hugepages reducing mem_free */ - if (set_zero_hugepages(&initial_nr_hugepages)) - ksft_exit_fail(); + if (!hugetlb_setup_default_exact(0)) + ksft_exit_skip("Could not reset nr_hugepages\n"); + + ksft_set_plan(1); lim.rlim_cur = RLIM_INFINITY; lim.rlim_max = RLIM_INFINITY; @@ -261,6 +181,9 @@ int main(int argc, char **argv) mem_fragmentable_MB -= MAP_SIZE_MB; } + /* Unmap every other entry in the list to create fragmentation with + * locked pages before invoking check_compaction(). + */ for (entry = list; entry != NULL; entry = entry->next) { munmap(entry->map, MAP_SIZE); if (!entry->next) @@ -268,8 +191,7 @@ int main(int argc, char **argv) entry = entry->next; } - if (check_compaction(mem_free, hugepage_size, - initial_nr_hugepages) == 0) + if (check_compaction(mem_free, hugepage_size) == 0) ksft_exit_pass(); ksft_exit_fail(); diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index d9c69c04b67d..0c627ea89ff7 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -29,7 +29,7 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" static size_t pagesize; static int pagemap_fd; @@ -37,7 +37,7 @@ static size_t pmdsize; static int nr_thpsizes; static size_t thpsizes[20]; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; static int gup_fd; static bool has_huge_zeropage; @@ -202,7 +202,7 @@ static void do_test_cow_in_parent(char *mem, size_t size, bool do_mprotect, log_test_result(KSFT_FAIL); goto close_comm_pipes; } else if (!ret) { - exit(fn(mem, size, &comm_pipes)); + _exit(fn(mem, size, &comm_pipes)); } while (read(comm_pipes.child_ready[0], &buf, 1) != 1) @@ -333,7 +333,7 @@ static void do_test_vmsplice_in_parent(char *mem, size_t size, ; /* Modify page content in the child. */ memset(mem, 0xff, size); - exit(0); + _exit(0); } if (!before_fork) { @@ -480,7 +480,7 @@ static void do_test_iouring(char *mem, size_t size, bool use_fork) write(comm_pipes.child_ready[1], "0", 1); while (read(comm_pipes.parent_ready[0], &buf, 1) != 1) ; - exit(0); + _exit(0); } while (read(comm_pipes.child_ready[0], &buf, 1) != 1) @@ -645,7 +645,7 @@ static void do_test_ro_pin(char *mem, size_t size, enum ro_pin_test test, write(comm_pipes.child_ready[1], "0", 1); while (read(comm_pipes.parent_ready[0], &buf, 1) != 1) ; - exit(0); + _exit(0); } /* Wait until our child is ready. */ @@ -956,7 +956,7 @@ static void do_run_with_thp(test_fn fn, enum thp_run thp_run, size_t thpsize) log_test_result(KSFT_FAIL); goto munmap; } else if (!ret) { - exit(0); + _exit(0); } wait(&ret); /* Allow for sharing all pages again. */ @@ -1347,13 +1347,13 @@ static void do_test_anon_thp_collapse(char *mem, size_t size, switch (test) { case ANON_THP_COLLAPSE_UNSHARED: case ANON_THP_COLLAPSE_FULLY_SHARED: - exit(child_memcmp_fn(mem, size, &comm_pipes)); + _exit(child_memcmp_fn(mem, size, &comm_pipes)); break; case ANON_THP_COLLAPSE_LOWER_SHARED: - exit(child_memcmp_fn(mem, size / 2, &comm_pipes)); + _exit(child_memcmp_fn(mem, size / 2, &comm_pipes)); break; case ANON_THP_COLLAPSE_UPPER_SHARED: - exit(child_memcmp_fn(mem + size / 2, size / 2, + _exit(child_memcmp_fn(mem + size / 2, size / 2, &comm_pipes)); break; default: @@ -1881,21 +1881,21 @@ int main(int argc, char **argv) ksft_print_header(); + thp_save_settings(); + pagesize = getpagesize(); pmdsize = read_pmd_pagesize(); if (pmdsize) { /* Only if THP is supported. */ thp_read_settings(&default_settings); default_settings.hugepages[sz2ord(pmdsize, pagesize)].enabled = THP_INHERIT; - thp_save_settings(); thp_push_settings(&default_settings); ksft_print_msg("[INFO] detected PMD size: %zu KiB\n", pmdsize / 1024); nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); } - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, - ARRAY_SIZE(hugetlbsizes)); + nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); has_huge_zeropage = detect_huge_zeropage(); ksft_set_plan(ARRAY_SIZE(anon_test_cases) * tests_per_anon_test_case() + @@ -1911,10 +1911,5 @@ int main(int argc, char **argv) run_anon_thp_test_cases(); run_non_anon_test_cases(); - if (pmdsize) { - /* Only if THP is supported. */ - thp_restore_settings(); - } - ksft_finished(); } diff --git a/tools/testing/selftests/mm/droppable.c b/tools/testing/selftests/mm/droppable.c index 30c8be37fcb9..57e1b6fc5569 100644 --- a/tools/testing/selftests/mm/droppable.c +++ b/tools/testing/selftests/mm/droppable.c @@ -17,10 +17,10 @@ int main(int argc, char *argv[]) { - size_t alloc_size = 134217728; - size_t page_size = getpagesize(); + const size_t alloc_size = 2 * 1024 * 1024; + int retry_count = 10; + bool dropped; void *alloc; - pid_t child; ksft_print_header(); ksft_set_plan(1); @@ -35,26 +35,32 @@ int main(int argc, char *argv[]) exit(KSFT_FAIL); } memset(alloc, 'A', alloc_size); - for (size_t i = 0; i < alloc_size; i += page_size) - assert(*(uint8_t *)(alloc + i)); - - child = fork(); - assert(child >= 0); - if (!child) { - for (;;) - *(char *)malloc(page_size) = 'B'; - } - for (bool done = false; !done;) { - for (size_t i = 0; i < alloc_size; i += page_size) { - if (!*(uint8_t *)(alloc + i)) { - done = true; - break; + while (retry_count--) { + if (madvise(alloc, alloc_size, MADV_PAGEOUT)) { + if (errno == EINVAL) { + ksft_test_result_skip("madvise(MADV_PAGEOUT) not supported\n"); + exit(KSFT_SKIP); } + ksft_test_result_fail("madvise(MADV_PAGEOUT) error: %s\n", strerror(errno)); + exit(KSFT_FAIL); } + + dropped = memchr(alloc, 'A', alloc_size) == NULL; + + /* + * Speculative reference can temporarily prevent some + * pages from getting dropped. So sleep and retry. + * + * If a page is not droppable for 10s, something + * is seriously messed up and we want to fail. + */ + if (dropped) + break; + sleep(1); } - kill(child, SIGTERM); - ksft_test_result_pass("MAP_DROPPABLE: PASS\n"); - exit(KSFT_PASS); + ksft_test_result(dropped, "madvise(MADV_PAGEOUT) behavior\n"); + + ksft_finished(); } diff --git a/tools/testing/selftests/mm/folio_split_race_test.c b/tools/testing/selftests/mm/folio_split_race_test.c index ff026f183ac7..6329e37fff4c 100644 --- a/tools/testing/selftests/mm/folio_split_race_test.c +++ b/tools/testing/selftests/mm/folio_split_race_test.c @@ -25,7 +25,7 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" uint64_t page_size; uint64_t pmd_pagesize; @@ -226,23 +226,6 @@ static uint64_t run_iteration(void) return reader_failures; } -static void thp_cleanup_handler(int signum) -{ - thp_restore_settings(); - /* - * Restore default handler and re-raise the signal to exit. - * This is to ensure the test process exits with the correct - * status code corresponding to the signal. - */ - signal(signum, SIG_DFL); - raise(signum); -} - -static void thp_settings_cleanup(void) -{ - thp_restore_settings(); -} - int main(void) { struct thp_settings current_settings; @@ -261,12 +244,6 @@ int main(void) ksft_exit_skip("Please run the test as root\n"); thp_save_settings(); - /* make sure thp settings are restored */ - if (atexit(thp_settings_cleanup) != 0) - ksft_exit_fail_msg("atexit failed\n"); - - signal(SIGINT, thp_cleanup_handler); - signal(SIGTERM, thp_cleanup_handler); thp_read_settings(¤t_settings); current_settings.shmem_enabled = SHMEM_ADVISE; diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index 48e8b1539be3..b21df3040b1c 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -21,7 +21,7 @@ #include #include #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "../pidfd/pidfd.h" @@ -2203,17 +2203,6 @@ TEST_F(guard_regions, collapse) if (variant->backing != ANON_BACKED) ASSERT_EQ(ftruncate(self->fd, size), 0); - /* - * We must close and re-open local-file backed as read-only for - * CONFIG_READ_ONLY_THP_FOR_FS to work. - */ - if (variant->backing == LOCAL_FILE_BACKED) { - ASSERT_EQ(close(self->fd), 0); - - self->fd = open(self->path, O_RDONLY); - ASSERT_GE(self->fd, 0); - } - ptr = mmap_(self, variant, NULL, size, PROT_READ, 0, 0); ASSERT_NE(ptr, MAP_FAILED); @@ -2237,9 +2226,10 @@ TEST_F(guard_regions, collapse) /* * Now collapse the entire region. This should fail in all cases. * - * The madvise() call will also fail if CONFIG_READ_ONLY_THP_FOR_FS is - * not set for the local file case, but we can't differentiate whether - * this occurred or if the collapse was rightly rejected. + * The madvise() call will also fail if the file system does not support + * large folio or the supported orders do not include PMD_ORDER for the + * local file case, but we can't differentiate whether this occurred or + * if the collapse was rightly rejected. */ EXPECT_NE(madvise(ptr, size, MADV_COLLAPSE), 0); diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index f61150d28eb2..eb8963e9d98f 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -29,10 +29,11 @@ #include "../../../../mm/gup_test.h" #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" static size_t pagesize; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; static int gup_fd; static __fsword_t get_fs_type(int fd) @@ -509,7 +510,7 @@ int main(int argc, char **argv) int i; pagesize = getpagesize(); - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, + nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); ksft_print_header(); diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/testing/selftests/mm/gup_test.c index fb8f9ae49efa..3f841a96f870 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/testing/selftests/mm/gup_test.c @@ -14,6 +14,7 @@ #include #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define MB (1UL << 20) @@ -94,6 +95,7 @@ int main(int argc, char **argv) int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; int flags = MAP_PRIVATE; char *file = "/dev/zero"; + bool hugetlb = false; pthread_t *tid; char *p; @@ -168,6 +170,7 @@ int main(int argc, char **argv) break; case 'H': flags |= (MAP_HUGETLB | MAP_ANONYMOUS); + hugetlb = true; break; default: ksft_exit_fail_msg("Wrong argument\n"); @@ -199,6 +202,18 @@ int main(int argc, char **argv) } ksft_print_header(); + + if (hugetlb) { + unsigned long hp_size = default_huge_page_size(); + + if (!hp_size) + ksft_exit_skip("HugeTLB is unavailable\n"); + + size = (size + hp_size - 1) & ~(hp_size - 1); + if (!hugetlb_setup_default(size / hp_size)) + ksft_exit_skip("Not enough huge pages\n"); + } + ksft_set_plan(nthreads); filed = open(file, O_RDWR|O_CREAT, 0664); diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 6a23c09ac2da..2f2b9879d100 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -11,6 +11,7 @@ */ #include "kselftest_harness.h" +#include "hugepage_settings.h" #include #include @@ -21,13 +22,14 @@ #include #include #include +#include +#include #include #include #include #include #include - /* * This is a private UAPI to the kernel test module so it isn't exported * in the usual include/uapi/... directory. @@ -69,6 +71,9 @@ enum { #ifndef FOLL_LONGTERM #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ #endif + +HUGETLB_SETUP_DEFAULT_PAGES(1) + FIXTURE(hmm) { int fd; @@ -632,7 +637,7 @@ TEST_F(hmm, anon_write_child) } close(child_fd); - exit(0); + _exit(0); } } } @@ -712,7 +717,7 @@ TEST_F(hmm, anon_write_child_shared) ASSERT_EQ(ptr[i], -i); close(child_fd); - exit(0); + _exit(0); } /* @@ -784,8 +789,8 @@ TEST_F(hmm, anon_write_hugetlbfs) int *ptr; int ret; - if (!default_hsize) - SKIP(return, "Huge page size could not be determined"); + if (!hugetlb_free_default_pages()) + SKIP(return, "Not enough huge pages"); size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1599,8 +1604,8 @@ TEST_F(hmm2, snapshot) } /* - * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that - * should be mapped by a large page table entry. + * Test the hmm_range_fault() handling of large pages (PMD or PUD) + * that should be mapped by a large page table entry. */ TEST_F(hmm, compound) { @@ -1610,13 +1615,13 @@ TEST_F(hmm, compound) unsigned long default_hsize = default_huge_page_size(); int *ptr; unsigned char *m; + unsigned char prot; int ret; unsigned long i; /* Skip test if we can't allocate a hugetlbfs page. */ - - if (!default_hsize) - SKIP(return, "Huge page size could not be determined"); + if (!hugetlb_free_default_pages()) + SKIP(return, "Not enough huge pages"); size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1646,11 +1651,20 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw. The region is backed by a single huge + * page that the device reports either at PMD or at PUD level depending + * on the configured default hugepage size. Determine that level from + * the first page and require every page in the range to match it + * exactly, so that a fragmented mapping mixing levels (or a missing + * large-page bit) is still caught and reported with its actual value. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_WRITE | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); /* Make the region read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); @@ -1661,11 +1675,17 @@ TEST_F(hmm, compound) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - /* Check what the device saw. */ + /* + * Check what the device saw after mprotect(PROT_READ). Same + * approach as above: determine the mapping level from the first + * page and require every page to match it exactly. + */ m = buffer->mirror; + prot = HMM_DMIRROR_PROT_READ | + ((m[0] & HMM_DMIRROR_PROT_PUD) ? HMM_DMIRROR_PROT_PUD : + HMM_DMIRROR_PROT_PMD); for (i = 0; i < npages; ++i) - ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | - HMM_DMIRROR_PROT_PMD); + ASSERT_EQ(m[i], prot); munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; @@ -1865,6 +1885,8 @@ TEST_F(hmm, exclusive_cow) unsigned long i; int *ptr; int ret; + pid_t pid; + int status; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); @@ -1893,14 +1915,37 @@ TEST_F(hmm, exclusive_cow) ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); - fork(); + pid = fork(); + if (pid == -1) + ASSERT_EQ(pid, 0); - /* Fault pages back to system memory and check them. */ + if (pid == 0) { + /* + * Child verifies COW independently, then _exit(0)s so it does + * not run the test teardown. A failed ASSERT_* here makes the + * harness abort() the child, so the parent sees + * !WIFEXITED(status) below and fails in turn. + */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i]++, i); + + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i + 1); + + _exit(0); + } + + /* Parent: also increment to verify COW works for both processes. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i]++, i); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) - ASSERT_EQ(ptr[i], i+1); + ASSERT_EQ(ptr[i], i + 1); + + /* Parent: wait for child and then free the buffer. */ + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(WEXITSTATUS(status), 0); hmm_buffer_free(buffer); } @@ -2062,7 +2107,7 @@ TEST_F(hmm, hmm_cow_in_device) if (pid == -1) ASSERT_EQ(pid, 0); if (!pid) { - /* Child process waits for SIGTERM from the parent. */ + /* Child process waits for SIGKILL from the parent. */ while (1) { } /* Should not reach this */ @@ -2075,10 +2120,10 @@ TEST_F(hmm, hmm_cow_in_device) ptr[i] = i; /* Terminate child and wait */ - EXPECT_EQ(0, kill(pid, SIGTERM)); + EXPECT_EQ(0, kill(pid, SIGKILL)); EXPECT_EQ(pid, waitpid(pid, &status, 0)); EXPECT_NE(0, WIFSIGNALED(status)); - EXPECT_EQ(SIGTERM, WTERMSIG(status)); + EXPECT_EQ(SIGKILL, WTERMSIG(status)); /* Take snapshot to CPU pagetables */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); @@ -2362,12 +2407,21 @@ TEST_F(hmm, migrate_partial_unmap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long unmap_size; + unsigned long offsets[3]; unsigned long i; void *old_ptr; void *map; int *ptr; int ret, j, use_thp; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + unmap_size = size / 2; + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (use_thp = 0; use_thp < 2; ++use_thp) { for (j = 0; j < ARRAY_SIZE(offsets); ++j) { @@ -2409,12 +2463,12 @@ TEST_F(hmm, migrate_partial_unmap_fault) for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); - munmap(buffer->ptr + offsets[j], ONEMEG); + munmap(buffer->ptr + offsets[j], unmap_size); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) if (i * sizeof(int) < offsets[j] || - i * sizeof(int) >= offsets[j] + ONEMEG) + i * sizeof(int) >= offsets[j] + unmap_size) ASSERT_EQ(ptr[i], i); buffer->ptr = old_ptr; @@ -2428,12 +2482,19 @@ TEST_F(hmm, migrate_remap_fault) struct hmm_buffer *buffer; unsigned long npages; unsigned long size = read_pmd_pagesize(); + unsigned long offsets[3]; unsigned long i; void *old_ptr, *new_ptr = NULL; void *map; int *ptr; int ret, j, use_thp, dont_unmap, before; - int offsets[] = { 0, 512 * ONEKB, ONEMEG }; + + if (!size) + size = TWOMEG; + + offsets[0] = 0; + offsets[1] = size / 4; + offsets[2] = size / 2; for (before = 0; before < 2; ++before) { for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) { @@ -2836,38 +2897,45 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120) { struct benchmark_results thp_results, regular_results; - size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */ + size_t thp_size = read_pmd_pagesize(); int iterations = 5; + if (!thp_size) + thp_size = TWOMEG; + printf("\nHMM THP Migration Benchmark\n"); printf("---------------------------\n"); printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE)); /* Test different buffer sizes */ size_t test_sizes[] = { - thp_size / 4, /* 512KB - smaller than THP */ - thp_size / 2, /* 1MB - half THP */ - thp_size, /* 2MB - single THP */ - thp_size * 2, /* 4MB - two THPs */ - thp_size * 4, /* 8MB - four THPs */ - thp_size * 8, /* 16MB - eight THPs */ - thp_size * 128, /* 256MB - one twenty eight THPs */ + thp_size / 4, /* quarter THP */ + thp_size / 2, /* half THP */ + thp_size, /* single THP */ + thp_size * 2, /* two THPs */ + thp_size * 4, /* four THPs */ + thp_size * 8, /* eight THPs */ + thp_size * 128, /* one twenty eight THPs */ }; static const char *const test_names[] = { - "Small Buffer (512KB)", - "Half THP Size (1MB)", - "Single THP Size (2MB)", - "Two THP Size (4MB)", - "Four THP Size (8MB)", - "Eight THP Size (16MB)", - "One twenty eight THP Size (256MB)" + "Small Buffer", + "Half THP Size", + "Single THP Size", + "Two THP Size", + "Four THP Size", + "Eight THP Size", + "One twenty eight THP Size" }; int num_tests = ARRAY_SIZE(test_sizes); /* Run all tests */ for (int i = 0; i < num_tests; i++) { + /* Skip test sizes exceeding INT_MAX to avoid overflow */ + if (test_sizes[i] > INT_MAX) + break; + /* Test with THP */ ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i], iterations, &thp_results), 0); diff --git a/tools/testing/selftests/mm/hugepage-mmap.c b/tools/testing/selftests/mm/hugepage-mmap.c deleted file mode 100644 index d543419de040..000000000000 --- a/tools/testing/selftests/mm/hugepage-mmap.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * hugepage-mmap: - * - * Example of using huge page memory in a user application using the mmap - * system call. Before running this application, make sure that the - * administrator has mounted the hugetlbfs filesystem (on some directory - * like /mnt) using the command mount -t hugetlbfs nodev /mnt. In this - * example, the app is requesting memory of size 256MB that is backed by - * huge pages. - */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include "kselftest.h" - -#define LENGTH (256UL*1024*1024) -#define PROTECTION (PROT_READ | PROT_WRITE) - -static void check_bytes(char *addr) -{ - ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); -} - -static void write_bytes(char *addr) -{ - unsigned long i; - - for (i = 0; i < LENGTH; i++) - *(addr + i) = (char)i; -} - -static int read_bytes(char *addr) -{ - unsigned long i; - - check_bytes(addr); - for (i = 0; i < LENGTH; i++) - if (*(addr + i) != (char)i) { - ksft_print_msg("Error: Mismatch at %lu\n", i); - return 1; - } - return 0; -} - -int main(void) -{ - void *addr; - int fd, ret; - - ksft_print_header(); - ksft_set_plan(1); - - fd = memfd_create("hugepage-mmap", MFD_HUGETLB); - if (fd < 0) - ksft_exit_fail_msg("memfd_create() failed: %s\n", strerror(errno)); - - addr = mmap(NULL, LENGTH, PROTECTION, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - close(fd); - ksft_exit_fail_msg("mmap(): %s\n", strerror(errno)); - } - - ksft_print_msg("Returned address is %p\n", addr); - check_bytes(addr); - write_bytes(addr); - ret = read_bytes(addr); - - munmap(addr, LENGTH); - close(fd); - - ksft_test_result(!ret, "Read same data\n"); - - ksft_exit(!ret); -} diff --git a/tools/testing/selftests/mm/thp_settings.c b/tools/testing/selftests/mm/hugepage_settings.c similarity index 58% rename from tools/testing/selftests/mm/thp_settings.c rename to tools/testing/selftests/mm/hugepage_settings.c index e748ebfb3d4e..2eab2110ac6a 100644 --- a/tools/testing/selftests/mm/thp_settings.c +++ b/tools/testing/selftests/mm/hugepage_settings.c @@ -1,13 +1,16 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include +#include +#include #include #include #include #include #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" #define MAX_SETTINGS_DEPTH 4 @@ -15,6 +18,7 @@ static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; static int settings_index; static struct thp_settings saved_settings; static char dev_queue_read_ahead_path[PATH_MAX]; +static bool thp_settings_saved; static const char * const thp_enabled_strings[] = { "never", @@ -44,47 +48,6 @@ static const char * const shmem_enabled_strings[] = { NULL }; -int read_file(const char *path, char *buf, size_t buflen) -{ - int fd; - ssize_t numread; - - fd = open(path, O_RDONLY); - if (fd == -1) - return 0; - - numread = read(fd, buf, buflen - 1); - if (numread < 1) { - close(fd); - return 0; - } - - buf[numread] = '\0'; - close(fd); - - return (unsigned int) numread; -} - -unsigned long read_num(const char *path) -{ - char buf[21]; - - if (read_file(path, buf, sizeof(buf)) < 0) { - perror("read_file()"); - exit(EXIT_FAILURE); - } - - return strtoul(buf, NULL, 10); -} - -void write_num(const char *path, unsigned long num) -{ - char buf[21]; - - sprintf(buf, "%ld", num); - write_file(path, buf, strlen(buf) + 1); -} - int thp_read_string(const char *name, const char * const strings[]) { char path[PATH_MAX]; @@ -298,12 +261,20 @@ void thp_pop_settings(void) void thp_restore_settings(void) { - thp_write_settings(&saved_settings); + if (thp_settings_saved) + thp_write_settings(&saved_settings); } -void thp_save_settings(void) +static void __thp_save_settings(void) { + if (!thp_available()) + return; + + if (thp_settings_saved) + return; + thp_read_settings(&saved_settings); + thp_settings_saved = true; } void thp_set_read_ahead_path(char *path) @@ -370,3 +341,260 @@ bool thp_is_enabled(void) /* THP is considered enabled if it's either "always" or "madvise" */ return mode == 1 || mode == 3; } + +#define HUGETLB_MAX_NR_PAGESIZES 10 +struct hugetlb_settings { + unsigned long nr_hugepages[HUGETLB_MAX_NR_PAGESIZES]; + unsigned long sizes[HUGETLB_MAX_NR_PAGESIZES]; + unsigned long default_size; + int nr_sizes; +}; + +static struct hugetlb_settings hugetlb_saved_settings; +static bool hugetlb_settings_saved; + +int detect_hugetlb_page_sizes(unsigned long sizes[], int max) +{ + static struct hugetlb_settings *settings = &hugetlb_saved_settings; + DIR *dir; + int count = 0; + + if (settings->nr_sizes) { + if (settings->nr_sizes < max) + max = settings->nr_sizes; + for (count = 0; count < max; count++) + sizes[count] = settings->sizes[count]; + return count; + } + + dir = opendir("/sys/kernel/mm/hugepages/"); + if (!dir) + return 0; + + while (count < max) { + struct dirent *entry = readdir(dir); + size_t kb; + + if (!entry) + break; + if (entry->d_type != DT_DIR) + continue; + if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) + continue; + sizes[count++] = kb * 1024; + ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", + kb); + } + closedir(dir); + return count; +} + +unsigned long default_huge_page_size(void) +{ + static struct hugetlb_settings *settings = &hugetlb_saved_settings; + unsigned long hps = 0; + char *line = NULL; + size_t linelen = 0; + FILE *f; + + if (settings->default_size) + return settings->default_size; + + f = fopen("/proc/meminfo", "r"); + if (!f) + return 0; + while (getline(&line, &linelen, f) > 0) { + if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { + hps <<= 10; + break; + } + } + + free(line); + fclose(f); + return hps; +} + +static void hugetlb_sysfs_path(char *buf, size_t buflen, + unsigned long size, const char *attr) +{ + snprintf(buf, buflen, "/sys/kernel/mm/hugepages/hugepages-%lukB/%s", + size / 1024, attr); +} + +unsigned long hugetlb_nr_pages(unsigned long size) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); + + return read_num(path); +} + +void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); + + write_num(path, nr); +} + +unsigned long hugetlb_free_pages(unsigned long size) +{ + char path[PATH_MAX]; + + hugetlb_sysfs_path(path, sizeof(path), size, "free_hugepages"); + + return read_num(path); +} + +static bool __hugetlb_setup(unsigned long size, unsigned long nr) +{ + unsigned long free = hugetlb_free_pages(size); + unsigned long total = hugetlb_nr_pages(size); + + if (free >= nr) + return true; + + hugetlb_set_nr_pages(size, total + (nr - free)); + + return hugetlb_free_pages(size) >= nr; +} + +bool hugetlb_setup_default(unsigned long nr) +{ + unsigned long size; + + hugetlb_save_settings(); + size = default_huge_page_size(); + if (!size) + return false; + + return __hugetlb_setup(size, nr); +} + +bool hugetlb_setup_default_exact(unsigned long nr) +{ + unsigned long size; + + hugetlb_save_settings(); + size = default_huge_page_size(); + if (!size) + return false; + + hugetlb_set_nr_pages(size, nr); + + return hugetlb_free_pages(size) == nr; +} + +unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], + int max) +{ + unsigned long enabled[10]; + int nr_sizes = 0; + int nr_enabled; + + hugetlb_save_settings(); + + nr_enabled = detect_hugetlb_page_sizes(enabled, ARRAY_SIZE(enabled)); + if (!nr_enabled) + return 0; + + if (nr_enabled > max) { + ksft_print_msg("detected %d huge page sizes, will only test %d\n", nr_enabled, max); + nr_enabled = max; + } + + /* request nr HugeTLB pages of every size. */ + for (int i = 0; i < nr_enabled; i++) { + if (!__hugetlb_setup(enabled[i], nr)) + continue; + sizes[nr_sizes++] = enabled[i]; + } + + return nr_sizes; +} + +static void __hugetlb_save_settings(void) +{ + struct hugetlb_settings *settings = &hugetlb_saved_settings; + int nr_sizes; + + if (hugetlb_settings_saved) + return; + + settings->default_size = default_huge_page_size(); + if (!settings->default_size) + return; + + nr_sizes = detect_hugetlb_page_sizes(settings->sizes, + HUGETLB_MAX_NR_PAGESIZES); + if (!nr_sizes) { + settings->default_size = 0; + return; + } + + for (int i = 0; i < nr_sizes; i++) { + unsigned long sz = settings->sizes[i]; + + if (!sz) + continue; + settings->nr_hugepages[i] = hugetlb_nr_pages(sz); + } + + settings->nr_sizes = nr_sizes; + hugetlb_settings_saved = true; +} + +void hugetlb_restore_settings(void) +{ + struct hugetlb_settings *settings = &hugetlb_saved_settings; + + if (!hugetlb_settings_saved || !settings->default_size) + return; + + for (int i = 0; i < HUGETLB_MAX_NR_PAGESIZES; i++) { + unsigned long sz = settings->sizes[i]; + + if (!sz) + continue; + + hugetlb_set_nr_pages(sz, settings->nr_hugepages[i]); + } +} + +static void hugepage_restore_settings_atexit(void) +{ + if (thp_settings_saved) + thp_restore_settings(); + if (hugetlb_settings_saved) + hugetlb_restore_settings(); +} + +static void hugepage_restore_settings_sighandler(int sig) +{ + /* exit() will invoke the hugepage_restore_settings_atexit handler. */ + exit(KSFT_FAIL); +} + +void hugepage_save_settings(bool thp, bool hugetlb) +{ + if (!thp && !hugetlb) + return; + + if (thp) + __thp_save_settings(); + if (hugetlb) + __hugetlb_save_settings(); + + /* + * setup exit hooks to make sure THP and HugeTLB settings are + * restored on graceful and error exits and signals + */ + atexit(hugepage_restore_settings_atexit); + signal(SIGTERM, hugepage_restore_settings_sighandler); + signal(SIGINT, hugepage_restore_settings_sighandler); + signal(SIGHUP, hugepage_restore_settings_sighandler); + signal(SIGQUIT, hugepage_restore_settings_sighandler); +} diff --git a/tools/testing/selftests/mm/thp_settings.h b/tools/testing/selftests/mm/hugepage_settings.h similarity index 51% rename from tools/testing/selftests/mm/thp_settings.h rename to tools/testing/selftests/mm/hugepage_settings.h index 7748a9009191..726c73c43c05 100644 --- a/tools/testing/selftests/mm/thp_settings.h +++ b/tools/testing/selftests/mm/hugepage_settings.h @@ -1,11 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __THP_SETTINGS_H__ -#define __THP_SETTINGS_H__ +#ifndef __HUGEPAGE_SETTINGS_H__ +#define __HUGEPAGE_SETTINGS_H__ #include #include #include +void hugepage_save_settings(bool thp, bool hugetlb); + +/* Transparent Huge Pages (THP) */ + enum thp_enabled { THP_NEVER, THP_ALWAYS, @@ -62,10 +66,6 @@ struct thp_settings { struct shmem_hugepages_settings shmem_hugepages[NR_ORDERS]; }; -int read_file(const char *path, char *buf, size_t buflen); -unsigned long read_num(const char *path); -void write_num(const char *path, unsigned long num); - int thp_read_string(const char *name, const char * const strings[]); void thp_write_string(const char *name, const char *val); unsigned long thp_read_num(const char *name); @@ -77,7 +77,11 @@ struct thp_settings *thp_current_settings(void); void thp_push_settings(struct thp_settings *settings); void thp_pop_settings(void); void thp_restore_settings(void); -void thp_save_settings(void); + +static inline void thp_save_settings(void) +{ + hugepage_save_settings(/* thp = */ true, /* hugetlb = */ false); +} void thp_set_read_ahead_path(char *path); unsigned long thp_supported_orders(void); @@ -86,4 +90,66 @@ unsigned long thp_shmem_supported_orders(void); bool thp_available(void); bool thp_is_enabled(void); -#endif /* __THP_SETTINGS_H__ */ +/* HugeTLB */ + +int detect_hugetlb_page_sizes(unsigned long sizes[], int max); +unsigned long default_huge_page_size(void); + +unsigned long hugetlb_nr_pages(unsigned long size); +void hugetlb_set_nr_pages(unsigned long size, unsigned long nr); +unsigned long hugetlb_free_pages(unsigned long size); + +static inline void hugetlb_save_settings(void) +{ + hugepage_save_settings(/* thp = */ false, /* hugetlb = */ true); +} + +void hugetlb_restore_settings(void); + +static inline unsigned long hugetlb_nr_default_pages(void) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return 0; + + return hugetlb_nr_pages(size); +} + +static inline void hugetlb_set_nr_default_pages(unsigned long nr) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return; + + hugetlb_set_nr_pages(size, nr); +} + +static inline unsigned long hugetlb_free_default_pages(void) +{ + unsigned long size = default_huge_page_size(); + + if (!size) + return 0; + + return hugetlb_free_pages(size); +} + +static inline bool hugetlb_available(void) +{ + return default_huge_page_size() != 0; +} + +bool hugetlb_setup_default(unsigned long nr); +bool hugetlb_setup_default_exact(unsigned long nr); +unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], + int max); + +#define HUGETLB_SETUP_DEFAULT_PAGES(nr_pages) \ +static void __attribute__((constructor)) __hugetlb_setup_default(void) \ +{ \ + hugetlb_setup_default((nr_pages)); \ +} + +#endif /* __HUGEPAGE_SETTINGS_H__ */ diff --git a/tools/testing/selftests/mm/hugetlb-madvise.c b/tools/testing/selftests/mm/hugetlb-madvise.c index 5b12041fa310..555b4b3d1430 100644 --- a/tools/testing/selftests/mm/hugetlb-madvise.c +++ b/tools/testing/selftests/mm/hugetlb-madvise.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-madvise: + * hugetlb-madvise: * * Basic functional testing of madvise MADV_DONTNEED and MADV_REMOVE * on hugetlb mappings. - * - * Before running this test, make sure the administrator has pre-allocated - * at least MIN_FREE_PAGES hugetlb pages and they are free. In addition, - * the test takes an argument that is the path to a file in a hugetlbfs - * filesystem. Therefore, a hugetlbfs filesystem must be mounted on some - * directory. */ #define _GNU_SOURCE @@ -20,18 +14,18 @@ #include #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #define MIN_FREE_PAGES 20 #define NR_HUGE_PAGES 10 /* common number of pages to map/allocate */ #define validate_free_pages(exp_free) \ do { \ - int fhp = get_free_hugepages(); \ - if (fhp != (exp_free)) { \ - printf("Unexpected number of free huge " \ - "pages line %d\n", __LINE__); \ - exit(1); \ - } \ + unsigned long fhp = hugetlb_free_default_pages(); \ + if (fhp != (exp_free)) \ + ksft_exit_fail_msg("Unexpected number of free " \ + "huge pages %lu, expected %lu line %d\n", \ + fhp, (exp_free), __LINE__); \ } while (0) unsigned long huge_page_size; @@ -57,28 +51,24 @@ int main(int argc, char **argv) int fd; int ret; + ksft_print_header(); + ksft_set_plan(1); + huge_page_size = default_huge_page_size(); - if (!huge_page_size) { - printf("Unable to determine huge page size, exiting!\n"); - exit(1); - } + if (!huge_page_size) + ksft_exit_skip("Unable to determine huge page size\n"); + base_page_size = sysconf(_SC_PAGE_SIZE); - if (!huge_page_size) { - printf("Unable to determine base page size, exiting!\n"); - exit(1); - } + if (!base_page_size) + ksft_exit_fail_msg("Unable to determine base page size\n"); - free_hugepages = get_free_hugepages(); - if (free_hugepages < MIN_FREE_PAGES) { - printf("Not enough free huge pages to test, exiting!\n"); - exit(KSFT_SKIP); - } + if (!hugetlb_setup_default(MIN_FREE_PAGES)) + ksft_exit_skip("Not enough free huge pages (have %lu, need %d)\n", hugetlb_free_default_pages(), MIN_FREE_PAGES); + free_hugepages = hugetlb_free_default_pages(); fd = memfd_create(argv[0], MFD_HUGETLB); - if (fd < 0) { - perror("memfd_create() failed"); - exit(1); - } + if (fd < 0) + ksft_exit_fail_perror("memfd_create"); /* * Test validity of MADV_DONTNEED addr and length arguments. mmap @@ -90,16 +80,13 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + if (munmap(addr, huge_page_size) || munmap(addr + (NR_HUGE_PAGES + 1) * huge_page_size, - huge_page_size)) { - perror("munmap"); - exit(1); - } + huge_page_size)) + ksft_exit_fail_perror("munmap"); addr = addr + huge_page_size; write_fault_pages(addr, NR_HUGE_PAGES); @@ -108,20 +95,14 @@ int main(int argc, char **argv) /* addr before mapping should fail */ ret = madvise(addr - base_page_size, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with invalid addr line %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with invalid addr unexpectedly succeeded line %d\n", __LINE__); /* addr + length after mapping should fail */ ret = madvise(addr, (NR_HUGE_PAGES * huge_page_size) + base_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with invalid length line %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with invalid length unexpectedly succeeded line %d\n", __LINE__); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); @@ -132,10 +113,9 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); @@ -143,19 +123,14 @@ int main(int argc, char **argv) ret = madvise(addr + base_page_size, NR_HUGE_PAGES * huge_page_size - base_page_size, MADV_DONTNEED); - if (!ret) { - printf("Unexpected success of madvise call with unaligned start address %d\n", - __LINE__); - exit(1); - } + if (!ret) + ksft_exit_fail_msg("madvise with unaligned start unexpectedly succeeded line %d\n", __LINE__); /* addr + length should be aligned down to huge page size */ if (madvise(addr, ((NR_HUGE_PAGES - 1) * huge_page_size) + base_page_size, - MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); /* should free all but last page in mapping */ validate_free_pages(free_hugepages - 1); @@ -170,17 +145,14 @@ int main(int argc, char **argv) PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); /* should free all pages in mapping */ validate_free_pages(free_hugepages); @@ -190,29 +162,25 @@ int main(int argc, char **argv) /* * Test MADV_DONTNEED on private mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* read should not consume any pages */ read_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* madvise should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* writes should allocate private pages */ @@ -220,10 +188,9 @@ int main(int argc, char **argv) validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise should free private pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* writes should allocate private pages */ @@ -238,10 +205,9 @@ int main(int argc, char **argv) * implementation. */ if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); @@ -249,29 +215,25 @@ int main(int argc, char **argv) /* * Test MADV_DONTNEED on shared mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* write should not consume any pages */ write_fault_pages(addr, NR_HUGE_PAGES); validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* madvise should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* @@ -279,29 +241,25 @@ int main(int argc, char **argv) * * madvise is same as hole punch and should free all pages. */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); /* * Test MADV_REMOVE on shared and private mapping of hugetlb file */ - if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) { - perror("fallocate"); - exit(1); - } + if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) + ksft_exit_fail_perror("fallocate"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* shared write should not consume any additional pages */ write_fault_pages(addr, NR_HUGE_PAGES); @@ -310,10 +268,8 @@ int main(int argc, char **argv) addr2 = mmap(NULL, NR_HUGE_PAGES * huge_page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - if (addr2 == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr2 == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* private read should not consume any pages */ read_fault_pages(addr2, NR_HUGE_PAGES); @@ -324,17 +280,15 @@ int main(int argc, char **argv) validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise of shared mapping should not free any pages */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES)); /* madvise of private mapping should free private pages */ - if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) { - perror("madvise"); - exit(1); - } + if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages - NR_HUGE_PAGES); /* private write should consume additional pages again */ @@ -346,15 +300,16 @@ int main(int argc, char **argv) * not correct. private pages should not be freed, but this is * expected. See comment associated with FALLOC_FL_PUNCH_HOLE call. */ - if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) { - perror("madvise"); - exit(1); - } + if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) + ksft_exit_fail_perror("madvise"); + validate_free_pages(free_hugepages); (void)munmap(addr, NR_HUGE_PAGES * huge_page_size); (void)munmap(addr2, NR_HUGE_PAGES * huge_page_size); close(fd); - return 0; + + ksft_test_result_pass("MADV_DONTNEED and MADV_REMOVE on hugetlb\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb-mmap.c b/tools/testing/selftests/mm/hugetlb-mmap.c new file mode 100644 index 000000000000..0f2aad1b7dbd --- /dev/null +++ b/tools/testing/selftests/mm/hugetlb-mmap.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * hugetlb-mmap: + * + * Example of using huge page memory in a user application using the mmap + * system call. Before running this application, make sure that the + * administrator has mounted the hugetlbfs filesystem (on some directory + * like /mnt) using the command mount -t hugetlbfs nodev /mnt. In this + * example, the app is requesting memory of size 256MB that is backed by + * huge pages. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "vm_util.h" +#include "kselftest.h" +#include "hugepage_settings.h" + +#define LENGTH (256UL*1024*1024) +#define PROTECTION (PROT_READ | PROT_WRITE) + +static void check_bytes(char *addr) +{ + ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); +} + +static void write_bytes(char *addr, size_t length) +{ + unsigned long i; + + for (i = 0; i < length; i++) + *(addr + i) = (char)i; +} + +static bool verify_bytes(char *addr, size_t length) +{ + unsigned long i; + + check_bytes(addr); + for (i = 0; i < length; i++) + if (*(addr + i) != (char)i) { + ksft_print_msg("Error: Mismatch at %lu(%p)\n", i, addr); + return false; + } + + return true; +} + +static void test_mmap(size_t length, int mmap_flags, int fd, + const char *test_name) +{ + bool passed = true; + void *addr; + + addr = mmap(NULL, length, PROTECTION, mmap_flags, fd, 0); + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); + + ksft_print_msg("Returned address is %p\n", addr); + check_bytes(addr); + write_bytes(addr, length); + if (!verify_bytes(addr, length)) + passed = false; + + /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ + if (munmap(addr, length)) + ksft_exit_fail_perror("munmap"); + + ksft_test_result(passed, "%s\n", test_name); +} + +static void test_anon_mmap(size_t length, int shift) +{ + const char *test_name = "hugetlb anonymous mmap"; + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB; + + if (shift) + mmap_flags |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT; + + test_mmap(length, mmap_flags, -1, test_name); +} + +static void test_file_mmap(size_t length, int shift) +{ + const char *test_name = "hugetlb file mmap"; + int mfd_flags = MFD_HUGETLB; + int fd; + + if (shift) + mfd_flags |= (shift & MFD_HUGE_MASK) << MFD_HUGE_SHIFT; + + fd = memfd_create("hugetlb-mmap", mfd_flags); + if (fd < 0) + ksft_exit_fail_perror("memfd_create"); + + test_mmap(length, MAP_SHARED, fd, test_name); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t hugepage_size; + size_t length = LENGTH; + int shift = 0, nr; + + ksft_print_header(); + + if (argc > 1) + length = atol(argv[1]) << 20; + if (argc > 2) + shift = atoi(argv[2]); + + hugetlb_save_settings(); + if (shift) { + hugepage_size = (1UL << shift); + ksft_print_msg("%lu kB hugepages\n", 1UL << (shift - 10)); + } else { + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Could not detect default hugetlb page size."); + ksft_print_msg("Default size hugepages (%lu kB)\n", hugepage_size >> 10); + } + + /* munmap will fail if the length is not page aligned */ + length = (length + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + hugetlb_set_nr_pages(hugepage_size, nr); + if (hugetlb_free_pages(hugepage_size) < nr) + ksft_exit_skip("Not enough %lu Kb pages\n", hugepage_size >> 10); + + ksft_set_plan(2); + ksft_print_msg("Mapping %lu Mbytes\n", (unsigned long)length >> 20); + + test_anon_mmap(length, shift); + test_file_mmap(length, shift); + + ksft_finished(); +} diff --git a/tools/testing/selftests/mm/hugepage-mremap.c b/tools/testing/selftests/mm/hugetlb-mremap.c similarity index 80% rename from tools/testing/selftests/mm/hugepage-mremap.c rename to tools/testing/selftests/mm/hugetlb-mremap.c index b8f7d92e5a35..ed3d92e862d8 100644 --- a/tools/testing/selftests/mm/hugepage-mremap.c +++ b/tools/testing/selftests/mm/hugetlb-mremap.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-mremap: + * hugetlb-mremap: * * Example of remapping huge page memory in a user application using the * mremap system call. The path to a file in a hugetlbfs filesystem must @@ -26,12 +26,13 @@ #include #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define DEFAULT_LENGTH_MB 10UL #define MB_TO_BYTES(x) (x * 1024 * 1024) #define PROTECTION (PROT_READ | PROT_WRITE | PROT_EXEC) -#define FLAGS (MAP_SHARED | MAP_ANONYMOUS) +#define FLAGS (MAP_HUGETLB | MAP_SHARED) static void check_bytes(char *addr) { @@ -85,31 +86,21 @@ static void register_region_with_uffd(char *addr, size_t len) if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1) ksft_exit_fail_msg("ioctl-UFFDIO_API: %s\n", strerror(errno)); - /* Create a private anonymous mapping. The memory will be - * demand-zero paged--that is, not yet allocated. When we - * actually touch the memory, it will be allocated via - * the userfaultfd. - */ - - addr = mmap(NULL, len, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); - - ksft_print_msg("Address returned by mmap() = %p\n", addr); - - /* Register the memory range of the mapping we just created for - * handling by the userfaultfd object. In mode, we request to track - * missing pages (i.e., pages that have not yet been faulted in). + /* Register the passed memory range for handling by the userfaultfd object. + * In mode, we request to track missing pages + * (i.e., pages that have not yet been faulted in). */ if (uffd_register(uffd, addr, len, true, false, false)) ksft_exit_fail_msg("ioctl-UFFDIO_REGISTER: %s\n", strerror(errno)); + + ksft_print_msg("Registered memory at address %p with userfaultfd\n", addr); } int main(int argc, char *argv[]) { + unsigned long hugepage_size; + int ret = 0, fd, nr; size_t length = 0; - int ret = 0, fd; ksft_print_header(); ksft_set_plan(1); @@ -125,30 +116,36 @@ int main(int argc, char *argv[]) else length = DEFAULT_LENGTH_MB; + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Could not detect default hugetlb page size\n"); length = MB_TO_BYTES(length); + length = (length + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Not enough huge pages\n"); + fd = memfd_create(argv[0], MFD_HUGETLB); if (fd < 0) ksft_exit_fail_msg("Open failed: %s\n", strerror(errno)); /* mmap to a PUD aligned address to hopefully trigger pmd sharing. */ unsigned long suggested_addr = 0x7eaa40000000; - void *haddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *haddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map haddr: Returned address is %p\n", haddr); if (haddr == MAP_FAILED) ksft_exit_fail_msg("mmap1: %s\n", strerror(errno)); /* mmap again to a dummy address to hopefully trigger pmd sharing. */ suggested_addr = 0x7daa40000000; - void *daddr = mmap((void *)suggested_addr, length, PROTECTION, - MAP_HUGETLB | MAP_SHARED | MAP_POPULATE, fd, 0); + void *daddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map daddr: Returned address is %p\n", daddr); if (daddr == MAP_FAILED) ksft_exit_fail_msg("mmap3: %s\n", strerror(errno)); suggested_addr = 0x7faa40000000; - void *vaddr = - mmap((void *)suggested_addr, length, PROTECTION, FLAGS, -1, 0); + void *vaddr = mmap((void *)suggested_addr, length, PROTECTION, FLAGS, fd, 0); ksft_print_msg("Map vaddr: Returned address is %p\n", vaddr); if (vaddr == MAP_FAILED) ksft_exit_fail_msg("mmap2: %s\n", strerror(errno)); diff --git a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c index 46230462ad48..70b24e3660c4 100644 --- a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c +++ b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c @@ -10,12 +10,10 @@ #include #include #include +#include #include "kselftest.h" -#define PREFIX " ... " -#define ERROR_PREFIX " !!! " - #define MAX_WRITE_READ_CHUNK_SIZE (getpagesize() * 16) #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -25,17 +23,22 @@ enum test_status { TEST_SKIPPED = 2, }; -static char *status_to_str(enum test_status status) +static void report_status(enum test_status status, const char *test_name, + size_t chunk_size) { switch (status) { case TEST_PASSED: - return "TEST_PASSED"; + ksft_test_result_pass("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; case TEST_FAILED: - return "TEST_FAILED"; + ksft_test_result_fail("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; case TEST_SKIPPED: - return "TEST_SKIPPED"; - default: - return "TEST_???"; + ksft_test_result_skip("%s chunk_size=0x%lx\n", + test_name, chunk_size); + break; } } @@ -58,8 +61,8 @@ static bool verify_chunk(char *buf, size_t len, char val) for (i = 0; i < len; ++i) { if (buf[i] != val) { - printf(PREFIX ERROR_PREFIX "check fail: buf[%lu] = %u != %u\n", - i, buf[i], val); + ksft_print_msg("check fail: buf[%lu] = %u != %u\n", + i, buf[i], val); return false; } } @@ -75,21 +78,21 @@ static bool seek_read_hugepage_filemap(int fd, size_t len, size_t wr_chunk_size, ssize_t total_ret_count = 0; char val = offset / wr_chunk_size + offset % wr_chunk_size; - printf(PREFIX PREFIX "init val=%u with offset=0x%lx\n", val, offset); - printf(PREFIX PREFIX "expect to read 0x%lx bytes of data in total\n", - expected); + ksft_print_msg("init val=%u with offset=0x%lx\n", val, offset); + ksft_print_msg("expect to read 0x%lx bytes of data in total\n", + expected); if (lseek(fd, offset, SEEK_SET) < 0) { - perror(PREFIX ERROR_PREFIX "seek failed"); + ksft_perror("seek failed"); return false; } while (offset + total_ret_count < len) { ret_count = read(fd, buf, wr_chunk_size); if (ret_count == 0) { - printf(PREFIX PREFIX "read reach end of the file\n"); + ksft_print_msg("read reach end of the file\n"); break; } else if (ret_count < 0) { - perror(PREFIX ERROR_PREFIX "read failed"); + ksft_perror("read failed"); break; } ++val; @@ -98,8 +101,8 @@ static bool seek_read_hugepage_filemap(int fd, size_t len, size_t wr_chunk_size, total_ret_count += ret_count; } - printf(PREFIX PREFIX "actually read 0x%lx bytes of data in total\n", - total_ret_count); + ksft_print_msg("actually read 0x%lx bytes of data in total\n", + total_ret_count); return total_ret_count == expected; } @@ -112,15 +115,15 @@ static bool read_hugepage_filemap(int fd, size_t len, ssize_t total_ret_count = 0; char val = 0; - printf(PREFIX PREFIX "expect to read 0x%lx bytes of data in total\n", - expected); + ksft_print_msg("expect to read 0x%lx bytes of data in total\n", + expected); while (total_ret_count < len) { ret_count = read(fd, buf, wr_chunk_size); if (ret_count == 0) { - printf(PREFIX PREFIX "read reach end of the file\n"); + ksft_print_msg("read reach end of the file\n"); break; } else if (ret_count < 0) { - perror(PREFIX ERROR_PREFIX "read failed"); + ksft_perror("read failed"); break; } ++val; @@ -129,8 +132,8 @@ static bool read_hugepage_filemap(int fd, size_t len, total_ret_count += ret_count; } - printf(PREFIX PREFIX "actually read 0x%lx bytes of data in total\n", - total_ret_count); + ksft_print_msg("actually read 0x%lx bytes of data in total\n", + total_ret_count); return total_ret_count == expected; } @@ -142,14 +145,14 @@ test_hugetlb_read(int fd, size_t len, size_t wr_chunk_size) char *filemap = NULL; if (ftruncate(fd, len) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate failed"); + ksft_perror("ftruncate failed"); return status; } filemap = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); if (filemap == MAP_FAILED) { - perror(PREFIX ERROR_PREFIX "mmap for primary mapping failed"); + ksft_perror("mmap for primary mapping failed"); goto done; } @@ -162,7 +165,7 @@ test_hugetlb_read(int fd, size_t len, size_t wr_chunk_size) munmap(filemap, len); done: if (ftruncate(fd, 0) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate back to 0 failed"); + ksft_perror("ftruncate back to 0 failed"); status = TEST_FAILED; } @@ -179,14 +182,14 @@ test_hugetlb_read_hwpoison(int fd, size_t len, size_t wr_chunk_size, const unsigned long pagesize = getpagesize(); if (ftruncate(fd, len) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate failed"); + ksft_perror("ftruncate failed"); return status; } filemap = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0); if (filemap == MAP_FAILED) { - perror(PREFIX ERROR_PREFIX "mmap for primary mapping failed"); + ksft_perror("mmap for primary mapping failed"); goto done; } @@ -201,7 +204,7 @@ test_hugetlb_read_hwpoison(int fd, size_t len, size_t wr_chunk_size, */ hwp_addr = filemap + len / 2 + pagesize; if (madvise(hwp_addr, pagesize, MADV_HWPOISON) < 0) { - perror(PREFIX ERROR_PREFIX "MADV_HWPOISON failed"); + ksft_perror("MADV_HWPOISON failed"); goto unmap; } @@ -228,7 +231,7 @@ test_hugetlb_read_hwpoison(int fd, size_t len, size_t wr_chunk_size, munmap(filemap, len); done: if (ftruncate(fd, 0) < 0) { - perror(PREFIX ERROR_PREFIX "ftruncate back to 0 failed"); + ksft_perror("ftruncate back to 0 failed"); status = TEST_FAILED; } @@ -241,17 +244,17 @@ static int create_hugetlbfs_file(struct statfs *file_stat) fd = memfd_create("hugetlb_tmp", MFD_HUGETLB); if (fd < 0) { - perror(PREFIX ERROR_PREFIX "could not open hugetlbfs file"); + ksft_perror("could not open hugetlbfs file"); return -1; } memset(file_stat, 0, sizeof(*file_stat)); if (fstatfs(fd, file_stat)) { - perror(PREFIX ERROR_PREFIX "fstatfs failed"); + ksft_perror("fstatfs failed"); goto close; } if (file_stat->f_type != HUGETLBFS_MAGIC) { - printf(PREFIX ERROR_PREFIX "not hugetlbfs file\n"); + ksft_print_msg("not hugetlbfs file\n"); goto close; } @@ -261,6 +264,10 @@ static int create_hugetlbfs_file(struct statfs *file_stat) return -1; } +static void sigbus_handler(int sig) +{ +} + int main(void) { int fd; @@ -273,50 +280,44 @@ int main(void) }; size_t i; + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(wr_chunk_sizes) * 3); + + signal(SIGBUS, sigbus_handler); for (i = 0; i < ARRAY_SIZE(wr_chunk_sizes); ++i) { - printf("Write/read chunk size=0x%lx\n", - wr_chunk_sizes[i]); + ksft_print_msg("Write/read chunk size=0x%lx\n", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB read regression test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read(fd, file_stat.f_bsize, wr_chunk_sizes[i]); - printf(PREFIX "HugeTLB read regression test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB read regression", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB read HWPOISON test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read_hwpoison(fd, file_stat.f_bsize, wr_chunk_sizes[i], false); - printf(PREFIX "HugeTLB read HWPOISON test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB read HWPOISON", + wr_chunk_sizes[i]); fd = create_hugetlbfs_file(&file_stat); if (fd < 0) - goto create_failure; - printf(PREFIX "HugeTLB seek then read HWPOISON test...\n"); + ksft_exit_fail_msg("Failed to create hugetlbfs file\n"); + status = test_hugetlb_read_hwpoison(fd, file_stat.f_bsize, wr_chunk_sizes[i], true); - printf(PREFIX "HugeTLB seek then read HWPOISON test...%s\n", - status_to_str(status)); close(fd); - if (status == TEST_FAILED) - return -1; + report_status(status, "HugeTLB seek then read HWPOISON", + wr_chunk_sizes[i]); } - return 0; - -create_failure: - printf(ERROR_PREFIX "Abort test: failed to create hugetlbfs file\n"); - return -1; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugepage-shm.c b/tools/testing/selftests/mm/hugetlb-shm.c similarity index 55% rename from tools/testing/selftests/mm/hugepage-shm.c rename to tools/testing/selftests/mm/hugetlb-shm.c index ef06260802b5..3ff7f062b7eb 100644 --- a/tools/testing/selftests/mm/hugepage-shm.c +++ b/tools/testing/selftests/mm/hugetlb-shm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * hugepage-shm: + * hugetlb-shm: * * Example of using huge page memory in a user application using Sys V shared * memory system calls. In this example the app is requesting 256MB of @@ -28,9 +28,27 @@ #include #include +#include "vm_util.h" +#include "hugepage_settings.h" + #define LENGTH (256UL*1024*1024) -#define dprintf(x) printf(x) +static void prepare(void) +{ + unsigned long length, hugepage_size, nr; + + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + ksft_exit_skip("Unable to determine huge page size\n"); + + length = (LENGTH + hugepage_size - 1) & ~(hugepage_size - 1); + nr = length / hugepage_size; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Not enough free huge pages\n"); + + shm_limits_prepare(length); +} int main(void) { @@ -38,44 +56,45 @@ int main(void) unsigned long i; char *shmaddr; + ksft_print_header(); + ksft_set_plan(1); + + prepare(); + shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - if (shmid < 0) { - perror("shmget"); - exit(1); - } - printf("shmid: 0x%x\n", shmid); + if (shmid < 0) + ksft_exit_fail_perror("shmget"); + + ksft_print_msg("shmid: 0x%x\n", shmid); shmaddr = shmat(shmid, NULL, 0); if (shmaddr == (char *)-1) { - perror("Shared memory attach failure"); + ksft_perror("Shared memory attach failure"); shmctl(shmid, IPC_RMID, NULL); - exit(2); + ksft_exit_fail(); } - printf("shmaddr: %p\n", shmaddr); + ksft_print_msg("shmaddr: %p\n", shmaddr); - dprintf("Starting the writes:\n"); - for (i = 0; i < LENGTH; i++) { + ksft_print_msg("Starting the writes:\n"); + for (i = 0; i < LENGTH; i++) shmaddr[i] = (char)(i); - if (!(i % (1024 * 1024))) - dprintf("."); - } - dprintf("\n"); - dprintf("Starting the Check..."); + ksft_print_msg("Starting the Check..."); for (i = 0; i < LENGTH; i++) - if (shmaddr[i] != (char)i) { - printf("\nIndex %lu mismatched\n", i); - exit(3); - } - dprintf("Done.\n"); + if (shmaddr[i] != (char)i) + ksft_exit_fail_msg("Data mismatch at index %lu\n", i); + ksft_print_msg("Done.\n"); if (shmdt((const void *)shmaddr) != 0) { - perror("Detach failure"); + ksft_perror("Detach failure"); shmctl(shmid, IPC_RMID, NULL); - exit(4); + ksft_exit_fail(); } shmctl(shmid, IPC_RMID, NULL); - return 0; + ksft_test_result_pass("hugepage using SysV shmget/shmat\n"); + ksft_finished(); } + +SHM_LIMITS_RESTORE() diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index a8bc02688085..bc202e4ed2bd 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -6,9 +6,7 @@ * - if enable_soft_offline = 1, a hugepage should be dissolved and * nr_hugepages/free_hugepages should be reduced by 1. * - * Before running, make sure more than 2 hugepages of default_hugepagesz - * are allocated. For example, if /proc/meminfo/Hugepagesize is 2048kB: - * echo 8 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + * The test allocates 8 default hugepages */ #define _GNU_SOURCE @@ -25,6 +23,7 @@ #include #include "kselftest.h" +#include "hugepage_settings.h" #ifndef MADV_SOFT_OFFLINE #define MADV_SOFT_OFFLINE 101 @@ -100,32 +99,6 @@ static int set_enable_soft_offline(int value) return 0; } -static int read_nr_hugepages(unsigned long hugepage_size, - unsigned long *nr_hugepages) -{ - char buffer[256] = {0}; - char cmd[256] = {0}; - - sprintf(cmd, "cat /sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages", - hugepage_size); - FILE *cmdfile = popen(cmd, "r"); - - if (cmdfile == NULL) { - ksft_perror(EPREFIX "failed to popen nr_hugepages"); - return -1; - } - - if (!fgets(buffer, sizeof(buffer), cmdfile)) { - ksft_perror(EPREFIX "failed to read nr_hugepages"); - pclose(cmdfile); - return -1; - } - - *nr_hugepages = atoll(buffer); - pclose(cmdfile); - return 0; -} - static int create_hugetlbfs_file(struct statfs *file_stat) { int fd; @@ -177,20 +150,14 @@ static void test_soft_offline_common(int enable_soft_offline) ksft_exit_fail_msg("Failed to set enable_soft_offline\n"); } - if (read_nr_hugepages(hugepagesize_kb, &nr_hugepages_before) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to read nr_hugepages\n"); - } + nr_hugepages_before = hugetlb_nr_default_pages(); ksft_print_msg("Before MADV_SOFT_OFFLINE nr_hugepages=%ld\n", nr_hugepages_before); ret = do_soft_offline(fd, 2 * file_stat.f_bsize, expect_errno); - if (read_nr_hugepages(hugepagesize_kb, &nr_hugepages_after) != 0) { - close(fd); - ksft_exit_fail_msg("Failed to read nr_hugepages\n"); - } + nr_hugepages_after = hugetlb_nr_default_pages(); ksft_print_msg("After MADV_SOFT_OFFLINE nr_hugepages=%ld\n", nr_hugepages_after); @@ -219,6 +186,10 @@ static void test_soft_offline_common(int enable_soft_offline) int main(int argc, char **argv) { ksft_print_header(); + + if (!hugetlb_setup_default(8)) + ksft_exit_skip("not enough hugetlb pages\n"); + ksft_set_plan(2); test_soft_offline_common(1); diff --git a/tools/testing/selftests/mm/hugepage-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c similarity index 76% rename from tools/testing/selftests/mm/hugepage-vmemmap.c rename to tools/testing/selftests/mm/hugetlb-vmemmap.c index df366a4d1b92..507df78a158d 100644 --- a/tools/testing/selftests/mm/hugepage-vmemmap.c +++ b/tools/testing/selftests/mm/hugetlb-vmemmap.c @@ -11,6 +11,7 @@ #include #include #include "vm_util.h" +#include "hugepage_settings.h" #define PAGE_COMPOUND_HEAD (1UL << 15) #define PAGE_COMPOUND_TAIL (1UL << 16) @@ -63,7 +64,7 @@ static int check_page_flags(unsigned long pfn) read(fd, &pageflags, sizeof(pageflags)); if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) { close(fd); - printf("Head page flags (%lx) is invalid\n", pageflags); + ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags); return -1; } @@ -77,7 +78,7 @@ static int check_page_flags(unsigned long pfn) if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS || (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) { close(fd); - printf("Tail page flags (%lx) is invalid\n", pageflags); + ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags); return -1; } } @@ -91,44 +92,41 @@ int main(int argc, char **argv) { void *addr; unsigned long pfn; + int ret; + + ksft_print_header(); + ksft_set_plan(1); + + if (!hugetlb_setup_default(1)) + ksft_exit_skip("Not enough free huge pages\n"); pagesize = psize(); maplength = default_huge_page_size(); - if (!maplength) { - printf("Unable to determine huge page size\n"); - exit(1); - } + if (!maplength) + ksft_exit_skip("Unable to determine huge page size\n"); addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if (addr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (addr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); /* Trigger allocation of HugeTLB page. */ write_bytes(addr, maplength); pfn = virt_to_pfn(addr); if (pfn == -1UL) { + ksft_perror("virt_to_pfn"); munmap(addr, maplength); - perror("virt_to_pfn"); - exit(1); + ksft_exit_fail(); } - printf("Returned address is %p whose pfn is %lx\n", addr, pfn); + ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn); - if (check_page_flags(pfn) < 0) { - munmap(addr, maplength); - perror("check_page_flags"); - exit(1); - } + ret = check_page_flags(pfn); - /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ - if (munmap(addr, maplength)) { - perror("munmap"); - exit(1); - } + if (munmap(addr, maplength)) + ksft_exit_fail_perror("munmap"); - return 0; + ksft_test_result(!ret, "HugeTLB vmemmap page flags\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb_dio.c b/tools/testing/selftests/mm/hugetlb_dio.c index 31a054fa8134..fb4600570e13 100644 --- a/tools/testing/selftests/mm/hugetlb_dio.c +++ b/tools/testing/selftests/mm/hugetlb_dio.c @@ -20,6 +20,7 @@ #include #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #ifndef STATX_DIOALIGN #define STATX_DIOALIGN 0x00002000U @@ -84,19 +85,13 @@ static void run_dio_using_hugetlb(int fd, unsigned int start_off, /* Get the default huge page size */ h_pagesize = default_huge_page_size(); - if (!h_pagesize) - ksft_exit_fail_msg("Unable to determine huge page size\n"); /* Reset file position since fd is shared across tests */ if (lseek(fd, 0, SEEK_SET) < 0) ksft_exit_fail_perror("lseek failed\n"); /* Get the free huge pages before allocation */ - free_hpage_b = get_free_hugepages(); - if (free_hpage_b == 0) { - close(fd); - ksft_exit_skip("No free hugepage, exiting!\n"); - } + free_hpage_b = hugetlb_free_default_pages(); /* Allocate a hugetlb page */ orig_buffer = mmap(NULL, h_pagesize, mmap_prot, mmap_flags, -1, 0); @@ -120,7 +115,7 @@ static void run_dio_using_hugetlb(int fd, unsigned int start_off, munmap(orig_buffer, h_pagesize); /* Get the free huge pages after unmap*/ - free_hpage_a = get_free_hugepages(); + free_hpage_a = hugetlb_free_default_pages(); ksft_print_msg("No. Free pages before allocation : %d\n", free_hpage_b); ksft_print_msg("No. Free pages after munmap : %d\n", free_hpage_a); @@ -140,8 +135,8 @@ int main(void) ksft_print_header(); - /* Check if huge pages are free */ - if (!get_free_hugepages()) + /* request a huge page */ + if (!hugetlb_setup_default(1)) ksft_exit_skip("No free hugepage, exiting\n"); fd = open("/tmp", O_TMPFILE | O_RDWR | O_DIRECT, 0664); diff --git a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c index b4b257775b74..2dc158054f66 100644 --- a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c +++ b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c @@ -10,6 +10,7 @@ #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #define INLOOP_ITER 100 @@ -53,7 +54,6 @@ void *madv(void *unused) int main(void) { - unsigned long free_hugepages; pthread_t thread1, thread2; /* * On kernel 6.4, we are able to reproduce the problem with ~1000 @@ -77,11 +77,8 @@ int main(void) ksft_print_msg("[INFO] detected default hugetlb page size: %zu KiB\n", huge_page_size / 1024); - free_hugepages = get_free_hugepages(); - if (free_hugepages != 1) { - ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - free_hugepages); - } + if (!hugetlb_setup_default(1)) + ksft_exit_skip("Not enough HugeTLB pages\n"); while (max--) { huge_ptr = mmap(NULL, huge_page_size, PROT_READ | PROT_WRITE, diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index efd774b41389..f94549efcc6f 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -25,7 +25,7 @@ #include #include "vm_util.h" -#include "kselftest.h" +#include "hugepage_settings.h" #define INLOOP_ITER 100 @@ -77,7 +77,6 @@ void *map_extra(void *unused) int main(void) { pthread_t thread1, thread2, thread3; - unsigned long free_hugepages; void *ret; /* @@ -86,12 +85,12 @@ int main(void) */ int max = 10; - free_hugepages = get_free_hugepages(); + ksft_print_header(); + ksft_set_plan(1); - if (free_hugepages != 1) { + if (!hugetlb_setup_default_exact(1)) ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n", - free_hugepages); - } + hugetlb_free_default_pages()); mmap_size = default_huge_page_size(); @@ -100,10 +99,8 @@ int main(void) MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); - if ((unsigned long)huge_ptr == -1) { - ksft_test_result_fail("Failed to allocate huge page\n"); - return KSFT_FAIL; - } + if ((unsigned long)huge_ptr == -1) + ksft_exit_fail_msg("Failed to allocate huge page\n"); pthread_create(&thread1, NULL, madv, NULL); pthread_create(&thread2, NULL, touch, NULL); @@ -115,12 +112,13 @@ int main(void) if (ret) { ksft_test_result_fail("Unexpected huge page allocation\n"); - return KSFT_FAIL; + ksft_finished(); } /* Unmap and restart */ munmap(huge_ptr, mmap_size); } - return KSFT_PASS; + ksft_test_result_pass("No unexpected huge page allocations\n"); + ksft_finished(); } diff --git a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh index 0dd31892ff67..95f517c3bd16 100755 --- a/tools/testing/selftests/mm/hugetlb_reparenting_test.sh +++ b/tools/testing/selftests/mm/hugetlb_reparenting_test.sh @@ -12,6 +12,8 @@ if [[ $(id -u) -ne 0 ]]; then fi nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) +trap 'echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages' EXIT INT TERM + usage_file=usage_in_bytes if [[ "$1" == "-cgroup-v2" ]]; then @@ -46,6 +48,13 @@ function get_machine_hugepage_size() { } MB=$(get_machine_hugepage_size) +if (( MB >= 1024 )); then + UNIT="GB" + MB_DISPLAY=$((MB / 1024)) +else + UNIT="MB" + MB_DISPLAY=$MB +fi function cleanup() { echo cleanup @@ -56,7 +65,6 @@ function cleanup() { rmdir "$CGROUP_ROOT"/a/b 2>/dev/null rmdir "$CGROUP_ROOT"/a 2>/dev/null rmdir "$CGROUP_ROOT"/test1 2>/dev/null - echo $nr_hugepgs >/proc/sys/vm/nr_hugepages set -e } @@ -87,6 +95,7 @@ function assert_with_retry() { if [[ $elapsed -ge $timeout ]]; then echo "actual = $((${actual%% *} / 1024 / 1024)) MB" echo "expected = $((${expected%% *} / 1024 / 1024)) MB" + echo FAIL cleanup exit 1 fi @@ -96,22 +105,19 @@ function assert_with_retry() { } function assert_state() { - local expected_a="$1" - local expected_a_hugetlb="$2" - local expected_b="" + local expected_a_hugetlb="$1" local expected_b_hugetlb="" - if [ ! -z ${3:-} ] && [ ! -z ${4:-} ]; then - expected_b="$3" - expected_b_hugetlb="$4" + if [ ! -z ${2:-} ]; then + expected_b_hugetlb="$2" fi - assert_with_retry "$CGROUP_ROOT/a/memory.$usage_file" "$expected_a" - assert_with_retry "$CGROUP_ROOT/a/hugetlb.${MB}MB.$usage_file" "$expected_a_hugetlb" + assert_with_retry \ + "$CGROUP_ROOT/a/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_a_hugetlb" - if [[ -n "$expected_b" && -n "$expected_b_hugetlb" ]]; then - assert_with_retry "$CGROUP_ROOT/a/b/memory.$usage_file" "$expected_b" - assert_with_retry "$CGROUP_ROOT/a/b/hugetlb.${MB}MB.$usage_file" "$expected_b_hugetlb" + if [[ -n "$expected_b_hugetlb" ]]; then + assert_with_retry \ + "$CGROUP_ROOT/a/b/hugetlb.${MB_DISPLAY}${UNIT}.$usage_file" "$expected_b_hugetlb" fi } @@ -143,18 +149,17 @@ write_hugetlbfs() { local size="$3" if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/$cgroup/cgroup.procs + cg_file="$CGROUP_ROOT/$cgroup/cgroup.procs" else echo 0 >$CGROUP_ROOT/$cgroup/cpuset.mems echo 0 >$CGROUP_ROOT/$cgroup/cpuset.cpus - echo $$ >"$CGROUP_ROOT/$cgroup/tasks" - fi - ./write_to_hugetlbfs -p "$path" -s "$size" -m 0 -o - if [[ $cgroup2 ]]; then - echo $$ >$CGROUP_ROOT/cgroup.procs - else - echo $$ >"$CGROUP_ROOT/tasks" + cg_file="$CGROUP_ROOT/$cgroup/tasks" fi + + # Spawn helper to join cgroup before exec to ensure correct cgroup accounting + bash -c 'echo $$ > "$1"; exec ./write_to_hugetlbfs -p "$2" -s "$3" -m 0 -o' _ \ + "$cg_file" "$path" "$size" & pid=$! + wait "$pid" echo } @@ -192,21 +197,21 @@ if [[ ! $cgroup2 ]]; then write_hugetlbfs a "$MNT"/test $size echo Assert memory charged correctly for parent use. - assert_state 0 $size 0 0 + assert_state $size 0 write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child use. - assert_state 0 $(($size * 2)) 0 $size + assert_state $(($size * 2)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. - assert_state 0 $(($size * 2)) + assert_state $(($size * 2)) rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. - assert_state 0 0 + assert_state 0 cleanup fi @@ -220,16 +225,16 @@ echo write write_hugetlbfs a/b "$MNT"/test2 $size echo Assert memory charged correctly for child only use. -assert_state 0 $(($size)) 0 $size +assert_state $(($size)) $size rmdir "$CGROUP_ROOT"/a/b echo Assert memory reparent correctly. -assert_state 0 $size +assert_state $size rm -rf "$MNT"/* umount "$MNT" echo Assert memory uncharged correctly. -assert_state 0 0 +assert_state 0 cleanup @@ -240,4 +245,3 @@ if [[ $do_umount ]]; then rm -rf $CGROUP_ROOT fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages diff --git a/tools/testing/selftests/mm/hwpoison-panic.sh b/tools/testing/selftests/mm/hwpoison-panic.sh new file mode 100755 index 000000000000..d953d1367332 --- /dev/null +++ b/tools/testing/selftests/mm/hwpoison-panic.sh @@ -0,0 +1,255 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Verify vm.panic_on_unrecoverable_memory_failure by injecting a hwpoison +# error on a kernel-owned page and confirming the kernel panics. +# +# Three "kinds" of kernel-owned page can be targeted, selectable via the +# first positional argument (default: rodata): +# +# rodata - a PG_reserved page in the kernel rodata range +# (sourced from /proc/iomem "Kernel rodata"). Exercises +# memory_failure() -> get_any_page() on a PageReserved page. +# +# slab - a slab page found via /proc/kpageflags (KPF_SLAB). +# Exercises memory_failure() -> get_any_page() on a non +# PG_reserved kernel-owned page. This path is what catches +# regressions where get_any_page() collapses kernel-owned +# pages into a transient -EIO instead of -ENOTRECOVERABLE. +# +# pgtable - a page-table page found via /proc/kpageflags (KPF_PGTABLE). +# Same path as slab, different page type. +# +# This test is DESTRUCTIVE: a successful run crashes the kernel. It is +# meant to be executed inside a disposable VM (e.g. virtme-ng) with a +# serial console captured by the harness. It is skipped unless the +# caller opts in via RUN_DESTRUCTIVE=1. +# +# Test passes externally: the kernel must panic with +# "Memory failure: : unrecoverable page" +# A return from the inject means no panic fired: that is a failure, +# unless the target PFN raced to a different page type before injection, +# in which case the run is inconclusive and is skipped. +# +# Author: Breno Leitao + +set -u + +# KTAP output helpers (ktap_print_msg, ktap_skip_all, ktap_exit_fail_msg, ...). +DIR="$(dirname "$(readlink -f "$0")")" +# shellcheck source=../kselftest/ktap_helpers.sh +source "${DIR}"/../kselftest/ktap_helpers.sh + +sysctl_path=/proc/sys/vm/panic_on_unrecoverable_memory_failure +inject_path=/sys/devices/system/memory/hard_offline_page +kpageflags_path=/proc/kpageflags +unpoison_path=/sys/kernel/debug/hwpoison/unpoison-pfn + +# /proc/kpageflags bit positions (see include/uapi/linux/kernel-page-flags.h) +KPF_SLAB=7 +KPF_COMPOUND_TAIL=16 +KPF_HWPOISON=19 +KPF_NOPAGE=20 +KPF_PGTABLE=26 +KPF_RESERVED=32 + +pagesize=$(getconf PAGE_SIZE) + +kind=${1:-rodata} + +if [ "$(id -u)" -ne 0 ]; then + ktap_skip_all "must run as root" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$sysctl_path" ]; then + ktap_skip_all "$sysctl_path not present (kernel without the sysctl?)" + exit "$KSFT_SKIP" +fi + +if [ ! -w "$inject_path" ]; then + ktap_skip_all "$inject_path not present (no MEMORY_HOTPLUG?)" + exit "$KSFT_SKIP" +fi + +if [ "${RUN_DESTRUCTIVE:-0}" != "1" ]; then + ktap_skip_all "destructive test; re-run with RUN_DESTRUCTIVE=1 inside a disposable VM" + exit "$KSFT_SKIP" +fi + +# Pick a PFN inside the kernel image rodata region of /proc/iomem. +# This is preferred over a top-level "Reserved" entry because top-level +# Reserved ranges are often firmware holes that have no backing struct +# page; pfn_to_online_page() returns NULL on those and memory_failure() +# bails out with -ENXIO before reaching the panic path. +# +# "Kernel rodata" is reported as a sub-resource of "System RAM" on every +# major architecture, which guarantees: +# - the PFN is backed by struct page (within an online memory range); +# - PG_reserved is set on the page (kernel image area); +# - the memory is read-only, so setting PG_hwpoison on it does not +# corrupt writable kernel state if the panic somehow does not fire. +# +# /proc/iomem entries look like (indented for sub-resources): +# " 02500000-02ffffff : Kernel rodata" +pick_rodata_phys_addr() { + awk -v pagesize="$(getconf PAGE_SIZE)" ' + # Convert a hex string to a number without relying on the gawk-only + # strtonum(). mawk lacks it and would otherwise spuriously skip + # this test on distros that ship mawk as /usr/bin/awk. + function hex2num(s, n, i, c, v) { + n = 0 + for (i = 1; i <= length(s); i++) { + c = tolower(substr(s, i, 1)) + v = index("0123456789abcdef", c) - 1 + if (v < 0) + return -1 + n = n * 16 + v + } + return n + } + /: Kernel rodata[[:space:]]*$/ { + sub(/^[[:space:]]+/, "") + n = split($0, a, /[- ]/) + start = hex2num(a[1]) + end = hex2num(a[2]) + if (end <= start) + next + # Page-align upward and emit the first byte of that page. + pfn = int((start + pagesize - 1) / pagesize) + printf "0x%x\n", pfn * pagesize + exit 0 + } + ' /proc/iomem +} + +# Walk /proc/kpageflags and return the phys addr of the first PFN that +# has bit $1 set, with KPF_HWPOISON, KPF_NOPAGE and KPF_COMPOUND_TAIL +# all clear (so we attack a real, non-tail, not-already-poisoned page). +# +# We skip the first 16 MiB of PFNs to step past low-memory special +# ranges (BIOS/EFI/ACPI/etc.) that often are PG_reserved and would not +# exhibit the slab/pgtable type we are looking for. +pick_kpageflags_phys_addr() { + local want_bit=$1 + local pagesize skip_pfn + + [ -r "$kpageflags_path" ] || return + + pagesize=$(getconf PAGE_SIZE) + skip_pfn=$(((16 * 1024 * 1024) / pagesize)) + + od -An -tx8 -v -w8 -j "$((skip_pfn * 8))" "$kpageflags_path" 2>/dev/null | \ + awk -v want_bit="$want_bit" \ + -v hwp_bit="$KPF_HWPOISON" \ + -v nopage_bit="$KPF_NOPAGE" \ + -v tail_bit="$KPF_COMPOUND_TAIL" \ + -v base_pfn="$skip_pfn" \ + -v pagesize="$pagesize" ' + # Test whether bit "b" is set in the 16-hex-digit value "hex". + # Done with substring + per-digit lookup so we never rely on awk + # bitwise operators (mawk lacks them), 64-bit FP precision or the + # gawk-only strtonum(). + function bit_set(hex, b, di, bi, c, v) { + di = int(b / 4) + bi = b - di * 4 + c = substr(hex, length(hex) - di, 1) + v = index("0123456789abcdef", tolower(c)) - 1 + if (bi == 0) return (v % 2) == 1 + if (bi == 1) return int(v / 2) % 2 == 1 + if (bi == 2) return int(v / 4) % 2 == 1 + return int(v / 8) % 2 == 1 + } + { + gsub(/^[[:space:]]+/, "") + h = $1 + if (bit_set(h, want_bit) && + !bit_set(h, hwp_bit) && + !bit_set(h, nopage_bit) && + !bit_set(h, tail_bit)) { + pfn = base_pfn + NR - 1 + printf "0x%x\n", pfn * pagesize + exit 0 + } + } + ' +} + +# Return 0 if /proc/kpageflags bit $2 is set for PFN $1, 1 if it is +# clear, or 2 if the word cannot be read. Used to re-confirm the target +# page type after a non-panicking inject. +kpageflags_bit_set() { + local word + + word=$(od -An -tx8 -v -j "$(($1 * 8))" -N 8 "$kpageflags_path" 2>/dev/null | tr -d '[:space:]') + [ -n "$word" ] || return 2 + (( (16#$word >> $2) & 1 )) +} + +# Best-effort: drop the PG_hwpoison marker set by the inject so a failed +# run does not leave a poisoned page behind. hard_offline_page() injects +# with MF_SW_SIMULATED, so the page stays unpoisonable through the +# hwpoison debugfs interface (needs CONFIG_HWPOISON_INJECT + debugfs). +try_unpoison() { + [ -w "$unpoison_path" ] || return 0 + echo "$1" > "$unpoison_path" 2>/dev/null || true +} + +case "$kind" in +rodata) + phys_addr=$(pick_rodata_phys_addr) + recheck_bit=$KPF_RESERVED + missing_msg='no "Kernel rodata" entry in /proc/iomem' + ;; +slab) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_SLAB") + recheck_bit=$KPF_SLAB + missing_msg="no usable slab PFN found in $kpageflags_path" + ;; +pgtable) + phys_addr=$(pick_kpageflags_phys_addr "$KPF_PGTABLE") + recheck_bit=$KPF_PGTABLE + missing_msg="no usable page-table PFN found in $kpageflags_path" + ;; +*) + ktap_exit_fail_msg "unknown kind '$kind' (expected: rodata|slab|pgtable)" + ;; +esac + +if [ -z "$phys_addr" ]; then + ktap_skip_all "$missing_msg" + exit "$KSFT_SKIP" +fi + +ktap_print_msg "enabling $sysctl_path" +prior=$(cat "$sysctl_path") +echo 1 > "$sysctl_path" || ktap_exit_fail_msg "failed to enable sysctl" + +pfn=$((phys_addr / pagesize)) +ktap_print_msg "injecting hwpoison at phys 0x$(printf '%x' "$phys_addr") (pfn 0x$(printf '%x' "$pfn"), kind=$kind)" +ktap_print_msg "expecting kernel panic: 'Memory failure: : unrecoverable page'" + +# A successful run never returns from the inject -- it panics the kernel. +# Reaching the code below therefore means no panic fired. Note whether +# the write itself succeeded, then put the machine back: restore the +# sysctl and best-effort unpoison the page we just marked. +if echo "$phys_addr" > "$inject_path"; then + verdict="inject returned without panic; sysctl ineffective" +else + verdict="inject failed before reaching the panic path" +fi + +echo "$prior" > "$sysctl_path" +try_unpoison "$pfn" + +# The page type can change between selection and injection (e.g. a slab +# or page-table page is freed and reused). Only treat a missing panic as +# a failure if the target PFN is still the kernel-owned type we aimed at; +# if it raced to another type the run is inconclusive, so skip instead. +kpageflags_bit_set "$pfn" "$recheck_bit" +case $? in +0) ktap_exit_fail_msg "$verdict (page still $kind)" ;; +1) ktap_skip_all "target PFN no longer $kind; raced before inject, inconclusive" + exit "$KSFT_SKIP" ;; +*) ktap_exit_fail_msg "$verdict (could not reconfirm page type via $kpageflags_path)" ;; +esac diff --git a/tools/testing/selftests/mm/khugepaged.c b/tools/testing/selftests/mm/khugepaged.c index c8393ca52cab..10e8dedcb087 100644 --- a/tools/testing/selftests/mm/khugepaged.c +++ b/tools/testing/selftests/mm/khugepaged.c @@ -22,7 +22,7 @@ #include "linux/magic.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; @@ -41,6 +41,12 @@ enum vma_type { VMA_SHMEM, }; +enum file_setup_ops { + FILE_SETUP_READ_ONLY_FS, + FILE_SETUP_READ_WRITE_FS_READ_DATA, + FILE_SETUP_READ_WRITE_FS_WRITE_DATA, +}; + struct mem_ops { void *(*setup_area)(int nr_hpages); void (*cleanup_area)(void *p, unsigned long size); @@ -49,7 +55,9 @@ struct mem_ops { const char *name; }; -static struct mem_ops *file_ops; +static struct mem_ops *read_only_file_ops; +static struct mem_ops *read_write_file_read_ops; +static struct mem_ops *read_write_file_write_ops; static struct mem_ops *anon_ops; static struct mem_ops *shmem_ops; @@ -72,57 +80,36 @@ struct file_info { }; static struct file_info finfo; -static bool skip_settings_restore; static int exit_status; static void success(const char *msg) { printf(" \e[32m%s\e[0m\n", msg); + exit_status = KSFT_PASS; } static void fail(const char *msg) { printf(" \e[31m%s\e[0m\n", msg); - exit_status++; + exit_status = KSFT_FAIL; } static void skip(const char *msg) { printf(" \e[33m%s\e[0m\n", msg); -} - -static void restore_settings_atexit(void) -{ - if (skip_settings_restore) - return; - - printf("Restore THP and khugepaged settings..."); - thp_restore_settings(); - success("OK"); - - skip_settings_restore = true; -} - -static void restore_settings(int sig) -{ - /* exit() will invoke the restore_settings_atexit handler. */ - exit(sig ? EXIT_FAILURE : exit_status); + exit_status = KSFT_SKIP; } static void save_settings(void) { - printf("Save THP and khugepaged settings..."); - if (file_ops && finfo.type == VMA_FILE) + ksft_print_msg("Save THP and khugepaged settings..."); + if ((read_only_file_ops || read_write_file_read_ops || + read_write_file_write_ops) && + finfo.type == VMA_FILE) thp_set_read_ahead_path(finfo.dev_queue_read_ahead_path); thp_save_settings(); success("OK"); - - atexit(restore_settings_atexit); - signal(SIGTERM, restore_settings); - signal(SIGINT, restore_settings); - signal(SIGHUP, restore_settings); - signal(SIGQUIT, restore_settings); } static void get_finfo(const char *dir) @@ -135,19 +122,13 @@ static void get_finfo(const char *dir) finfo.dir = dir; stat(finfo.dir, &path_stat); - if (!S_ISDIR(path_stat.st_mode)) { - printf("%s: Not a directory (%s)\n", __func__, finfo.dir); - exit(EXIT_FAILURE); - } + if (!S_ISDIR(path_stat.st_mode)) + ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir); if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE, - finfo.dir) >= sizeof(finfo.path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } - if (statfs(finfo.dir, &fs)) { - perror("statfs()"); - exit(EXIT_FAILURE); - } + finfo.dir) >= sizeof(finfo.path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); + if (statfs(finfo.dir, &fs)) + ksft_exit_fail_perror("statfs()"); finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE; if (finfo.type == VMA_SHMEM) return; @@ -155,40 +136,30 @@ static void get_finfo(const char *dir) /* Find owning device's queue/read_ahead_kb control */ if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent", major(path_stat.st_dev), minor(path_stat.st_dev)) - >= sizeof(path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } - if (read_file(path, buf, sizeof(buf)) < 0) { - perror("read_file(read_num)"); - exit(EXIT_FAILURE); - } + >= sizeof(path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); + if (read_file(path, buf, sizeof(buf)) < 0) + ksft_exit_fail_perror("read_file(read_num)"); if (strstr(buf, "DEVTYPE=disk")) { /* Found it */ if (snprintf(finfo.dev_queue_read_ahead_path, sizeof(finfo.dev_queue_read_ahead_path), "/sys/dev/block/%d:%d/queue/read_ahead_kb", major(path_stat.st_dev), minor(path_stat.st_dev)) - >= sizeof(finfo.dev_queue_read_ahead_path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } + >= sizeof(finfo.dev_queue_read_ahead_path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); return; } - if (!strstr(buf, "DEVTYPE=partition")) { - printf("%s: Unknown device type: %s\n", __func__, path); - exit(EXIT_FAILURE); - } + if (!strstr(buf, "DEVTYPE=partition")) + ksft_exit_fail_msg("%s: Unknown device type: %s\n", __func__, path); /* * Partition of block device - need to find actual device. * Using naming convention that devnameN is partition of * device devname. */ str = strstr(buf, "DEVNAME="); - if (!str) { - printf("%s: Could not read: %s", __func__, path); - exit(EXIT_FAILURE); - } + if (!str) + ksft_exit_fail_msg("%s: Could not read: %s", __func__, path); str += 8; end = str; while (*end) { @@ -197,16 +168,13 @@ static void get_finfo(const char *dir) if (snprintf(finfo.dev_queue_read_ahead_path, sizeof(finfo.dev_queue_read_ahead_path), "/sys/block/%s/queue/read_ahead_kb", - str) >= sizeof(finfo.dev_queue_read_ahead_path)) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } + str) >= sizeof(finfo.dev_queue_read_ahead_path)) + ksft_exit_fail_msg("%s: Pathname is too long\n", __func__); return; } ++end; } - printf("%s: Could not read: %s\n", __func__, path); - exit(EXIT_FAILURE); + ksft_exit_fail_msg("%s: Could not read: %s\n", __func__, path); } static bool check_swap(void *addr, unsigned long size) @@ -219,26 +187,19 @@ static bool check_swap(void *addr, unsigned long size) ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", (unsigned long) addr); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); - } - + if (ret >= MAX_LINE_LENGTH) + ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); fp = fopen(PID_SMAPS, "r"); - if (!fp) { - printf("%s: Failed to open file %s\n", __func__, PID_SMAPS); - exit(EXIT_FAILURE); - } + if (!fp) + ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, PID_SMAPS); if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) goto err_out; ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB", size >> 10); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); - exit(EXIT_FAILURE); - } + if (ret >= MAX_LINE_LENGTH) + ksft_exit_fail_msg("%s: Pattern is too long\n", __func__); /* * Fetch the Swap: in the same block and check whether it got * the expected number of hugeepages next. @@ -261,10 +222,8 @@ static void *alloc_mapping(int nr) p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (p != BASE_ADDR) { - printf("Failed to allocate VMA at %p\n", BASE_ADDR); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_msg("Failed to allocate VMA at %p\n", BASE_ADDR); return p; } @@ -314,19 +273,13 @@ static void *alloc_hpage(struct mem_ops *ops) * khugepaged on low-load system (like a test machine), which * would cause MADV_COLLAPSE to fail with EAGAIN. */ - printf("Allocate huge page..."); - if (madvise_collapse_retry(p, hpage_pmd_size)) { - perror("madvise(MADV_COLLAPSE)"); - exit(EXIT_FAILURE); - } - if (!ops->check_huge(p, 1)) { - perror("madvise(MADV_COLLAPSE)"); - exit(EXIT_FAILURE); - } - if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) { - perror("madvise(MADV_HUGEPAGE)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Allocate huge page..."); + if (madvise_collapse_retry(p, hpage_pmd_size)) + ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); + if (!ops->check_huge(p, 1)) + ksft_exit_fail_perror("madvise(MADV_COLLAPSE)"); + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) + ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)"); success("OK"); return p; } @@ -336,11 +289,9 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) int i; for (i = start / page_size; i < end / page_size; i++) { - if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) { - printf("Page %d is corrupted: %#x\n", - i, p[i * page_size / sizeof(*p)]); - exit(EXIT_FAILURE); - } + if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) + ksft_exit_fail_msg("Page %d is corrupted: %#x\n", + i, p[i * page_size / sizeof(*p)]); } } @@ -364,21 +315,21 @@ static bool anon_check_huge(void *addr, int nr_hpages) return check_huge_anon(addr, nr_hpages, hpage_pmd_size); } -static void *file_setup_area(int nr_hpages) +static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup) { + const int open_opt = setup == FILE_SETUP_READ_ONLY_FS ? O_RDONLY : O_RDWR; + const int mmap_prot = setup == FILE_SETUP_READ_ONLY_FS ? PROT_READ : (PROT_READ | PROT_WRITE); int fd; void *p; unsigned long size; unlink(finfo.path); /* Cleanup from previous failed tests */ - printf("Creating %s for collapse%s...", finfo.path, - finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); + ksft_print_msg("Creating %s for collapse%s...", finfo.path, + finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); fd = open(finfo.path, O_CREAT | O_RDWR | O_TRUNC | O_EXCL, 777); - if (fd < 0) { - perror("open()"); - exit(EXIT_FAILURE); - } + if (fd < 0) + ksft_exit_fail_perror("open()"); size = nr_hpages * hpage_pmd_size; if (ftruncate(fd, size)) { @@ -399,19 +350,17 @@ static void *file_setup_area(int nr_hpages) close(fd); munmap(p, size); success("OK"); - - printf("Opening %s read only for collapse...", finfo.path); - finfo.fd = open(finfo.path, O_RDONLY, 777); - if (finfo.fd < 0) { - perror("open()"); - exit(EXIT_FAILURE); - } - p = mmap(BASE_ADDR, size, PROT_READ, - MAP_PRIVATE, finfo.fd, 0); - if (p == MAP_FAILED || p != BASE_ADDR) { - perror("mmap()"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Opening %s %s for collapse...", finfo.path, + setup == FILE_SETUP_READ_ONLY_FS ? "read-only" : + setup == FILE_SETUP_READ_WRITE_FS_READ_DATA ? + "read-write (read)" : + "read-write (write)"); + finfo.fd = open(finfo.path, open_opt, 777); + if (finfo.fd < 0) + ksft_exit_fail_perror("open()"); + p = mmap(BASE_ADDR, size, mmap_prot, MAP_SHARED, finfo.fd, 0); + if (p == MAP_FAILED || p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); /* Drop page cache */ write_file("/proc/sys/vm/drop_caches", "3", 2); @@ -419,6 +368,21 @@ static void *file_setup_area(int nr_hpages) return p; } +static void *file_setup_read_only_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_ONLY_FS); +} + +static void *file_setup_read_write_fs_read_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_READ_DATA); +} + +static void *file_setup_read_write_fs_write_area(int nr_hpages) +{ + return file_setup_area_common(nr_hpages, FILE_SETUP_READ_WRITE_FS_WRITE_DATA); +} + static void file_cleanup_area(void *p, unsigned long size) { munmap(p, size); @@ -426,12 +390,26 @@ static void file_cleanup_area(void *p, unsigned long size) unlink(finfo.path); } -static void file_fault(void *p, unsigned long start, unsigned long end) +static void file_fault_read(void *p, unsigned long start, unsigned long end) { - if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) { - perror("madvise(MADV_POPULATE_READ"); - exit(EXIT_FAILURE); - } + if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) + ksft_exit_fail_perror("madvise(MADV_POPULATE_READ)"); +} + +static void file_fault_read_and_flush(void *p, unsigned long start, unsigned long end) +{ + file_fault_read(p, start, end); + /* + * make folio clean, since dirty folios from read&write file are + * rejected and not flushed + */ + msync((char *)p + start, end - start, MS_SYNC); +} + +static void file_fault_write(void *p, unsigned long start, unsigned long end) +{ + if (madvise(((char *)p) + start, end - start, MADV_POPULATE_WRITE)) + ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)"); } static bool file_check_huge(void *addr, int nr_hpages) @@ -453,20 +431,14 @@ static void *shmem_setup_area(int nr_hpages) unsigned long size = nr_hpages * hpage_pmd_size; finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0); - if (finfo.fd < 0) { - perror("memfd_create()"); - exit(EXIT_FAILURE); - } - if (ftruncate(finfo.fd, size)) { - perror("ftruncate()"); - exit(EXIT_FAILURE); - } + if (finfo.fd < 0) + ksft_exit_fail_perror("memfd_create()"); + if (ftruncate(finfo.fd, size)) + ksft_exit_fail_perror("ftruncate()"); p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, 0); - if (p != BASE_ADDR) { - perror("mmap()"); - exit(EXIT_FAILURE); - } + if (p != BASE_ADDR) + ksft_exit_fail_perror("mmap()"); return p; } @@ -489,10 +461,26 @@ static struct mem_ops __anon_ops = { .name = "anon", }; -static struct mem_ops __file_ops = { - .setup_area = &file_setup_area, +static struct mem_ops __read_only_file_ops = { + .setup_area = &file_setup_read_only_area, + .cleanup_area = &file_cleanup_area, + .fault = &file_fault_read, + .check_huge = &file_check_huge, + .name = "file", +}; + +static struct mem_ops __read_write_file_read_ops = { + .setup_area = &file_setup_read_write_fs_read_area, .cleanup_area = &file_cleanup_area, - .fault = &file_fault, + .fault = &file_fault_read_and_flush, + .check_huge = &file_check_huge, + .name = "file", +}; + +static struct mem_ops __read_write_file_write_ops = { + .setup_area = &file_setup_read_write_fs_write_area, + .cleanup_area = &file_cleanup_area, + .fault = &file_fault_write, .check_huge = &file_check_huge, .name = "file", }; @@ -505,13 +493,32 @@ static struct mem_ops __shmem_ops = { .name = "shmem", }; +static bool is_tmpfs(struct mem_ops *ops) +{ + return (ops == &__read_only_file_ops || + ops == &__read_write_file_read_ops || + ops == &__read_write_file_write_ops) && + finfo.type == VMA_SHMEM; +} + +static bool is_anon(struct mem_ops *ops) +{ + return ops == &__anon_ops; +} + static void __madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { int ret; struct thp_settings settings = *thp_current_settings(); - printf("%s...", msg); + ksft_print_msg("%s...", msg); + + /* + * read&write file collapse succeeds for MADV_COLLAPSE because dirty + * folios are written back after collapse fails for dirty folios and + * another collapse is attempted. + */ /* * Prevent khugepaged interference and tests that MADV_COLLAPSE @@ -538,10 +545,8 @@ static void madvise_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { /* Sanity check */ - if (!ops->check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(p, 0)) + ksft_exit_fail_msg("Unexpected huge page\n"); __madvise_collapse(msg, p, nr_hpages, ops, expect); } @@ -553,17 +558,15 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, int timeout = 6; /* 3 seconds */ /* Sanity check */ - if (!ops->check_huge(p, 0)) { - printf("Unexpected huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(p, 0)) + ksft_exit_fail_msg("Unexpected huge page\n"); madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); /* Wait until the second full_scan completed */ full_scans = thp_read_num("khugepaged/full_scans") + 2; - printf("%s...", msg); + ksft_print_msg("%s...", msg); while (timeout--) { if (ops->check_huge(p, nr_hpages)) break; @@ -579,6 +582,13 @@ static bool wait_for_scan(const char *msg, char *p, int nr_hpages, static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, struct mem_ops *ops, bool expect) { + /* + * read&write file collapse fails since khugepaged does not flush + * the target dirty folios + */ + if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops) + expect = false; + if (wait_for_scan(msg, p, nr_hpages, ops)) { if (expect) fail("Timeout"); @@ -613,16 +623,6 @@ static struct collapse_context __madvise_context = { .name = "madvise", }; -static bool is_tmpfs(struct mem_ops *ops) -{ - return ops == &__file_ops && finfo.type == VMA_SHMEM; -} - -static bool is_anon(struct mem_ops *ops) -{ - return ops == &__anon_ops; -} - static void alloc_at_fault(void) { struct thp_settings settings = *thp_current_settings(); @@ -633,7 +633,7 @@ static void alloc_at_fault(void) p = alloc_mapping(1); *p = 1; - printf("Allocate huge page on fault..."); + ksft_print_msg("Allocate huge page on fault..."); if (check_huge_anon(p, 1, hpage_pmd_size)) success("OK"); else @@ -642,12 +642,14 @@ static void alloc_at_fault(void) thp_pop_settings(); madvise(p, page_size, MADV_DONTNEED); - printf("Split huge PMD on MADV_DONTNEED..."); + ksft_print_msg("Split huge PMD on MADV_DONTNEED..."); if (check_huge_anon(p, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); munmap(p, hpage_pmd_size); + + ksft_test_result_report(exit_status, "allocate on fault and split\n"); } static void collapse_full(struct collapse_context *c, struct mem_ops *ops) @@ -662,6 +664,8 @@ static void collapse_full(struct collapse_context *c, struct mem_ops *ops) ops, true); validate_memory(p, 0, size); ops->cleanup_area(p, size); + + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) @@ -671,6 +675,7 @@ static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) p = ops->setup_area(1); c->collapse("Do not collapse empty PTE table", p, 1, ops, false); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops) @@ -682,6 +687,7 @@ static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops c->collapse("Collapse PTE table with single PTE entry present", p, 1, ops, true); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops) @@ -709,6 +715,9 @@ static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *o validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size); if (c->enforce_pte_scan_limits) { + ops->cleanup_area(p, hpage_pmd_size); + p = ops->setup_area(1); + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, true); @@ -718,6 +727,7 @@ static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *o skip: ops->cleanup_area(p, hpage_pmd_size); thp_pop_settings(); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops) @@ -727,11 +737,9 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op p = ops->setup_area(1); ops->fault(p, 0, hpage_pmd_size); - printf("Swapout one page..."); - if (madvise(p, page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Swapout one page..."); + if (madvise(p, page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, page_size)) { success("OK"); } else { @@ -744,6 +752,7 @@ static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_op validate_memory(p, 0, hpage_pmd_size); out: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops) @@ -754,11 +763,9 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o p = ops->setup_area(1); ops->fault(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); - if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); + if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, (max_ptes_swap + 1) * page_size)) { success("OK"); } else { @@ -772,12 +779,10 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o if (c->enforce_pte_scan_limits) { ops->fault(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap, + ksft_print_msg("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); - if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } + if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) + ksft_exit_fail_perror("madvise(MADV_PAGEOUT)"); if (check_swap(p, max_ptes_swap * page_size)) { success("OK"); } else { @@ -791,6 +796,7 @@ static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *o } out: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops) @@ -807,7 +813,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc } madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - printf("Split huge page leaving single PTE mapping compound page..."); + ksft_print_msg("Split huge page leaving single PTE mapping compound page..."); madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); if (ops->check_huge(p, 0)) success("OK"); @@ -819,6 +825,7 @@ static void collapse_single_pte_entry_compound(struct collapse_context *c, struc validate_memory(p, 0, page_size); skip: ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops) @@ -826,7 +833,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops void *p; p = alloc_hpage(ops); - printf("Split huge page leaving single PTE page table full of compound pages..."); + ksft_print_msg("Split huge page leaving single PTE page table full of compound pages..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); if (ops->check_huge(p, 0)) @@ -838,6 +845,7 @@ static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops true); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops) @@ -846,16 +854,12 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops int i; p = ops->setup_area(1); + ksft_print_msg("Construct PTE page table full of different PTE-mapped compound pages\n"); for (i = 0; i < hpage_pmd_nr; i++) { - printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", - i + 1, hpage_pmd_nr); - madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); ops->fault(BASE_ADDR, 0, hpage_pmd_size); - if (!ops->check_huge(BASE_ADDR, 1)) { - printf("Failed to allocate huge page\n"); - exit(EXIT_FAILURE); - } + if (!ops->check_huge(BASE_ADDR, 1)) + ksft_exit_fail_msg("Failed to allocate huge page\n"); madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE); p = mremap(BASE_ADDR - i * page_size, @@ -863,20 +867,16 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops (i + 1) * page_size, MREMAP_MAYMOVE | MREMAP_FIXED, BASE_ADDR + 2 * hpage_pmd_size); - if (p == MAP_FAILED) { - perror("mremap+unmap"); - exit(EXIT_FAILURE); - } + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap+unmap"); p = mremap(BASE_ADDR + 2 * hpage_pmd_size, (i + 1) * page_size, (i + 1) * page_size + hpage_pmd_size, MREMAP_MAYMOVE | MREMAP_FIXED, BASE_ADDR - (i + 1) * page_size); - if (p == MAP_FAILED) { - perror("mremap+alloc"); - exit(EXIT_FAILURE); - } + if (p == MAP_FAILED) + ksft_exit_fail_perror("mremap+alloc"); } ops->cleanup_area(BASE_ADDR, hpage_pmd_size); @@ -891,6 +891,7 @@ static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) @@ -900,19 +901,16 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) p = ops->setup_area(1); - printf("Allocate small page..."); + ksft_print_msg("Allocate small page..."); ops->fault(p, 0, page_size); if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - printf("Share small page over fork()..."); + ksft_print_msg("Share small page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 0)) success("OK"); else @@ -924,19 +922,20 @@ static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) validate_memory(p, 0, page_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has small page..."); + ksft_print_msg("Check if parent still has small page..."); if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); validate_memory(p, 0, page_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops) @@ -945,18 +944,15 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o void *p; p = alloc_hpage(ops); - printf("Share huge page over fork()..."); + ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); - printf("Split huge page PMD in child process..."); + ksft_print_msg("Split huge page PMD in child process..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); if (ops->check_huge(p, 0)) @@ -973,19 +969,20 @@ static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *o validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has huge page..."); + ksft_print_msg("Check if parent still has huge page..."); if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops) @@ -995,18 +992,15 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops void *p; p = alloc_hpage(ops); - printf("Share huge page over fork()..."); + ksft_print_msg("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ - skip_settings_restore = true; - exit_status = 0; - if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); - printf("Trigger CoW on page %d of %d...", + ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); if (ops->check_huge(p, 0)) @@ -1018,7 +1012,7 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops 1, ops, !c->enforce_pte_scan_limits); if (c->enforce_pte_scan_limits) { - printf("Trigger CoW on page %d of %d...", + ksft_print_msg("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); @@ -1033,19 +1027,20 @@ static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); - exit(exit_status); + _exit(exit_status); } wait(&wstatus); - exit_status += WEXITSTATUS(wstatus); + exit_status = WEXITSTATUS(wstatus); - printf("Check if parent still has huge page..."); + ksft_print_msg("Check if parent still has huge page..."); if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void madvise_collapse_existing_thps(struct collapse_context *c, @@ -1062,6 +1057,7 @@ static void madvise_collapse_existing_thps(struct collapse_context *c, __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); validate_memory(p, 0, hpage_pmd_size); ops->cleanup_area(p, hpage_pmd_size); + ksft_test_result_report(exit_status, "%s\n", __func__); } /* @@ -1089,6 +1085,7 @@ static void madvise_retracted_page_tables(struct collapse_context *c, true); validate_memory(p, 0, size); ops->cleanup_area(p, size); + ksft_test_result_report(exit_status, "%s\n", __func__); } static void usage(void) @@ -1098,8 +1095,8 @@ static void usage(void) fprintf(stderr, "\t\t: [all|khugepaged|madvise]\n"); fprintf(stderr, "\t\t: [all|anon|file|shmem]\n"); fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n"); - fprintf(stderr, "\n\t\"file,all\" mem_type requires kernel built with\n"); - fprintf(stderr, "\tCONFIG_READ_ONLY_THP_FOR_FS=y\n"); + fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n"); + fprintf(stderr, "\twith PMD-sized large folio support\n"); fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n"); fprintf(stderr, "\tmounted with huge=advise option for khugepaged tests to work\n"); fprintf(stderr, "\n\tSupported Options:\n"); @@ -1155,20 +1152,25 @@ static void parse_test_type(int argc, char **argv) usage(); if (!strcmp(buf, "all")) { - file_ops = &__file_ops; + read_only_file_ops = &__read_only_file_ops; + read_write_file_read_ops = &__read_write_file_read_ops; + read_write_file_write_ops = &__read_write_file_write_ops; anon_ops = &__anon_ops; shmem_ops = &__shmem_ops; } else if (!strcmp(buf, "anon")) { anon_ops = &__anon_ops; } else if (!strcmp(buf, "file")) { - file_ops = &__file_ops; + read_only_file_ops = &__read_only_file_ops; + read_write_file_read_ops = &__read_write_file_read_ops; + read_write_file_write_ops = &__read_write_file_write_ops; } else if (!strcmp(buf, "shmem")) { shmem_ops = &__shmem_ops; } else { usage(); } - if (!file_ops) + if (!read_only_file_ops && !read_write_file_read_ops && + !read_write_file_write_ops) return; if (argc != 2) @@ -1177,6 +1179,32 @@ static void parse_test_type(int argc, char **argv) get_finfo(argv[1]); } +typedef void (*test_fn)(struct collapse_context *c, struct mem_ops *ops); + +struct test_case { + struct collapse_context *ctx; + struct mem_ops *ops; + const char *desc; + test_fn fn; +}; + +#define MAX_TEST_CASES 64 +static struct test_case test_cases[MAX_TEST_CASES]; +static int nr_test_cases; + +#define TEST(t, c, o) do { \ + if (c && o) { \ + if (nr_test_cases >= MAX_TEST_CASES) \ + ksft_exit_fail_msg("MAX_TEST_CASES is too small\n"); \ + test_cases[nr_test_cases++] = (struct test_case){ \ + .ctx = c, \ + .ops = o, \ + .desc = #t, \ + .fn = t, \ + }; \ + } \ + } while (0) + int main(int argc, char **argv) { int hpage_pmd_order; @@ -1200,10 +1228,10 @@ int main(int argc, char **argv) .read_ahead_kb = 0, }; - if (!thp_is_enabled()) { - printf("Transparent Hugepages not available\n"); - return KSFT_SKIP; - } + ksft_print_header(); + + if (!thp_is_enabled()) + ksft_exit_skip("Transparent Hugepages not available\n"); parse_test_type(argc, argv); @@ -1211,10 +1239,8 @@ int main(int argc, char **argv) page_size = getpagesize(); hpage_pmd_size = read_pmd_pagesize(); - if (!hpage_pmd_size) { - printf("Reading PMD pagesize failed"); - exit(EXIT_FAILURE); - } + if (!hpage_pmd_size) + ksft_exit_fail_msg("Reading PMD pagesize failed\n"); hpage_pmd_nr = hpage_pmd_size / page_size; hpage_pmd_order = __builtin_ctz(hpage_pmd_nr); @@ -1230,47 +1256,54 @@ int main(int argc, char **argv) save_settings(); thp_push_settings(&default_settings); - alloc_at_fault(); - -#define TEST(t, c, o) do { \ - if (c && o) { \ - printf("\nRun test: " #t " (%s:%s)\n", c->name, o->name); \ - t(c, o); \ - } \ - } while (0) - TEST(collapse_full, khugepaged_context, anon_ops); - TEST(collapse_full, khugepaged_context, file_ops); + TEST(collapse_full, khugepaged_context, read_only_file_ops); + TEST(collapse_full, khugepaged_context, read_write_file_read_ops); + TEST(collapse_full, khugepaged_context, read_write_file_write_ops); TEST(collapse_full, khugepaged_context, shmem_ops); TEST(collapse_full, madvise_context, anon_ops); - TEST(collapse_full, madvise_context, file_ops); + TEST(collapse_full, madvise_context, read_only_file_ops); + TEST(collapse_full, madvise_context, read_write_file_read_ops); + TEST(collapse_full, madvise_context, read_write_file_write_ops); TEST(collapse_full, madvise_context, shmem_ops); TEST(collapse_empty, khugepaged_context, anon_ops); TEST(collapse_empty, madvise_context, anon_ops); TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops); + TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_write_ops); TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops); TEST(collapse_single_pte_entry, madvise_context, anon_ops); - TEST(collapse_single_pte_entry, madvise_context, file_ops); + TEST(collapse_single_pte_entry, madvise_context, read_only_file_ops); + TEST(collapse_single_pte_entry, madvise_context, read_write_file_read_ops); + TEST(collapse_single_pte_entry, madvise_context, read_write_file_write_ops); TEST(collapse_single_pte_entry, madvise_context, shmem_ops); TEST(collapse_max_ptes_none, khugepaged_context, anon_ops); - TEST(collapse_max_ptes_none, khugepaged_context, file_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_only_file_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_read_ops); + TEST(collapse_max_ptes_none, khugepaged_context, read_write_file_write_ops); TEST(collapse_max_ptes_none, madvise_context, anon_ops); - TEST(collapse_max_ptes_none, madvise_context, file_ops); + TEST(collapse_max_ptes_none, madvise_context, read_only_file_ops); + TEST(collapse_max_ptes_none, madvise_context, read_write_file_read_ops); + TEST(collapse_max_ptes_none, madvise_context, read_write_file_write_ops); TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops); - TEST(collapse_single_pte_entry_compound, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry_compound, khugepaged_context, read_only_file_ops); + TEST(collapse_single_pte_entry_compound, khugepaged_context, read_write_file_read_ops); TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops); - TEST(collapse_single_pte_entry_compound, madvise_context, file_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, read_only_file_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, read_write_file_read_ops); TEST(collapse_full_of_compound, khugepaged_context, anon_ops); - TEST(collapse_full_of_compound, khugepaged_context, file_ops); + TEST(collapse_full_of_compound, khugepaged_context, read_only_file_ops); + TEST(collapse_full_of_compound, khugepaged_context, read_write_file_read_ops); TEST(collapse_full_of_compound, khugepaged_context, shmem_ops); TEST(collapse_full_of_compound, madvise_context, anon_ops); - TEST(collapse_full_of_compound, madvise_context, file_ops); + TEST(collapse_full_of_compound, madvise_context, read_only_file_ops); + TEST(collapse_full_of_compound, madvise_context, read_write_file_read_ops); TEST(collapse_full_of_compound, madvise_context, shmem_ops); TEST(collapse_compound_extreme, khugepaged_context, anon_ops); @@ -1292,11 +1325,23 @@ int main(int argc, char **argv) TEST(collapse_max_ptes_shared, madvise_context, anon_ops); TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); - TEST(madvise_collapse_existing_thps, madvise_context, file_ops); + TEST(madvise_collapse_existing_thps, madvise_context, read_only_file_ops); + TEST(madvise_collapse_existing_thps, madvise_context, read_write_file_read_ops); TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops); - TEST(madvise_retracted_page_tables, madvise_context, file_ops); + TEST(madvise_retracted_page_tables, madvise_context, read_only_file_ops); + TEST(madvise_retracted_page_tables, madvise_context, read_write_file_read_ops); TEST(madvise_retracted_page_tables, madvise_context, shmem_ops); - restore_settings(0); + ksft_set_plan(nr_test_cases + 1); + + alloc_at_fault(); + for (int i = 0; i < nr_test_cases; i++) { + struct test_case *t = &test_cases[i]; + + ksft_print_msg("\n# Run test: %s (%s:%s)\n", t->desc, t->ctx->name, t->ops->name); + t->fn(t->ctx, t->ops); + } + + ksft_finished(); } diff --git a/tools/testing/selftests/mm/ksft_process_madv.sh b/tools/testing/selftests/mm/ksft_process_madv.sh index 2c3137ae8bc8..edad2d2d888f 100755 --- a/tools/testing/selftests/mm/ksft_process_madv.sh +++ b/tools/testing/selftests/mm/ksft_process_madv.sh @@ -1,4 +1,4 @@ #!/bin/sh -e # SPDX-License-Identifier: GPL-2.0 -./run_vmtests.sh -t mmap +./run_vmtests.sh -t process_madv diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index a0b48b839d54..a050f4840cfa 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -15,7 +15,7 @@ #include "kselftest.h" #include #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" #define KSM_FP(s) (KSM_SYSFS_PATH s) @@ -174,13 +174,13 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_ { void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); - if (!map_ptr) { - perror("mmap"); + if (map_ptr == MAP_FAILED) { + ksft_perror("mmap"); return NULL; } memset(map_ptr, data, map_size); if (mprotect(map_ptr, map_size, prot)) { - perror("mprotect"); + ksft_perror("mprotect"); munmap(map_ptr, map_size); return NULL; } @@ -201,11 +201,11 @@ static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) return 1; if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return 1; } if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { - printf("Scan time limit exceeded\n"); + ksft_print_msg("Scan time limit exceeded\n"); return 1; } } @@ -218,12 +218,12 @@ static int ksm_merge_pages(int merge_type, void *addr, size_t size, { if (merge_type == KSM_MERGE_MADVISE) { if (madvise(addr, size, MADV_MERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } } else if (merge_type == KSM_MERGE_PRCTL) { if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { - perror("prctl"); + ksft_perror("prctl"); return 1; } } @@ -242,7 +242,7 @@ static int ksm_unmerge_pages(void *addr, size_t size, struct timespec start_time, int timeout) { if (madvise(addr, size, MADV_UNMERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } return 0; @@ -324,7 +324,7 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -338,7 +338,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, /* verify that the right number of pages are merged */ if (assert_ksm_pages_count(page_count)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); if (merge_type == KSM_MERGE_PRCTL) prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); @@ -346,7 +345,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -358,7 +356,7 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, int page_count = 2; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -380,13 +378,11 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, /* check that unmerging was successful and 0 pages are currently merged */ if (assert_ksm_pages_count(0)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -398,7 +394,7 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -425,12 +421,10 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) goto err_out; - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -465,16 +459,16 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } if (numa_available() < 0) { - perror("NUMA support not enabled"); + ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } if (numa_num_configured_nodes() <= 1) { - printf("At least 2 NUMA nodes must be available\n"); + ksft_print_msg("At least 2 NUMA nodes must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) @@ -485,7 +479,7 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo numa1_map_ptr = numa_alloc_onnode(page_size, first_node); numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); if (!numa1_map_ptr || !numa2_map_ptr) { - perror("numa_alloc_onnode"); + ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; } @@ -510,13 +504,11 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("OK\n"); return KSFT_PASS; err_out: numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("Not OK\n"); return KSFT_FAIL; } @@ -529,7 +521,7 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, int pagemap_fd, n_normal_pages, n_huge_pages; if (!thp_is_enabled()) { - printf("Transparent Hugepages not available\n"); + ksft_print_msg("Transparent Hugepages not available\n"); return KSFT_SKIP; } @@ -559,36 +551,35 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, else n_huge_pages++; } - printf("Number of normal pages: %d\n", n_normal_pages); - printf("Number of huge pages: %d\n", n_huge_pages); + ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); + ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); memset(map_ptr, '*', len); if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_FAIL; } @@ -606,30 +597,29 @@ static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, si return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -646,37 +636,36 @@ static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, if (!map_ptr) return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -695,24 +684,24 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); - printf("Not merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); + ksft_print_msg("Not merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); /* Create 2000 pairs of duplicate pages */ @@ -724,30 +713,29 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -765,6 +753,10 @@ int main(int argc, char *argv[]) bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; long size_MB = 0; + const char *test_descr = "KSM merging"; + + ksft_print_header(); + ksft_set_plan(1); while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { switch (opt) { @@ -773,17 +765,13 @@ int main(int argc, char *argv[]) break; case 'p': page_count = atol(optarg); - if (page_count <= 0) { - printf("The number of pages must be greater than 0\n"); - return KSFT_FAIL; - } + if (page_count <= 0) + ksft_exit_fail_msg("The number of pages must be greater than 0\n"); break; case 'l': ksm_scan_limit_sec = atoi(optarg); - if (ksm_scan_limit_sec <= 0) { - printf("Timeout value must be greater than 0\n"); - return KSFT_FAIL; - } + if (ksm_scan_limit_sec <= 0) + ksft_exit_fail_msg("Timeout value must be greater than 0\n"); break; case 'h': print_help(); @@ -805,19 +793,15 @@ int main(int argc, char *argv[]) break; case 's': size_MB = atoi(optarg); - if (size_MB <= 0) { - printf("Size must be greater than 0\n"); - return KSFT_FAIL; - } + if (size_MB <= 0) + ksft_exit_fail_msg("Size must be greater than 0\n"); break; case 't': { int tmp = atoi(optarg); - if (tmp < 0 || tmp > KSM_MERGE_LAST) { - printf("Invalid merge type\n"); - return KSFT_FAIL; - } + if (tmp < 0 || tmp > KSM_MERGE_LAST) + ksft_exit_fail_msg("Invalid merge type\n"); merge_type = tmp; } break; @@ -845,82 +829,80 @@ int main(int argc, char *argv[]) test_name = KSM_COW_TIME; break; default: - return KSFT_FAIL; + ksft_exit_fail_msg("Unknown option\n"); } } if (prot == 0) prot = str_to_prot(KSM_PROT_STR_DEFAULT); - if (access(KSM_SYSFS_PATH, F_OK)) { - printf("Config KSM not enabled\n"); - return KSFT_SKIP; - } + if (access(KSM_SYSFS_PATH, F_OK)) + ksft_exit_skip("Config KSM not enabled\n"); - if (ksm_save_def(&ksm_sysfs_old)) { - printf("Cannot save default tunables\n"); - return KSFT_FAIL; - } + if (ksm_save_def(&ksm_sysfs_old)) + ksft_exit_fail_msg("Cannot save default tunables\n"); if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || numa_available() ? 0 : ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) - return KSFT_FAIL; + ksft_exit_fail_msg("Cannot set up KSM tunables\n"); switch (test_name) { case CHECK_KSM_MERGE: + test_descr = "KSM merging"; ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_UNMERGE: + test_descr = "KSM unmerging"; ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_ZERO_PAGE_MERGE: + test_descr = "KSM zero page merging"; ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, use_zero_pages, page_size); break; case CHECK_KSM_NUMA_MERGE: + test_descr = "KSM NUMA merging"; ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, merge_across_nodes, page_size); break; case KSM_MERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time"; ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_MERGE_TIME_HUGE_PAGES: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time with huge pages"; ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_UNMERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM unmerge time"; ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_COW_TIME: + test_descr = "KSM COW time"; ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; } - if (ksm_restore(&ksm_sysfs_old)) { - printf("Cannot restore default tunables\n"); - return KSFT_FAIL; - } + if (ksm_restore(&ksm_sysfs_old)) + ksft_print_msg("Cannot restore default tunables\n"); + + ksft_test_result_report(ret, "%s\n", test_descr); - return ret; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c index 88050e0f829a..7fce5d0b622b 100644 --- a/tools/testing/selftests/mm/madv_populate.c +++ b/tools/testing/selftests/mm/madv_populate.c @@ -34,7 +34,7 @@ static void sense_support(void) addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); ret = madvise(addr, pagesize, MADV_POPULATE_READ); diff --git a/tools/testing/selftests/mm/map_hugetlb.c b/tools/testing/selftests/mm/map_hugetlb.c deleted file mode 100644 index aa409107611b..000000000000 --- a/tools/testing/selftests/mm/map_hugetlb.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Example of using hugepage memory in a user application using the mmap - * system call with MAP_HUGETLB flag. Before running this program make - * sure the administrator has allocated enough default sized huge pages - * to cover the 256 MB allocation. - */ -#include -#include -#include -#include -#include -#include "vm_util.h" -#include "kselftest.h" - -#define LENGTH (256UL*1024*1024) -#define PROTECTION (PROT_READ | PROT_WRITE) - -static void check_bytes(char *addr) -{ - ksft_print_msg("First hex is %x\n", *((unsigned int *)addr)); -} - -static void write_bytes(char *addr, size_t length) -{ - unsigned long i; - - for (i = 0; i < length; i++) - *(addr + i) = (char)i; -} - -static void read_bytes(char *addr, size_t length) -{ - unsigned long i; - - check_bytes(addr); - for (i = 0; i < length; i++) - if (*(addr + i) != (char)i) - ksft_exit_fail_msg("Mismatch at %lu\n", i); - - ksft_test_result_pass("Read correct data\n"); -} - -int main(int argc, char **argv) -{ - void *addr; - size_t hugepage_size; - size_t length = LENGTH; - int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB; - int shift = 0; - - hugepage_size = default_huge_page_size(); - /* munmap with fail if the length is not page aligned */ - if (hugepage_size > length) - length = hugepage_size; - - ksft_print_header(); - ksft_set_plan(1); - - if (argc > 1) - length = atol(argv[1]) << 20; - if (argc > 2) { - shift = atoi(argv[2]); - if (shift) - flags |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT; - } - - if (shift) - ksft_print_msg("%u kB hugepages\n", 1 << (shift - 10)); - else - ksft_print_msg("Default size hugepages\n"); - ksft_print_msg("Mapping %lu Mbytes\n", (unsigned long)length >> 20); - - addr = mmap(NULL, length, PROTECTION, flags, -1, 0); - if (addr == MAP_FAILED) - ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); - - ksft_print_msg("Returned address is %p\n", addr); - check_bytes(addr); - write_bytes(addr, length); - read_bytes(addr, length); - - /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */ - if (munmap(addr, length)) - ksft_exit_fail_msg("munmap: %s\n", strerror(errno)); - - ksft_finished(); -} diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index 60e78bbfc0e3..29f7492453d4 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -5,7 +5,7 @@ */ #include "kselftest_harness.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include #include @@ -23,6 +23,8 @@ #define MAX_RETRIES 100 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) +HUGETLB_SETUP_DEFAULT_PAGES(1) + FIXTURE(migration) { pthread_t *threads; @@ -32,13 +34,26 @@ FIXTURE(migration) int n2; }; +static void reset_signals(void) +{ + struct sigaction sa = { .sa_handler = SIG_DFL }; + + sigemptyset(&sa.sa_mask); + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); +} + FIXTURE_SETUP(migration) { int n; + reset_signals(); + if (numa_available() < 0) SKIP(return, "NUMA not available"); - self->nthreads = numa_num_task_cpus() - 1; + self->nthreads = numa_num_task_cpus() - 2; self->n1 = -1; self->n2 = -1; @@ -52,6 +67,9 @@ FIXTURE_SETUP(migration) } } + if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0) + SKIP(return, "Not enough threads or NUMA nodes available"); + self->threads = malloc(self->nthreads * sizeof(*self->threads)); ASSERT_NE(self->threads, NULL); self->pids = malloc(self->nthreads * sizeof(*self->pids)); @@ -64,6 +82,29 @@ FIXTURE_TEARDOWN(migration) free(self->pids); } +static bool kill_children(FIXTURE_DATA(migration) * self) +{ + bool err = false; + pid_t pid; + int i; + + for (i = 0; i < self->nthreads; i++) { + int status = 0; + + pid = self->pids[i]; + if (pid < 0) + continue; + if (kill(pid, SIGTERM)) + err = true; + if (pid != waitpid(pid, &status, 0)) + err = true; + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM) + err = true; + } + + return !err; +} + int migrate(uint64_t *ptr, int n1, int n2) { int ret, tmp; @@ -127,20 +168,17 @@ TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME) uint64_t *ptr; int i; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -151,17 +189,14 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) { pid_t pid; uint64_t *ptr; - int i; - - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + int i, err; ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -174,9 +209,9 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } /* @@ -184,28 +219,30 @@ TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME) { + uint64_t pmdsize; uint64_t *ptr; int i; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + pmdsize = read_pmd_pagesize(); + if (!pmdsize) + SKIP(return, "Reading PMD pagesize failed"); - ptr = mmap(NULL, 2*TWOMEG, PROT_READ | PROT_WRITE, + ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG); - ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); + ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); + memset(ptr, 0xde, pmdsize); + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -215,25 +252,27 @@ TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME) TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) { + uint64_t pmdsize; pid_t pid; uint64_t *ptr; - int i; + int i, err; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + pmdsize = read_pmd_pagesize(); + if (!pmdsize) + SKIP(return, "Reading PMD pagesize failed"); - ptr = mmap(NULL, 2 * TWOMEG, PROT_READ | PROT_WRITE, + ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - ptr = (uint64_t *) ALIGN((uintptr_t) ptr, TWOMEG); - ASSERT_EQ(madvise(ptr, TWOMEG, MADV_HUGEPAGE), 0); + ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); + ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + memset(ptr, 0xde, pmdsize); + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -246,9 +285,9 @@ TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } /* @@ -256,23 +295,28 @@ TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME) { + unsigned long hugepage_size; uint64_t *ptr; int i; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + SKIP(return, "Reading HugeTLB pagesize failed"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, + if (hugetlb_free_default_pages() < 1) + SKIP(return, "Not enough huge pages"); + + ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) + memset(ptr, 0xde, hugepage_size); + for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) + for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0); } @@ -281,19 +325,24 @@ TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME) */ TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME) { + unsigned long hugepage_size; pid_t pid; uint64_t *ptr; - int i; + int i, err; - if (self->nthreads < 2 || self->n1 < 0 || self->n2 < 0) - SKIP(return, "Not enough threads or NUMA nodes available"); + hugepage_size = default_huge_page_size(); + if (!hugepage_size) + SKIP(return, "Reading HugeTLB pagesize failed"); - ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, + if (hugetlb_free_default_pages() < 1) + SKIP(return, "Not enough huge pages"); + + ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); - memset(ptr, 0xde, TWOMEG); - for (i = 0; i < self->nthreads - 1; i++) { + memset(ptr, 0xde, hugepage_size); + for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); @@ -306,9 +355,9 @@ TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME) } } - ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); - for (i = 0; i < self->nthreads - 1; i++) - ASSERT_EQ(kill(self->pids[i], SIGTERM), 0); + err = migrate(ptr, self->n1, self->n2); + ASSERT_EQ(kill_children(self), true); + ASSERT_EQ(err, 0); } TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/page_frag/page_frag_test.c b/tools/testing/selftests/mm/page_frag/page_frag_test.c index e806c1866e36..c8584d0fdeab 100644 --- a/tools/testing/selftests/mm/page_frag/page_frag_test.c +++ b/tools/testing/selftests/mm/page_frag/page_frag_test.c @@ -131,6 +131,8 @@ static int __init page_frag_test_init(void) init_completion(&wait); if (test_alloc_len > PAGE_SIZE || test_alloc_len <= 0 || + test_push_cpu < 0 || test_push_cpu >= nr_cpu_ids || + test_pop_cpu < 0 || test_pop_cpu >= nr_cpu_ids || !cpu_active(test_push_cpu) || !cpu_active(test_pop_cpu)) return -EINVAL; diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 7f9428d6062c..6f8971d5b3ce 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -7,8 +7,6 @@ #include #include #include -#include "vm_util.h" -#include "kselftest.h" #include #include #include @@ -23,6 +21,10 @@ #include #include +#include "vm_util.h" +#include "kselftest.h" +#include "hugepage_settings.h" + #define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ PAGE_IS_FILE | PAGE_IS_PRESENT | \ PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ @@ -1366,7 +1368,7 @@ void *thread_proc(void *mem) ksft_exit_fail_msg("pthread_barrier_wait\n"); for (i = 0; i < access_per_thread; ++i) - __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST); + __atomic_add_fetch(m + i * (page_size / sizeof(*m)), 1, __ATOMIC_SEQ_CST); ret = pthread_barrier_wait(&end_barrier); if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) @@ -1401,15 +1403,15 @@ static void transact_test(int page_size) if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1)) ksft_exit_fail_msg("pthread_barrier_init\n"); - mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, + mem = mmap(NULL, page_size * nthreads * pages_per_thread, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (mem == MAP_FAILED) ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno)); - wp_init(mem, 0x1000 * nthreads * pages_per_thread); - wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread); + wp_init(mem, page_size * nthreads * pages_per_thread); + wp_addr_range(mem, page_size * nthreads * pages_per_thread); - memset(mem, 0, 0x1000 * nthreads * pages_per_thread); + memset(mem, 0, page_size * nthreads * pages_per_thread); count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size); ksft_test_result(count > 0, "%s count %u\n", __func__, count); @@ -1418,7 +1420,7 @@ static void transact_test(int page_size) finish = 0; for (i = 0; i < nthreads; ++i) - pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread); + pthread_create(&th, NULL, thread_proc, mem + page_size * i * pages_per_thread); extra_pages = 0; for (i = 0; i < iter_count; ++i) { @@ -1554,6 +1556,9 @@ int main(int __attribute__((unused)) argc, char *argv[]) if (init_uffd()) ksft_exit_skip("Failed to initialize userfaultfd\n"); + if (!hugetlb_setup_default(4)) + ksft_print_msg("HugeTLB test will be skipped\n"); + ksft_set_plan(117); page_size = getpagesize(); @@ -1605,7 +1610,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) } /* 5. SHM Hugetlb page testing */ - mem_size = 2*1024*1024; + mem_size = default_huge_page_size(); mem = gethugetlb_mem(mem_size, &shmid); if (mem) { wp_init(mem, mem_size); @@ -1633,7 +1638,7 @@ int main(int __attribute__((unused)) argc, char *argv[]) } /* 7. File Hugetlb testing */ - mem_size = 2*1024*1024; + mem_size = default_huge_page_size(); fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL); if (fd < 0) ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno)); diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index 7c29f075e40b..46a8a1878dc1 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -68,16 +68,19 @@ static inline void sigsafe_printf(const char *format, ...) #define dprintf3(args...) dprintf_level(3, args) #define dprintf4(args...) dprintf_level(4, args) -extern void abort_hooks(void); +void tracing_on(void); +void tracing_off(void); +void abort_hooks(void); #define pkey_assert(condition) do { \ if (!(condition)) { \ - dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \ - __FILE__, __LINE__, \ - test_nr, iteration_nr); \ - dprintf0("errno at assert: %d", errno); \ - abort_hooks(); \ - exit(__LINE__); \ - } \ + dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \ + __FILE__, __LINE__, \ + test_nr, iteration_nr); \ + dprintf0("# errno at assert: %d\n", errno); \ + abort_hooks(); \ + ksft_exit_fail_msg("test %d (iteration %d)\n", \ + test_nr, iteration_nr); \ + } \ } while (0) #define barrier() __asm__ __volatile__("": : :"memory") diff --git a/tools/testing/selftests/mm/pkey-powerpc.h b/tools/testing/selftests/mm/pkey-powerpc.h index 17bf2d1b0192..2ce85580b404 100644 --- a/tools/testing/selftests/mm/pkey-powerpc.h +++ b/tools/testing/selftests/mm/pkey-powerpc.h @@ -126,7 +126,7 @@ static inline void *malloc_pkey_with_mprotect_subpage(long size, int prot, u16 p size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = syscall(__NR_subpage_prot, ptr, size, NULL); if (ret) { diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c index 302fef54049c..cbc24d6cf770 100644 --- a/tools/testing/selftests/mm/pkey_sighandler_tests.c +++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -207,22 +206,25 @@ static void test_sigsegv_handler_with_pkey0_disabled(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + ret = pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -247,22 +249,25 @@ static void test_sigsegv_handler_cannot_access_stack(void) struct sigaction sa; pthread_attr_t attr; pthread_t thr; + int ret; sa.sa_flags = SA_SIGINFO; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + ret = pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL); + if (ret) { + errno = ret; + pkey_assert(0); + } pthread_mutex_lock(&mutex); while (siginfo.si_signo == 0) @@ -288,21 +293,20 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigsegv_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* Allow access to MPK 0 and MPK 1 */ pkey_reg = pkey_reg_restrictive_default(); @@ -317,23 +321,24 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; memset(&siginfo, 0, sizeof(siginfo)); /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | + CLONE_DETACHED, + stack + STACK_SIZE, + &parent_pid, + &child_pid); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_segv_maperr_ptr(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); @@ -358,6 +363,7 @@ static void test_pkru_preserved_after_sigusr1(void) { struct sigaction sa; u64 pkey_reg; + int ret; /* Allow access to MPK 0 and an arbitrary set of keys */ pkey_reg = pkey_reg_restrictive_default(); @@ -369,10 +375,8 @@ static void test_pkru_preserved_after_sigusr1(void) sa.sa_sigaction = sigusr1_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR1, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR1, &sa, NULL); + pkey_assert(ret == 0); memset(&siginfo, 0, sizeof(siginfo)); @@ -444,6 +448,13 @@ static void test_pkru_sigreturn(void) int parent_pid = 0; int child_pid = 0; u64 pkey_reg; + long ret; + + /* + * SIGSEGV handler is reset to SIG_DFL below; turn tracing off first + * so a crash does not leave ftrace enabled. + */ + tracing_off(); sa.sa_handler = SIG_DFL; sa.sa_flags = 0; @@ -453,24 +464,20 @@ static void test_pkru_sigreturn(void) * For this testcase, we do not want to handle SIGSEGV. Reset handler * to default so that the application can crash if it receives SIGSEGV. */ - if (sigaction(SIGSEGV, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGSEGV, &sa, NULL); + pkey_assert(ret == 0); sa.sa_flags = SA_SIGINFO | SA_ONSTACK; sa.sa_sigaction = sigusr2_handler; sigemptyset(&sa.sa_mask); - if (sigaction(SIGUSR2, &sa, NULL) == -1) { - perror("sigaction"); - exit(EXIT_FAILURE); - } + ret = sigaction(SIGUSR2, &sa, NULL); + pkey_assert(ret == 0); stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - assert(stack != MAP_FAILED); + pkey_assert(stack != MAP_FAILED); /* * Allow access to MPK 0 and MPK 2. The child thread (to be created @@ -490,21 +497,22 @@ static void test_pkru_sigreturn(void) /* Set up alternate signal stack that will use the default MPK */ sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + pkey_assert(sigstack.ss_sp != MAP_FAILED); sigstack.ss_flags = 0; sigstack.ss_size = STACK_SIZE; /* Use clone to avoid newer glibcs using rseq on new threads */ - long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | - CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | - CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | - CLONE_DETACHED, - stack + STACK_SIZE, - &parent_pid, - &child_pid); + ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES | + CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM | + CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | + CLONE_DETACHED, + stack + STACK_SIZE, + &parent_pid, + &child_pid); if (ret < 0) { errno = -ret; - perror("clone"); + pkey_assert(0); } else if (ret == 0) { thread_sigusr2_self(&sigstack); syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0); @@ -530,16 +538,17 @@ static void (*pkey_tests[])(void) = { int main(int argc, char *argv[]) { - int i; - ksft_print_header(); ksft_set_plan(ARRAY_SIZE(pkey_tests)); if (!is_pkeys_supported()) ksft_exit_skip("pkeys not supported\n"); - for (i = 0; i < ARRAY_SIZE(pkey_tests); i++) - (*pkey_tests[i])(); + for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) { + tracing_on(); + (*pkey_tests[test_nr])(); + tracing_off(); + } ksft_finished(); return 0; diff --git a/tools/testing/selftests/mm/pkey_util.c b/tools/testing/selftests/mm/pkey_util.c index 255b332f7a08..fbef3cd45447 100644 --- a/tools/testing/selftests/mm/pkey_util.c +++ b/tools/testing/selftests/mm/pkey_util.c @@ -2,9 +2,99 @@ #define __SANE_USERSPACE_TYPES__ #include #include +#include +#include +#include #include "pkey-helpers.h" +int iteration_nr = 1; +int test_nr; +int dprint_in_signal; + +#if CONTROL_TRACING > 0 +static void cat_into_file(char *str, char *file) +{ + int fd = open(file, O_RDWR); + int ret; + + dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); + /* + * these need to be raw because they are called under + * pkey_assert() + */ + if (fd < 0) { + fprintf(stderr, "error opening '%s'\n", file); + perror("error: "); + exit(__LINE__); + } + + ret = write(fd, str, strlen(str)); + if (ret != strlen(str)) { + perror("write to file failed"); + fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); + exit(__LINE__); + } + close(fd); +} + +static int warned_tracing; +static int tracing_root_ok(void) +{ + if (geteuid() != 0) { + if (!warned_tracing) + fprintf(stderr, "WARNING: not run as root, " + "can not do tracing control\n"); + warned_tracing = 1; + return 0; + } + return 1; +} +#endif + +void tracing_on(void) +{ +#if CONTROL_TRACING > 0 +#define TRACEDIR "/sys/kernel/tracing" + char pidstr[32]; + + if (!tracing_root_ok()) + return; + + sprintf(pidstr, "%d", getpid()); + cat_into_file("0", TRACEDIR "/tracing_on"); + cat_into_file("\n", TRACEDIR "/trace"); + if (1) { + cat_into_file("function_graph", TRACEDIR "/current_tracer"); + cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); + } else { + cat_into_file("nop", TRACEDIR "/current_tracer"); + } + cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); + cat_into_file("1", TRACEDIR "/tracing_on"); + dprintf1("enabled tracing\n"); +#endif +} + +void tracing_off(void) +{ +#if CONTROL_TRACING > 0 + if (!tracing_root_ok()) + return; + cat_into_file("0", "/sys/kernel/tracing/tracing_on"); +#endif +} + +void abort_hooks(void) +{ + fflush(stdout); + fprintf(stderr, "running %s()...\n", __func__); + tracing_off(); +#ifdef SLEEP_ON_ABORT + sleep(SLEEP_ON_ABORT); +#endif +} + int sys_pkey_alloc(unsigned long flags, unsigned long init_val) { int ret = syscall(SYS_pkey_alloc, flags, init_val); diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index ca27200596a4..d8d9d1de57b8 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -14,7 +14,7 @@ #include #include "kselftest_harness.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "vm_util.h" #ifndef PR_THP_DISABLE_EXCEPT_ADVISED diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c index 2085982dba69..ae6e1530b354 100644 --- a/tools/testing/selftests/mm/protection_keys.c +++ b/tools/testing/selftests/mm/protection_keys.c @@ -46,13 +46,10 @@ #include #include +#include "hugepage_settings.h" #include "pkey-helpers.h" -int iteration_nr = 1; -int test_nr; - u64 shadow_pkey_reg; -int dprint_in_signal; noinline int read_ptr(int *ptr) { @@ -61,88 +58,6 @@ noinline int read_ptr(int *ptr) return *ptr; } -static void cat_into_file(char *str, char *file) -{ - int fd = open(file, O_RDWR); - int ret; - - dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file); - /* - * these need to be raw because they are called under - * pkey_assert() - */ - if (fd < 0) { - fprintf(stderr, "error opening '%s'\n", str); - perror("error: "); - exit(__LINE__); - } - - ret = write(fd, str, strlen(str)); - if (ret != strlen(str)) { - perror("write to file failed"); - fprintf(stderr, "filename: '%s' str: '%s'\n", file, str); - exit(__LINE__); - } - close(fd); -} - -#if CONTROL_TRACING > 0 -static int warned_tracing; -static int tracing_root_ok(void) -{ - if (geteuid() != 0) { - if (!warned_tracing) - fprintf(stderr, "WARNING: not run as root, " - "can not do tracing control\n"); - warned_tracing = 1; - return 0; - } - return 1; -} -#endif - -static void tracing_on(void) -{ -#if CONTROL_TRACING > 0 -#define TRACEDIR "/sys/kernel/tracing" - char pidstr[32]; - - if (!tracing_root_ok()) - return; - - sprintf(pidstr, "%d", getpid()); - cat_into_file("0", TRACEDIR "/tracing_on"); - cat_into_file("\n", TRACEDIR "/trace"); - if (1) { - cat_into_file("function_graph", TRACEDIR "/current_tracer"); - cat_into_file("1", TRACEDIR "/options/funcgraph-proc"); - } else { - cat_into_file("nop", TRACEDIR "/current_tracer"); - } - cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid"); - cat_into_file("1", TRACEDIR "/tracing_on"); - dprintf1("enabled tracing\n"); -#endif -} - -static void tracing_off(void) -{ -#if CONTROL_TRACING > 0 - if (!tracing_root_ok()) - return; - cat_into_file("0", "/sys/kernel/tracing/tracing_on"); -#endif -} - -void abort_hooks(void) -{ - fprintf(stderr, "running %s()...\n", __func__); - tracing_off(); -#ifdef SLEEP_ON_ABORT - sleep(SLEEP_ON_ABORT); -#endif -} - /* * This attempts to have roughly a page of instructions followed by a few * instructions that do a write, and another page of instructions. That @@ -370,8 +285,8 @@ static void signal_handler(int signum, siginfo_t *si, void *vucontext) if ((si->si_code == SEGV_MAPERR) || (si->si_code == SEGV_ACCERR) || (si->si_code == SEGV_BNDERR)) { - printf("non-PK si_code, exiting...\n"); - exit(4); + dprintf0("# non-PK si_code: %d, exiting...\n", si->si_code); + exit(1); } si_pkey_ptr = siginfo_get_pkey_ptr(si); @@ -667,7 +582,7 @@ static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey) size, prot, pkey); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey); pkey_assert(!ret); record_pkey_malloc(ptr, size, prot); @@ -690,7 +605,7 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) */ size = ALIGN_UP(size, HPAGE_SIZE * 2); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); record_pkey_malloc(ptr, size, prot); mprotect_pkey(ptr, size, prot, pkey); @@ -708,50 +623,28 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey) } static int hugetlb_setup_ok; -#define SYSFS_FMT_NR_HUGE_PAGES "/sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages" #define GET_NR_HUGE_PAGES 10 static void setup_hugetlbfs(void) { - int err; - int fd; - char buf[256]; - long hpagesz_kb; - long hpagesz_mb; + long hpagesz_mb = HPAGE_SIZE / 1024 / 1024; + unsigned long free_pages; if (geteuid() != 0) { - fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n"); + ksft_print_msg("WARNING: not run as root, can not do hugetlb test\n"); return; } - cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages"); - /* - * Now go make sure that we got the pages and that they + * Make sure that we got the pages and that they * are PMD-level pages. Someone might have made PUD-level * pages the default. */ - hpagesz_kb = HPAGE_SIZE / 1024; - hpagesz_mb = hpagesz_kb / 1024; - sprintf(buf, SYSFS_FMT_NR_HUGE_PAGES, hpagesz_kb); - fd = open(buf, O_RDONLY); - if (fd < 0) { - fprintf(stderr, "opening sysfs %ldM hugetlb config: %s\n", - hpagesz_mb, strerror(errno)); - return; - } - - /* -1 to guarantee leaving the trailing \0 */ - err = read(fd, buf, sizeof(buf)-1); - close(fd); - if (err <= 0) { - fprintf(stderr, "reading sysfs %ldM hugetlb config: %s\n", - hpagesz_mb, strerror(errno)); - return; - } - - if (atoi(buf) != GET_NR_HUGE_PAGES) { - fprintf(stderr, "could not confirm %ldM pages, got: '%s' expected %d\n", - hpagesz_mb, buf, GET_NR_HUGE_PAGES); + hugetlb_save_settings(); + hugetlb_set_nr_pages(HPAGE_SIZE, GET_NR_HUGE_PAGES); + free_pages = hugetlb_free_pages(HPAGE_SIZE); + if (free_pages < GET_NR_HUGE_PAGES) { + ksft_print_msg("could not confirm %ldM pages, got: '%lu' expected %d\n", + hpagesz_mb, free_pages, GET_NR_HUGE_PAGES); return; } @@ -770,7 +663,7 @@ static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey) size = ALIGN_UP(size, HPAGE_SIZE * 2); pkey_assert(pkey < NR_PKEYS); ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0); - pkey_assert(ptr != (void *)-1); + pkey_assert(ptr != MAP_FAILED); mprotect_pkey(ptr, size, prot, pkey); record_pkey_malloc(ptr, size, prot); @@ -799,7 +692,7 @@ static void *malloc_pkey(long size, int prot, u16 pkey) pkey_assert(malloc_type < nr_malloc_types); ret = pkey_malloc[malloc_type](size, prot, pkey); - pkey_assert(ret != (void *)-1); + pkey_assert(ret != MAP_FAILED); malloc_type++; if (malloc_type >= nr_malloc_types) @@ -855,7 +748,7 @@ void expected_pkey_fault(int pkey) #define do_not_expect_pkey_fault(msg) do { \ if (last_pkey_faults != pkey_faults) \ - dprintf0("unexpected PKey fault: %s\n", msg); \ + dprintf0("# unexpected PKey fault: %s\n", msg); \ pkey_assert(last_pkey_faults == pkey_faults); \ } while (0) @@ -1128,7 +1021,7 @@ static void become_child(void) /* in the child */ return; } - exit(0); + _exit(0); } /* Assumes that all pkeys other than 'pkey' are unallocated */ @@ -1217,6 +1110,7 @@ static void arch_force_pkey_reg_init(void) * doing the XSAVE size enumeration dance. */ buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + pkey_assert(buf != MAP_FAILED); /* These __builtins require compiling with -mxsave */ @@ -1507,18 +1401,18 @@ static void test_ptrace_modifies_pkru(int *ptr, u16 pkey) * checking */ if (__read_pkey_reg() != new_pkru) - exit(1); + _exit(1); /* Stop and allow the tracer to clear XSTATE_BV for PKRU */ raise(SIGSTOP); if (__read_pkey_reg() != 0) - exit(1); + _exit(1); /* Stop and allow the tracer to examine PKRU */ raise(SIGSTOP); - exit(0); + _exit(0); } pkey_assert(child == waitpid(child, &status, 0)); @@ -1692,29 +1586,36 @@ static void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey) pkey_assert(sret < 0); } -static void (*pkey_tests[])(int *ptr, u16 pkey) = { - test_read_of_write_disabled_region, - test_read_of_access_disabled_region, - test_read_of_access_disabled_region_with_page_already_mapped, - test_write_of_write_disabled_region, - test_write_of_write_disabled_region_with_page_already_mapped, - test_write_of_access_disabled_region, - test_write_of_access_disabled_region_with_page_already_mapped, - test_kernel_write_of_access_disabled_region, - test_kernel_write_of_write_disabled_region, - test_kernel_gup_of_access_disabled_region, - test_kernel_gup_write_to_write_disabled_region, - test_executing_on_unreadable_memory, - test_implicit_mprotect_exec_only_memory, - test_mprotect_with_pkey_0, - test_ptrace_of_child, - test_pkey_init_state, - test_pkey_syscalls_on_non_allocated_pkey, - test_pkey_syscalls_bad_args, - test_pkey_alloc_exhaust, - test_pkey_alloc_free_attach_pkey0, +struct pkey_test { + void (*func)(int *ptr, u16 pkey); + const char *name; +}; + +#define PKEY_TEST(fn) { fn, #fn } + +static struct pkey_test pkey_tests[] = { + PKEY_TEST(test_read_of_write_disabled_region), + PKEY_TEST(test_read_of_access_disabled_region), + PKEY_TEST(test_read_of_access_disabled_region_with_page_already_mapped), + PKEY_TEST(test_write_of_write_disabled_region), + PKEY_TEST(test_write_of_write_disabled_region_with_page_already_mapped), + PKEY_TEST(test_write_of_access_disabled_region), + PKEY_TEST(test_write_of_access_disabled_region_with_page_already_mapped), + PKEY_TEST(test_kernel_write_of_access_disabled_region), + PKEY_TEST(test_kernel_write_of_write_disabled_region), + PKEY_TEST(test_kernel_gup_of_access_disabled_region), + PKEY_TEST(test_kernel_gup_write_to_write_disabled_region), + PKEY_TEST(test_executing_on_unreadable_memory), + PKEY_TEST(test_implicit_mprotect_exec_only_memory), + PKEY_TEST(test_mprotect_with_pkey_0), + PKEY_TEST(test_ptrace_of_child), + PKEY_TEST(test_pkey_init_state), + PKEY_TEST(test_pkey_syscalls_on_non_allocated_pkey), + PKEY_TEST(test_pkey_syscalls_bad_args), + PKEY_TEST(test_pkey_alloc_exhaust), + PKEY_TEST(test_pkey_alloc_free_attach_pkey0), #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) - test_ptrace_modifies_pkru, + PKEY_TEST(test_ptrace_modifies_pkru), #endif }; @@ -1735,7 +1636,7 @@ static void run_tests_once(void) dprintf1("test %d starting with pkey: %d\n", test_nr, pkey); ptr = malloc_pkey(PAGE_SIZE, prot, pkey); dprintf1("test %d starting...\n", test_nr); - pkey_tests[test_nr](ptr, pkey); + pkey_tests[test_nr].func(ptr, pkey); dprintf1("freeing test memory: %p\n", ptr); free_pkey_malloc(ptr); sys_pkey_free(pkey); @@ -1746,7 +1647,7 @@ static void run_tests_once(void) tracing_off(); close_test_fds(); - printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr); + ksft_test_result_pass("test %s (iteration %d)\n", pkey_tests[test_nr].name, iteration_nr); dprintf1("======================\n\n"); } iteration_nr++; @@ -1766,27 +1667,31 @@ int main(void) setup_handlers(); - printf("has pkeys: %d\n", pkeys_supported); + ksft_print_header(); if (!pkeys_supported) { int size = PAGE_SIZE; int *ptr; - printf("running PKEY tests for unsupported CPU/OS\n"); + ksft_set_plan(1); + ksft_print_msg("running PKEY tests for unsupported CPU/OS\n"); ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - assert(ptr != (void *)-1); + if (ptr == MAP_FAILED) + ksft_exit_fail_perror("mmap"); test_mprotect_pkey_on_unsupported_cpu(ptr, 1); - exit(0); + ksft_test_result_pass("pkey on unsupported CPU/OS\n"); + ksft_finished(); } + ksft_set_plan(ARRAY_SIZE(pkey_tests) * nr_iterations); + pkey_setup_shadow(); - printf("startup pkey_reg: %016llx\n", read_pkey_reg()); + ksft_print_msg("startup pkey_reg: %016llx\n", read_pkey_reg()); setup_hugetlbfs(); while (nr_iterations-- > 0) run_tests_once(); - printf("done (all tests OK)\n"); - return 0; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 3b61677fe984..8c296dedf047 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -132,7 +132,7 @@ test_selected() { run_gup_matrix() { # -t: thp=on, -T: thp=off, -H: hugetlb=on - local hugetlb_mb=$(( needmem_KB / 1024 )) + local hugetlb_mb=256 for huge in -t -T "-H -m $hugetlb_mb"; do # -u: gup-fast, -U: gup-basic, -a: pin-fast, -b: pin-basic, -L: pin-longterm @@ -154,60 +154,6 @@ run_gup_matrix() { done } -# get huge pagesize and freepages from /proc/meminfo -while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs="$size" - fi - if [ "$name" = "Hugepagesize:" ]; then - hpgsize_KB="$size" - fi -done < /proc/meminfo - -# Simple hugetlbfs tests have a hardcoded minimum requirement of -# huge pages totaling 256MB (262144KB) in size. The userfaultfd -# hugetlb test requires a minimum of 2 * nr_cpus huge pages. Take -# both of these requirements into account and attempt to increase -# number of huge pages available. -nr_cpus=$(nproc) -uffd_min_KB=$((hpgsize_KB * nr_cpus * 2)) -hugetlb_min_KB=$((256 * 1024)) -if [[ $uffd_min_KB -gt $hugetlb_min_KB ]]; then - needmem_KB=$uffd_min_KB -else - needmem_KB=$hugetlb_min_KB -fi - -# set proper nr_hugepages -if [ -n "$freepgs" ] && [ -n "$hpgsize_KB" ]; then - orig_nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - needpgs=$((needmem_KB / hpgsize_KB)) - tries=2 - while [ "$tries" -gt 0 ] && [ "$freepgs" -lt "$needpgs" ]; do - lackpgs=$((needpgs - freepgs)) - echo 3 > /proc/sys/vm/drop_caches - if ! echo $((lackpgs + orig_nr_hugepgs)) > /proc/sys/vm/nr_hugepages; then - echo "Please run this test as root" - exit $ksft_skip - fi - while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs=$size - fi - done < /proc/meminfo - tries=$((tries - 1)) - done - nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - if [ "$freepgs" -lt "$needpgs" ]; then - printf "Not enough huge pages available (%d < %d)\n" \ - "$freepgs" "$needpgs" - fi - HAVE_HUGEPAGES=1 -else - echo "no hugetlbfs support in kernel?" - HAVE_HUGEPAGES=0 -fi - # filter 64bit architectures ARCH64STR="arm64 mips64 parisc64 ppc64 ppc64le riscv64 s390x sparc64 x86_64" if [ -z "$ARCH" ]; then @@ -235,32 +181,61 @@ pretty_name() { run_test() { if test_selected ${CATEGORY}; then local skip=0 + local LOADED_HWPOISON_INJECT_MOD=0 # On memory constrainted systems some tests can fail to allocate hugepages. # perform some cleanup before the test for a higher success rate. if [ ${CATEGORY} == "thp" -o ${CATEGORY} == "hugetlb" ]; then - if [ "${HAVE_HUGEPAGES}" = "1" ]; then + mem_kb=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + mem_Mb=$((mem_kb / 1024)) + + if (( $mem_Mb < 256 )); then echo 3 > /proc/sys/vm/drop_caches sleep 2 echo 1 > /proc/sys/vm/compact_memory sleep 2 - else - echo "hugepages not supported" | tap_prefix - skip=1 fi fi + # Ensure hwpoison_inject is available for memory-failure tests + if [ "${CATEGORY}" = "memory-failure" ]; then + # Try to load hwpoison_inject if not present. + HWPOISON_DIR=/sys/kernel/debug/hwpoison/ + if [ ! -d "$HWPOISON_DIR" ]; then + if ! modprobe -n hwpoison_inject > /dev/null 2>&1; then + echo "Module hwpoison_inject not found, skipping..." \ + | tap_prefix + skip=1 + else + modprobe hwpoison_inject > /dev/null 2>&1 + LOADED_HWPOISON_INJECT_MOD=1 + if [ ! -d "$HWPOISON_DIR" ]; then + echo "hwpoison debugfs interface not present" \ + | tap_prefix + skip=1 + fi + fi + fi + + fi + local test=$(pretty_name "$*") local title="running $*" local sep=$(echo -n "$title" | tr "[:graph:][:space:]" -) printf "%s\n%s\n%s\n" "$sep" "$title" "$sep" | tap_prefix - if [ "${skip}" != "1" ]; then + if [ $skip -eq 1 ]; then + local ret=$ksft_skip + else ("$@" 2>&1) | tap_prefix local ret=${PIPESTATUS[0]} - else - local ret=$ksft_skip fi + + # Unload hwpoison_inject if we loaded it + if [ "${LOADED_HWPOISON_INJECT_MOD}" = "1" ]; then + modprobe -r hwpoison_inject > /dev/null 2>&1 + fi + count_total=$(( count_total + 1 )) if [ $ret -eq 0 ]; then count_pass=$(( count_pass + 1 )) @@ -270,7 +245,9 @@ run_test() { count_skip=$(( count_skip + 1 )) echo "[SKIP]" | tap_prefix echo "ok ${count_total} ${test} # SKIP" | tap_output - exitcode=$ksft_skip + if [ $exitcode -eq 0 ]; then + exitcode=$ksft_skip + fi else count_fail=$(( count_fail + 1 )) echo "[FAIL]" | tap_prefix @@ -282,31 +259,14 @@ run_test() { echo "TAP version 13" | tap_output -CATEGORY="hugetlb" run_test ./hugepage-mmap - -shmmax=$(cat /proc/sys/kernel/shmmax) -shmall=$(cat /proc/sys/kernel/shmall) -echo 268435456 > /proc/sys/kernel/shmmax -echo 4194304 > /proc/sys/kernel/shmall -CATEGORY="hugetlb" run_test ./hugepage-shm -echo "$shmmax" > /proc/sys/kernel/shmmax -echo "$shmall" > /proc/sys/kernel/shmall - -CATEGORY="hugetlb" run_test ./map_hugetlb -CATEGORY="hugetlb" run_test ./hugepage-mremap -CATEGORY="hugetlb" run_test ./hugepage-vmemmap +CATEGORY="hugetlb" run_test ./hugetlb-mmap +CATEGORY="hugetlb" run_test ./hugetlb-shm +CATEGORY="hugetlb" run_test ./hugetlb-mremap +CATEGORY="hugetlb" run_test ./hugetlb-vmemmap CATEGORY="hugetlb" run_test ./hugetlb-madvise CATEGORY="hugetlb" run_test ./hugetlb_dio - -if [ "${HAVE_HUGEPAGES}" = "1" ]; then - nr_hugepages_tmp=$(cat /proc/sys/vm/nr_hugepages) - # For this test, we need one and just one huge page - echo 1 > /proc/sys/vm/nr_hugepages - CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv - CATEGORY="hugetlb" run_test ./hugetlb_madv_vs_map - # Restore the previous number of huge pages, since further tests rely on it - echo "$nr_hugepages_tmp" > /proc/sys/vm/nr_hugepages -fi +CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv +CATEGORY="hugetlb" run_test ./hugetlb_madv_vs_map if test_selected "hugetlb"; then echo "NOTE: These hugetlb tests provide minimal coverage. Use" | tap_prefix @@ -333,44 +293,11 @@ CATEGORY="gup_test" run_test ./gup_longterm CATEGORY="userfaultfd" run_test ./uffd-unit-tests uffd_stress_bin=./uffd-stress CATEGORY="userfaultfd" run_test ${uffd_stress_bin} anon 20 16 -# Hugetlb tests require source and destination huge pages. Pass in almost half -# the size of the free pages we have, which is used for *each*. An adjustment -# of (nr_parallel - 1) is done (see nr_parallel in uffd-stress.c) to have some -# extra hugepages - this is done to prevent the test from failing by racily -# reserving more hugepages than strictly required. -# uffd-stress expects a region expressed in MiB, so we adjust -# half_ufd_size_MB accordingly. -adjustment=$(( (31 < (nr_cpus - 1)) ? 31 : (nr_cpus - 1) )) -half_ufd_size_MB=$((((freepgs - adjustment) * hpgsize_KB) / 1024 / 2)) -CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb "$half_ufd_size_MB" 32 -CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb-private "$half_ufd_size_MB" 32 +CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb 128 32 +CATEGORY="userfaultfd" run_test ${uffd_stress_bin} hugetlb-private 128 32 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} shmem 20 16 CATEGORY="userfaultfd" run_test ${uffd_stress_bin} shmem-private 20 16 -# uffd-wp-mremap requires at least one page of each size. -have_all_size_hugepgs=true -declare -A nr_size_hugepgs -for f in /sys/kernel/mm/hugepages/**/nr_hugepages; do - old=$(cat $f) - nr_size_hugepgs["$f"]="$old" - if [ "$old" == 0 ]; then - echo 1 > "$f" - fi - if [ $(cat "$f") == 0 ]; then - have_all_size_hugepgs=false - break - fi -done -if $have_all_size_hugepgs; then - CATEGORY="userfaultfd" run_test ./uffd-wp-mremap -else - echo "# SKIP ./uffd-wp-mremap" -fi - -#cleanup -for f in "${!nr_size_hugepgs[@]}"; do - echo "${nr_size_hugepgs["$f"]}" > "$f" -done -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages +CATEGORY="userfaultfd" run_test ./uffd-wp-mremap CATEGORY="compaction" run_test ./compaction_test @@ -395,12 +322,10 @@ CATEGORY="mremap" run_test ./mremap_test CATEGORY="hugetlb" run_test ./thuge-gen CATEGORY="hugetlb" run_test ./charge_reserved_hugetlb.sh -cgroup-v2 CATEGORY="hugetlb" run_test ./hugetlb_reparenting_test.sh -cgroup-v2 + if $RUN_DESTRUCTIVE; then -nr_hugepages_tmp=$(cat /proc/sys/vm/nr_hugepages) enable_soft_offline=$(cat /proc/sys/vm/enable_soft_offline) -echo 8 > /proc/sys/vm/nr_hugepages CATEGORY="hugetlb" run_test ./hugetlb-soft-offline -echo "$nr_hugepages_tmp" > /proc/sys/vm/nr_hugepages echo "$enable_soft_offline" > /proc/sys/vm/enable_soft_offline CATEGORY="hugetlb" run_test ./hugetlb-read-hwpoison fi @@ -437,9 +362,7 @@ CATEGORY="memfd_secret" run_test ./memfd_secret fi # KSM KSM_MERGE_TIME_HUGE_PAGES test with size of 100 -if [ "${HAVE_HUGEPAGES}" = "1" ]; then - CATEGORY="ksm" run_test ./ksm_tests -H -s 100 -fi +CATEGORY="ksm" run_test ./ksm_tests -H -s 100 # KSM KSM_MERGE_TIME test with size of 100 CATEGORY="ksm" run_test ./ksm_tests -P -s 100 # KSM MADV_MERGEABLE test with 10 identical pages @@ -458,7 +381,6 @@ CATEGORY="ksm_numa" run_test ./ksm_tests -N -m 0 CATEGORY="ksm" run_test ./ksm_functional_tests # protection_keys tests -nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) if [ -x ./protection_keys_32 ] then CATEGORY="pkey" run_test ./protection_keys_32 @@ -468,7 +390,6 @@ if [ -x ./protection_keys_64 ] then CATEGORY="pkey" run_test ./protection_keys_64 fi -echo "$nr_hugepgs" > /proc/sys/vm/nr_hugepages if [ -x ./soft-dirty ] then @@ -490,24 +411,28 @@ CATEGORY="thp" run_test ./khugepaged all:shmem CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem -CATEGORY="thp" run_test ./transhuge-stress -d 20 - # Try to create XFS if not provided if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then - if [ "${HAVE_HUGEPAGES}" = "1" ]; then - if test_selected "thp"; then - if grep xfs /proc/filesystems &>/dev/null; then - XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) - SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) - truncate -s 314572800 ${XFS_IMG} - mkfs.xfs -q ${XFS_IMG} - mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} - MOUNTED_XFS=1 - fi + if test_selected "thp"; then + if grep xfs /proc/filesystems &>/dev/null; then + XFS_IMG=$(mktemp /tmp/xfs_img_XXXXXX) + SPLIT_HUGE_PAGE_TEST_XFS_PATH=$(mktemp -d /tmp/xfs_dir_XXXXXX) + truncate -s 314572800 ${XFS_IMG} + mkfs.xfs -q ${XFS_IMG} + mount -o loop ${XFS_IMG} ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} + MOUNTED_XFS=1 fi fi fi +if [ -n "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then +CATEGORY="thp" run_test ./khugepaged all:file ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} +elif test_selected thp; then + count_total=$(( count_total + 1 )) + count_skip=$(( count_skip + 1 )) + echo "[SKIP] ./khugepaged all:file" | tap_prefix +fi + CATEGORY="thp" run_test ./split_huge_page_test ${SPLIT_HUGE_PAGE_TEST_XFS_PATH} if [ -n "${MOUNTED_XFS}" ]; then @@ -516,6 +441,8 @@ if [ -n "${MOUNTED_XFS}" ]; then rm -f ${XFS_IMG} fi +CATEGORY="thp" run_test ./transhuge-stress -d 20 + CATEGORY="thp" run_test ./folio_split_race_test CATEGORY="migration" run_test ./migration @@ -532,28 +459,7 @@ CATEGORY="page_frag" run_test ./test_page_frag.sh nonaligned CATEGORY="rmap" run_test ./rmap -# Try to load hwpoison_inject if not present. -HWPOISON_DIR=/sys/kernel/debug/hwpoison/ -if [ ! -d "$HWPOISON_DIR" ]; then - if ! modprobe -q -R hwpoison_inject; then - echo "Module hwpoison_inject not found, skipping..." - else - modprobe hwpoison_inject > /dev/null 2>&1 - LOADED_MOD=1 - fi -fi - -if [ -d "$HWPOISON_DIR" ]; then - CATEGORY="memory-failure" run_test ./memory-failure -fi - -if [ -n "${LOADED_MOD}" ]; then - modprobe -r hwpoison_inject > /dev/null 2>&1 -fi - -if [ "${HAVE_HUGEPAGES}" = 1 ]; then - echo "$orig_nr_hugepgs" > /proc/sys/vm/nr_hugepages -fi +CATEGORY="memory-failure" run_test ./memory-failure echo "SUMMARY: PASS=${count_pass} SKIP=${count_skip} FAIL=${count_fail}" | tap_prefix echo "1..${count_total}" | tap_output diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index bcfcac99b436..fb1864a68e1c 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -9,7 +9,7 @@ #include "kselftest.h" #include "vm_util.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #define PAGEMAP_FILE_PATH "/proc/self/pagemap" #define TEST_ITERATIONS 10000 @@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) if (anon) { map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("anon mmap failed\n"); } else { test_fd = open(fname, O_RDWR | O_CREAT, 0664); @@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon) ftruncate(test_fd, pagesize); map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_SHARED, test_fd, 0); - if (!map) + if (map == MAP_FAILED) ksft_exit_fail_msg("file mmap failed\n"); } diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 40a5093917e7..32b991472f74 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -21,7 +21,7 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" uint64_t pagesize; unsigned int pageshift; @@ -470,13 +470,18 @@ static void split_file_backed_thp(int order) char tmpfs_template[] = "/tmp/thp_split_XXXXXX"; const char *tmpfs_loc = mkdtemp(tmpfs_template); char testfile[INPUT_MAX]; + unsigned long size = 2 * pmd_pagesize; + char opts[64]; ssize_t num_written, num_read; - char *file_buf1, *file_buf2; + char *file_buf1 = NULL, *file_buf2 = NULL; uint64_t pgoff_start = 0, pgoff_end = 1024; int i; ksft_print_msg("Please enable pr_debug in split_huge_pages_in_file() for more info.\n"); + if (!tmpfs_loc) + ksft_exit_fail_msg("mkdtemp failed\n"); + file_buf1 = (char *)malloc(pmd_pagesize); file_buf2 = (char *)malloc(pmd_pagesize); @@ -489,10 +494,13 @@ static void split_file_backed_thp(int order) file_buf1[i] = (char)i; memset(file_buf2, 0, pmd_pagesize); - status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m"); + snprintf(opts, sizeof(opts), "huge=always,size=%lu", size); + status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, opts); - if (status) - ksft_exit_fail_msg("Unable to create a tmpfs for testing\n"); + if (status) { + ksft_print_msg("Unable to create a tmpfs for testing\n"); + goto out; + } status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc); if (status >= INPUT_MAX) { @@ -544,10 +552,13 @@ static void split_file_backed_thp(int order) status = umount(tmpfs_loc); if (status) { - rmdir(tmpfs_loc); - ksft_exit_fail_msg("Unable to umount %s\n", tmpfs_loc); + ksft_print_msg("Unable to umount %s\n", tmpfs_loc); + goto out; } + free(file_buf1); + free(file_buf2); + status = rmdir(tmpfs_loc); if (status) ksft_exit_fail_msg("cannot remove tmp dir: %s\n", strerror(errno)); @@ -560,8 +571,10 @@ static void split_file_backed_thp(int order) close(fd); cleanup: umount(tmpfs_loc); - rmdir(tmpfs_loc); out: + free(file_buf1); + free(file_buf2); + rmdir(tmpfs_loc); ksft_exit_fail_msg("Error occurred\n"); } diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 77813d34dcc2..22b9c2f1c35d 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -1,17 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/* Test selecting other page sizes for mmap/shmget. - - Before running this huge pages for each huge page size must have been - reserved. - For large pages beyond MAX_PAGE_ORDER (like 1GB on x86) boot options must - be used. 1GB wouldn't be tested if it isn't available. - Also shmmax must be increased. - And you need to run as root to work around some weird permissions in shm. - And nothing using huge pages should run in parallel. - When the program aborts you may need to clean up the shm segments with - ipcrm -m by hand, like this - sudo ipcs | awk '$1 == "0x00000000" {print $2}' | xargs -n1 sudo ipcrm -m - (warning this will remove all if someone else uses them) */ +/* Test selecting other page sizes for mmap/shmget. */ #define _GNU_SOURCE #include @@ -21,13 +9,12 @@ #include #include #include -#include -#include #include #include #include #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" #if !defined(MAP_HUGETLB) #define MAP_HUGETLB 0x40000 @@ -37,15 +24,6 @@ #ifndef SHM_HUGE_SHIFT #define SHM_HUGE_SHIFT 26 #endif -#ifndef SHM_HUGE_MASK -#define SHM_HUGE_MASK 0x3f -#endif -#ifndef SHM_HUGE_2MB -#define SHM_HUGE_2MB (21 << SHM_HUGE_SHIFT) -#endif -#ifndef SHM_HUGE_1GB -#define SHM_HUGE_1GB (30 << SHM_HUGE_SHIFT) -#endif #define NUM_PAGESIZES 5 #define NUM_PAGES 4 @@ -63,32 +41,10 @@ int ilog2(unsigned long v) void show(unsigned long ps) { - char buf[100]; - if (ps == getpagesize()) return; - ksft_print_msg("%luMB: ", ps >> 20); - - fflush(stdout); - snprintf(buf, sizeof buf, - "cat /sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages", - ps >> 10); - system(buf); -} - -unsigned long read_free(unsigned long ps) -{ - unsigned long val = 0; - char buf[100]; - - snprintf(buf, sizeof(buf), - "/sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages", - ps >> 10); - if (read_sysfs(buf, &val) && ps != getpagesize()) - ksft_print_msg("missing %s\n", buf); - - return val; + ksft_print_msg("%luMB: %lu\n", ps >> 20, hugetlb_free_pages(ps)); } void test_mmap(unsigned long size, unsigned flags) @@ -96,14 +52,14 @@ void test_mmap(unsigned long size, unsigned flags) char *map; unsigned long before, after; - before = read_free(size); + before = hugetlb_free_pages(size); map = mmap(NULL, size*NUM_PAGES, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|flags, -1, 0); if (map == MAP_FAILED) ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); memset(map, 0xff, size*NUM_PAGES); - after = read_free(size); + after = hugetlb_free_pages(size); show(size); ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, @@ -120,7 +76,7 @@ void test_shmget(unsigned long size, unsigned flags) struct shm_info i; char *map; - before = read_free(size); + before = hugetlb_free_pages(size); id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags); if (id < 0) { if (errno == EPERM) { @@ -141,7 +97,7 @@ void test_shmget(unsigned long size, unsigned flags) shmctl(id, IPC_RMID, NULL); memset(map, 0xff, size*NUM_PAGES); - after = read_free(size); + after = hugetlb_free_pages(size); show(size); ksft_test_result(size == getpagesize() || (before - after) == NUM_PAGES, @@ -153,43 +109,15 @@ void test_shmget(unsigned long size, unsigned flags) void find_pagesizes(void) { unsigned long largest = getpagesize(); - unsigned long shmmax_val = 0; int i; - glob_t g; - glob("/sys/kernel/mm/hugepages/hugepages-*kB", 0, NULL, &g); - assert(g.gl_pathc <= NUM_PAGESIZES); - for (i = 0; (i < g.gl_pathc) && (num_page_sizes < NUM_PAGESIZES); i++) { - sscanf(g.gl_pathv[i], "/sys/kernel/mm/hugepages/hugepages-%lukB", - &page_sizes[num_page_sizes]); - page_sizes[num_page_sizes] <<= 10; - ksft_print_msg("Found %luMB\n", page_sizes[i] >> 20); + num_page_sizes = hugetlb_setup(NUM_PAGES, page_sizes, ARRAY_SIZE(page_sizes)); - if (page_sizes[num_page_sizes] > largest) + for (i = 0; i < num_page_sizes; i++) + if (page_sizes[i] > largest) largest = page_sizes[i]; - if (read_free(page_sizes[num_page_sizes]) >= NUM_PAGES) - num_page_sizes++; - else - ksft_print_msg("SKIP for size %lu MB as not enough huge pages, need %u\n", - page_sizes[num_page_sizes] >> 20, NUM_PAGES); - } - globfree(&g); - - read_sysfs("/proc/sys/kernel/shmmax", &shmmax_val); - if (shmmax_val < NUM_PAGES * largest) { - ksft_print_msg("WARNING: shmmax is too small to run this test.\n"); - ksft_print_msg("Please run the following command to increase shmmax:\n"); - ksft_print_msg("echo %lu > /proc/sys/kernel/shmmax\n", largest * NUM_PAGES); - ksft_exit_skip("Test skipped due to insufficient shmmax value.\n"); - } - -#if defined(__x86_64__) - if (largest != 1U<<30) { - ksft_exit_skip("No GB pages available on x86-64\n" - "Please boot with hugepagesz=1G hugepages=%d\n", NUM_PAGES); - } -#endif + shm_limits_prepare(NUM_PAGES * largest); } int main(void) @@ -232,3 +160,5 @@ int main(void) ksft_finished(); } + +SHM_LIMITS_RESTORE() diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 7a9f1035099b..8eb0c5630e7e 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -17,7 +17,7 @@ #include #include "vm_util.h" #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" int backing_fd = -1; int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE; diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c index edd02328f77b..f48f5d4594ab 100644 --- a/tools/testing/selftests/mm/uffd-common.c +++ b/tools/testing/selftests/mm/uffd-common.c @@ -639,8 +639,13 @@ int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) { - /* real retval in ufdio_copy.copy */ - if (uffdio_copy.copy != -EEXIST) + /* + * real retval in uffdio_copy.copy + * + * -EEXIST: the page was faulted in concurrently + * -ENOENT: the destination range was concurrently removed + */ + if (uffdio_copy.copy != -EEXIST && uffdio_copy.copy != -ENOENT) err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy); wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size); diff --git a/tools/testing/selftests/mm/uffd-common.h b/tools/testing/selftests/mm/uffd-common.h index 844a85ab31eb..92a21b97f745 100644 --- a/tools/testing/selftests/mm/uffd-common.h +++ b/tools/testing/selftests/mm/uffd-common.h @@ -37,24 +37,24 @@ #include "kselftest.h" #include "vm_util.h" +#include "hugepage_settings.h" #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) -#define _err(fmt, ...) \ - do { \ - int ret = errno; \ - fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__); \ - fprintf(stderr, " (errno=%d, @%s:%d)\n", \ - ret, __FILE__, __LINE__); \ +#define _err(fmt, ...) \ + do { \ + int ret = errno; \ + ksft_print_msg("ERROR: " fmt " (errno=%d, @%s:%d)\n", \ + ##__VA_ARGS__, ret, __FILE__, __LINE__); \ } while (0) -#define errexit(exitcode, fmt, ...) \ +#define errexit(fmt, ...) \ do { \ _err(fmt, ##__VA_ARGS__); \ - exit(exitcode); \ + ksft_exit_fail(); \ } while (0) -#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__) +#define err(fmt, ...) errexit(fmt, ##__VA_ARGS__) struct uffd_global_test_opts { unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; diff --git a/tools/testing/selftests/mm/uffd-stress.c b/tools/testing/selftests/mm/uffd-stress.c index 700fbaa18d44..3401dd6028f0 100644 --- a/tools/testing/selftests/mm/uffd-stress.c +++ b/tools/testing/selftests/mm/uffd-stress.c @@ -286,18 +286,12 @@ static int userfaultfd_stress(uffd_global_test_opts_t *gopts) pthread_attr_setstacksize(&attr, 16*1024*1024); while (bounces--) { - printf("bounces: %d, mode:", bounces); - if (bounces & BOUNCE_RANDOM) - printf(" rnd"); - if (bounces & BOUNCE_RACINGFAULTS) - printf(" racing"); - if (bounces & BOUNCE_VERIFY) - printf(" ver"); - if (bounces & BOUNCE_POLL) - printf(" poll"); - else - printf(" read"); - printf(", "); + ksft_print_msg("bounces: %d, mode:%s%s%s%s, ", + bounces, + bounces & BOUNCE_RANDOM ? " rnd" : "", + bounces & BOUNCE_RACINGFAULTS ? " racing" : "", + bounces & BOUNCE_VERIFY ? " ver" : "", + bounces & BOUNCE_POLL ? " poll" : " read"); fflush(stdout); if (bounces & BOUNCE_POLL) @@ -461,6 +455,9 @@ int main(int argc, char **argv) if (argc < 4) usage(); + ksft_print_header(); + ksft_set_plan(1); + if (signal(SIGALRM, sigalrm) == SIG_ERR) err("failed to arm SIGALRM"); alarm(ALARM_INTERVAL_SECS); @@ -483,17 +480,17 @@ int main(int argc, char **argv) * Ensure nr_parallel - 1 hugepages on top of that to account * for racy extra reservation of hugepages. */ - if (gopts->test_type == TEST_HUGETLB && - get_free_hugepages() < 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1) { - printf("skip: Skipping userfaultfd... not enough hugepages\n"); - return KSFT_SKIP; + if (gopts->test_type == TEST_HUGETLB) { + unsigned long nr = 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1; + + if (!hugetlb_setup_default(nr)) + ksft_exit_skip("Skipping userfaultfd... not enough hugepages\n"); } gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel; if (!gopts->nr_pages_per_cpu) { - _err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)", - bytes, gopts->page_size, gopts->nr_parallel); - usage(); + ksft_exit_skip("pages_per_cpu = 0, cannot test (%zu / %lu / %lu)\n", + bytes, gopts->page_size, gopts->nr_parallel); } bounces = atoi(argv[3]); @@ -503,9 +500,12 @@ int main(int argc, char **argv) } gopts->nr_pages = gopts->nr_pages_per_cpu * gopts->nr_parallel; - printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n", - gopts->nr_pages, gopts->nr_pages_per_cpu); - return userfaultfd_stress(gopts); + ksft_print_msg("nr_pages: %lu, nr_pages_per_cpu: %lu\n", + gopts->nr_pages, gopts->nr_pages_per_cpu); + + ksft_test_result(!userfaultfd_stress(gopts), + "uffd-stress %s\n", argv[1]); + ksft_finished(); } #else /* __NR_userfaultfd */ @@ -514,8 +514,8 @@ int main(int argc, char **argv) int main(void) { - printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n"); - return KSFT_SKIP; + ksft_print_header(); + ksft_exit_skip("missing __NR_userfaultfd definition\n"); } #endif /* __NR_userfaultfd */ diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c index 6f5e404a446c..a6c14109e818 100644 --- a/tools/testing/selftests/mm/uffd-unit-tests.c +++ b/tools/testing/selftests/mm/uffd-unit-tests.c @@ -86,47 +86,28 @@ typedef struct { uffd_test_case_ops_t *test_case_ops; } uffd_test_case_t; -static void uffd_test_report(void) -{ - printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n", - ksft_get_pass_cnt(), - ksft_get_xskip_cnt(), - ksft_get_fail_cnt(), - ksft_test_num()); -} +static char current_test[256]; static void uffd_test_pass(void) { - printf("done\n"); - ksft_inc_pass_cnt(); + ksft_test_result_pass("%s\n", current_test); } #define uffd_test_start(...) do { \ - printf("Testing "); \ - printf(__VA_ARGS__); \ - printf("... "); \ - fflush(stdout); \ + snprintf(current_test, sizeof(current_test), __VA_ARGS__); \ } while (0) -#define uffd_test_fail(...) do { \ - printf("failed [reason: "); \ - printf(__VA_ARGS__); \ - printf("]\n"); \ - ksft_inc_fail_cnt(); \ +#define uffd_test_fail(fmt, ...) do { \ + ksft_print_msg("failed reason: [" fmt "]\n", ##__VA_ARGS__); \ + ksft_test_result_fail("%s\n", current_test); \ } while (0) static void uffd_test_skip(const char *message) { - printf("skipped [reason: %s]\n", message); - ksft_inc_xskip_cnt(); + ksft_test_result_skip("%s (%s)\n", current_test, message); } -/* - * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll - * return 1 even if some test failed as long as uffd supported, because in - * that case we still want to proceed with the rest uffd unit tests. - */ -static int test_uffd_api(bool use_dev) +static void test_uffd_api(bool use_dev) { struct uffdio_api uffdio_api; int uffd; @@ -140,7 +121,7 @@ static int test_uffd_api(bool use_dev) uffd = uffd_open_sys(UFFD_FLAGS); if (uffd < 0) { uffd_test_skip("cannot open userfaultfd handle"); - return 0; + return; } /* Test wrong UFFD_API */ @@ -177,8 +158,6 @@ static int test_uffd_api(bool use_dev) uffd_test_pass(); out: close(uffd); - /* We have a valid uffd handle */ - return 1; } @@ -320,7 +299,7 @@ static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bo if (test_pin) unpin_pages(&args); /* Succeed */ - exit(0); + _exit(0); } waitpid(child, &result, 0); @@ -788,7 +767,7 @@ static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp) err("fork"); if (!pid) - exit(faulting_process(gopts, 2, wp)); + _exit(faulting_process(gopts, 2, wp)); waitpid(pid, &err, 0); if (err) @@ -842,7 +821,7 @@ static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp) err("fork"); if (!pid) - exit(faulting_process(gopts, 0, wp)); + _exit(faulting_process(gopts, 0, wp)); waitpid(pid, &err, 0); if (err) @@ -1701,18 +1680,58 @@ static void usage(const char *prog) exit(KSFT_FAIL); } +static int uffd_count_tests(int n_tests, int n_mems, const char *test_filter) +{ + uffd_test_case_t *test; + int i, j, count = 0; + + if (!test_filter) + count += 2; /* test_uffd_api(false) + test_uffd_api(true) */ + + for (i = 0; i < n_tests; i++) { + test = &uffd_tests[i]; + if (test_filter && !strstr(test->name, test_filter)) + continue; + for (j = 0; j < n_mems; j++) + if (test->mem_targets & mem_types[j].mem_flag) + count++; + } + + return count; +} + +static unsigned long uffd_setup_hugetlb(void) +{ + unsigned long nr_hugepages, hp_size; + + hugetlb_save_settings(); + hp_size = default_huge_page_size(); + + if (!hp_size) + return 0; + + /* need twice UFFD_TEST_MEM_SIZE, one for src area and one for dst */ + nr_hugepages = 2 * MAX(UFFD_TEST_MEM_SIZE, hp_size * 2) / hp_size; + hugetlb_set_nr_default_pages(nr_hugepages); + + if (hugetlb_free_default_pages() < nr_hugepages) + return 0; + + return hp_size; +} + int main(int argc, char *argv[]) { int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t); int n_mems = sizeof(mem_types) / sizeof(mem_type_t); const char *test_filter = NULL; + unsigned long hugepage_size; bool list_only = false; uffd_test_case_t *test; mem_type_t *mem_type; uffd_test_args_t args; const char *errmsg; - int has_uffd, opt; - int i, j; + int i, j, opt; while ((opt = getopt(argc, argv, "f:hl")) != -1) { switch (opt) { @@ -1730,24 +1749,30 @@ int main(int argc, char *argv[]) } } - if (!test_filter && !list_only) { - has_uffd = test_uffd_api(false); - has_uffd |= test_uffd_api(true); - - if (!has_uffd) { - printf("Userfaultfd not supported or unprivileged, skip all tests\n"); - exit(KSFT_SKIP); + if (list_only) { + for (i = 0; i < n_tests; i++) { + test = &uffd_tests[i]; + if (test_filter && !strstr(test->name, test_filter)) + continue; + printf("%s\n", test->name); } + return KSFT_PASS; + } + + hugepage_size = uffd_setup_hugetlb(); + + ksft_print_header(); + ksft_set_plan(uffd_count_tests(n_tests, n_mems, test_filter)); + + if (!test_filter) { + test_uffd_api(false); + test_uffd_api(true); } for (i = 0; i < n_tests; i++) { test = &uffd_tests[i]; if (test_filter && !strstr(test->name, test_filter)) continue; - if (list_only) { - printf("%s\n", test->name); - continue; - } for (j = 0; j < n_mems; j++) { mem_type = &mem_types[j]; @@ -1758,10 +1783,14 @@ int main(int argc, char *argv[]) uffd_test_ops = mem_type->mem_ops; uffd_test_case_ops = test->test_case_ops; + if (!(test->mem_targets & mem_type->mem_flag)) + continue; + + uffd_test_start("%s on %s", test->name, mem_type->name); if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) { - gopts.page_size = default_huge_page_size(); + gopts.page_size = hugepage_size; if (gopts.page_size == 0) { - uffd_test_skip("huge page size is 0, feature missing?"); + uffd_test_skip("not enough HugeTLB pages"); continue; } } else { @@ -1777,10 +1806,6 @@ int main(int argc, char *argv[]) /* Initialize test arguments */ args.mem_type = mem_type; - if (!(test->mem_targets & mem_type->mem_flag)) - continue; - - uffd_test_start("%s on %s", test->name, mem_type->name); if (!uffd_feature_supported(test)) { uffd_test_skip("feature missing"); continue; @@ -1794,10 +1819,7 @@ int main(int argc, char *argv[]) } } - if (!list_only) - uffd_test_report(); - - return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + ksft_finished(); } #else /* __NR_userfaultfd */ @@ -1806,8 +1828,8 @@ int main(int argc, char *argv[]) int main(void) { - printf("Skipping %s (missing __NR_userfaultfd)\n", __file__); - return KSFT_SKIP; + ksft_print_header(); + ksft_exit_skip("missing __NR_userfaultfd definition\n"); } #endif /* __NR_userfaultfd */ diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index 17186d4a4147..c973d6722720 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -8,16 +8,27 @@ #include #include #include "kselftest.h" -#include "thp_settings.h" +#include "hugepage_settings.h" #include "uffd-common.h" static int pagemap_fd; -static size_t pagesize; static int nr_pagesizes = 1; +static unsigned long pagesize; static int nr_thpsizes; static size_t thpsizes[20]; static int nr_hugetlbsizes; -static size_t hugetlbsizes[10]; +static unsigned long hugetlbsizes[10]; + +static void check_uffd_wp_feature_supported(void) +{ + uint64_t features = 0; + + if (uffd_get_features(&features)) + ksft_exit_skip("failed to get available features (%d)\n", errno); + + if (!(features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) + ksft_exit_skip("uffd-wp feature not supported\n"); +} static int detect_thp_sizes(size_t sizes[], int max) { @@ -245,7 +256,7 @@ static void test_one_folio(uffd_global_test_opts_t *gopts, size_t size, bool pri } struct testcase { - size_t *sizes; + unsigned long *sizes; int *nr_sizes; bool private; bool swapout; @@ -336,14 +347,16 @@ int main(int argc, char **argv) struct thp_settings settings; int i, j, plan = 0; + hugepage_save_settings(true, true); + + check_uffd_wp_feature_supported(); + pagesize = getpagesize(); nr_thpsizes = detect_thp_sizes(thpsizes, ARRAY_SIZE(thpsizes)); - nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes, - ARRAY_SIZE(hugetlbsizes)); + nr_hugetlbsizes = hugetlb_setup(1, hugetlbsizes, ARRAY_SIZE(hugetlbsizes)); - /* If THP is supported, save THP settings and initially disable THP. */ + /* If THP is supported, initially disable THP. */ if (nr_thpsizes) { - thp_save_settings(); thp_read_settings(&settings); for (i = 0; i < NR_ORDERS; i++) { settings.hugepages[i].enabled = THP_NEVER; @@ -368,10 +381,6 @@ int main(int argc, char **argv) tc->swapout, tc->hugetlb); } - /* If THP is supported, restore original THP settings. */ - if (nr_thpsizes) - thp_restore_settings(); - i = ksft_get_fail_cnt(); if (i) ksft_exit_fail_msg("%d out of %d tests failed\n", diff --git a/tools/testing/selftests/mm/va_high_addr_switch.c b/tools/testing/selftests/mm/va_high_addr_switch.c index 51401e081b20..e24d7ba00b44 100644 --- a/tools/testing/selftests/mm/va_high_addr_switch.c +++ b/tools/testing/selftests/mm/va_high_addr_switch.c @@ -11,6 +11,7 @@ #include "vm_util.h" #include "kselftest.h" +#include "hugepage_settings.h" /* * The hint addr value is used to allocate addresses @@ -257,40 +258,35 @@ void testcases_init(void) switch_hint = addr_switch_hint; } -static int run_test(struct testcase *test, int count) +static void run_test(struct testcase *test, int count) { void *p; - int i, ret = KSFT_PASS; + int i; for (i = 0; i < count; i++) { struct testcase *t = test + i; p = mmap(t->addr, t->size, PROT_READ | PROT_WRITE, t->flags, -1, 0); - - printf("%s: %p - ", t->msg, p); - if (p == MAP_FAILED) { - printf("FAILED\n"); - ret = KSFT_FAIL; + ksft_perror("MAP_FAILED"); + ksft_test_result_fail("%s\n", t->msg); continue; } if (t->low_addr_required && p >= (void *)(switch_hint)) { - printf("FAILED\n"); - ret = KSFT_FAIL; + ksft_print_msg("%p not below switch hint\n", p); + ksft_test_result_fail("%s\n", t->msg); } else { /* * Do a dereference of the address returned so that we catch * bugs in page fault handling */ memset(p, 0, t->size); - printf("OK\n"); + ksft_test_result_pass("%s\n", t->msg); } if (!t->keep_mapped) munmap(p, t->size); } - - return ret; } #ifdef __aarch64__ @@ -322,19 +318,23 @@ static int supported_arch(void) int main(int argc, char **argv) { - int ret, hugetlb_ret = KSFT_PASS; + bool run_hugetlb = false; + + ksft_print_header(); if (!supported_arch()) - return KSFT_SKIP; + ksft_exit_skip("Architecture not supported\n"); + + if (hugetlb_setup_default(6)) + run_hugetlb = true; testcases_init(); - ret = run_test(testcases, sz_testcases); - if (argc == 2 && !strcmp(argv[1], "--run-hugetlb")) - hugetlb_ret = run_test(hugetlb_testcases, sz_hugetlb_testcases); + ksft_set_plan(sz_testcases + (run_hugetlb ? sz_hugetlb_testcases : 0)); + + run_test(testcases, sz_testcases); + if (run_hugetlb) + run_test(hugetlb_testcases, sz_hugetlb_testcases); - if (ret == KSFT_PASS && hugetlb_ret == KSFT_PASS) - return KSFT_PASS; - else - return KSFT_FAIL; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/va_high_addr_switch.sh b/tools/testing/selftests/mm/va_high_addr_switch.sh index 9492c2d72634..01c15fe3c799 100755 --- a/tools/testing/selftests/mm/va_high_addr_switch.sh +++ b/tools/testing/selftests/mm/va_high_addr_switch.sh @@ -9,7 +9,6 @@ # Kselftest framework requirement - SKIP code is 4. ksft_skip=4 -orig_nr_hugepages=0 skip() { @@ -77,43 +76,5 @@ check_test_requirements() esac } -save_nr_hugepages() -{ - orig_nr_hugepages=$(cat /proc/sys/vm/nr_hugepages) -} - -restore_nr_hugepages() -{ - echo "$orig_nr_hugepages" > /proc/sys/vm/nr_hugepages -} - -setup_nr_hugepages() -{ - local needpgs=$1 - while read -r name size unit; do - if [ "$name" = "HugePages_Free:" ]; then - freepgs="$size" - break - fi - done < /proc/meminfo - if [ "$freepgs" -ge "$needpgs" ]; then - return - fi - local hpgs=$((orig_nr_hugepages + needpgs)) - echo $hpgs > /proc/sys/vm/nr_hugepages - - local nr_hugepgs=$(cat /proc/sys/vm/nr_hugepages) - if [ "$nr_hugepgs" != "$hpgs" ]; then - restore_nr_hugepages - skip "$0: no enough hugepages for testing" - fi -} - check_test_requirements -save_nr_hugepages -# The HugeTLB tests require 6 pages -setup_nr_hugepages 6 -./va_high_addr_switch --run-hugetlb -retcode=$? -restore_nr_hugepages -exit $retcode +./va_high_addr_switch diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index db94564f4431..311fc5b4513e 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -291,53 +290,6 @@ int64_t allocate_transhuge(void *ptr, int pagemap_fd) return -1; } -unsigned long default_huge_page_size(void) -{ - unsigned long hps = 0; - char *line = NULL; - size_t linelen = 0; - FILE *f = fopen("/proc/meminfo", "r"); - - if (!f) - return 0; - while (getline(&line, &linelen, f) > 0) { - if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { - hps <<= 10; - break; - } - } - - free(line); - fclose(f); - return hps; -} - -int detect_hugetlb_page_sizes(size_t sizes[], int max) -{ - DIR *dir = opendir("/sys/kernel/mm/hugepages/"); - int count = 0; - - if (!dir) - return 0; - - while (count < max) { - struct dirent *entry = readdir(dir); - size_t kb; - - if (!entry) - break; - if (entry->d_type != DT_DIR) - continue; - if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) - continue; - sizes[count++] = kb * 1024; - ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", - kb); - } - closedir(dir); - return count; -} - int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags) { size_t count; @@ -396,25 +348,6 @@ int uffd_unregister(int uffd, void *addr, uint64_t len) return ret; } -unsigned long get_free_hugepages(void) -{ - unsigned long fhp = 0; - char *line = NULL; - size_t linelen = 0; - FILE *f = fopen("/proc/meminfo", "r"); - - if (!f) - return fhp; - while (getline(&line, &linelen, f) > 0) { - if (sscanf(line, "HugePages_Free: %lu", &fhp) == 1) - break; - } - - free(line); - fclose(f); - return fhp; -} - static bool check_vmflag(void *addr, const char *flag) { char buffer[MAX_LINE_LENGTH]; @@ -463,7 +396,7 @@ bool softdirty_supported(void) /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */ addr = mmap(0, pagesize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (!addr) + if (addr == MAP_FAILED) ksft_exit_fail_msg("mmap failed\n"); supported = check_vmflag(addr, "sd"); @@ -765,6 +698,27 @@ int unpoison_memory(unsigned long pfn) return ret > 0 ? 0 : -errno; } +int read_file(const char *path, char *buf, size_t buflen) +{ + int fd; + ssize_t numread; + + fd = open(path, O_RDONLY); + if (fd == -1) + return 0; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + close(fd); + return 0; + } + + buf[numread] = '\0'; + close(fd); + + return (unsigned int) numread; +} + void write_file(const char *path, const char *buf, size_t buflen) { int fd, saved_errno; @@ -788,3 +742,49 @@ void write_file(const char *path, const char *buf, size_t buflen) ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n", path, (int)(buflen - 1), buf, buflen - 1, numwritten); } + +unsigned long read_num(const char *path) +{ + char buf[21]; + + if (read_file(path, buf, sizeof(buf)) < 0) + ksft_exit_fail_perror("read_file()"); + + return strtoul(buf, NULL, 10); +} + +void write_num(const char *path, unsigned long num) +{ + char buf[21]; + + sprintf(buf, "%lu", num); + write_file(path, buf, strlen(buf) + 1); +} + +static unsigned long shmall, shmmax; + +void __shm_limits_restore(void) +{ + if (shmmax) + write_num("/proc/sys/kernel/shmmax", shmmax); + if (shmall) + write_num("/proc/sys/kernel/shmall", shmall); +} + +void shm_limits_prepare(unsigned long length) +{ + unsigned long nr = length / psize(); + unsigned long val; + + val = read_num("/proc/sys/kernel/shmmax"); + if (val < length) { + write_num("/proc/sys/kernel/shmmax", length); + shmmax = val; + } + + val = read_num("/proc/sys/kernel/shmall"); + if (val < nr) { + write_num("/proc/sys/kernel/shmall", nr); + shmall = val; + } +} diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index 1a07305ceff4..ea8fc8fdf0eb 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -94,8 +94,6 @@ bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size); bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size); int64_t allocate_transhuge(void *ptr, int pagemap_fd); -unsigned long default_huge_page_size(void); -int detect_hugetlb_page_sizes(size_t sizes[], int max); int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags); int uffd_register(int uffd, void *addr, uint64_t len, @@ -103,7 +101,6 @@ int uffd_register(int uffd, void *addr, uint64_t len, int uffd_unregister(int uffd, void *addr, uint64_t len); int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len, bool miss, bool wp, bool minor, uint64_t *ioctls); -unsigned long get_free_hugepages(void); bool check_vmflag_io(void *addr); bool check_vmflag_pfnmap(void *addr); bool check_vmflag_guard(void *addr); @@ -168,3 +165,15 @@ int unpoison_memory(unsigned long pfn); #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1)) void write_file(const char *path, const char *buf, size_t buflen); +int read_file(const char *path, char *buf, size_t buflen); +unsigned long read_num(const char *path); +void write_num(const char *path, unsigned long num); + +void shm_limits_prepare(unsigned long length); +void __shm_limits_restore(void); + +#define SHM_LIMITS_RESTORE() \ +static void __attribute__((destructor)) shm_limits_restore(void) \ +{ \ + __shm_limits_restore(); \ +} diff --git a/tools/testing/selftests/net/broadcast_ether_dst.sh b/tools/testing/selftests/net/broadcast_ether_dst.sh index 334a7eca8a80..cc571f607429 100755 --- a/tools/testing/selftests/net/broadcast_ether_dst.sh +++ b/tools/testing/selftests/net/broadcast_ether_dst.sh @@ -44,7 +44,7 @@ test_broadcast_ether_dst() { # tcpdump will exit after receiving a single packet # timeout will kill tcpdump if it is still running after 2s timeout 2s ip netns exec "${CLIENT_NS}" \ - tcpdump -i link0 -c 1 -w "${CAPFILE}" icmp &> "${OUTPUT}" & + tcpdump -i link0 -c 1 -w "${CAPFILE}" -Z root icmp &> "${OUTPUT}" & pid=$! slowwait 1 grep -qs "listening" "${OUTPUT}" diff --git a/tools/testing/selftests/net/getsockopt_iter.c b/tools/testing/selftests/net/getsockopt_iter.c index 209569354d0e..fe5a5268bc34 100644 --- a/tools/testing/selftests/net/getsockopt_iter.c +++ b/tools/testing/selftests/net/getsockopt_iter.c @@ -11,6 +11,8 @@ * that always reports the required buffer length back via optlen, * even when the user buffer is too small to receive any group bits. * - vsock: SO_VM_SOCKETS_BUFFER_SIZE covers the u64 path. + * - raw: ICMP_FILTER covers a fixed-size struct payload that clamps + * the length down on a short buffer instead of failing. * * Author: Breno Leitao */ @@ -24,12 +26,20 @@ #include #include #include +#include +#include #include #include "kselftest_harness.h" #ifndef AF_VSOCK #define AF_VSOCK 40 #endif +#ifndef SOL_RAW +#define SOL_RAW 255 +#endif +#ifndef ICMP_FILTER +#define ICMP_FILTER 1 +#endif /* ---------- netlink ---------- */ @@ -297,4 +307,91 @@ TEST_F(vsock, connect_timeout_old_exact) ASSERT_EQ(sizeof(tv), optlen); } +/* ---------- raw (ipv4) ---------- */ + +FIXTURE(raw) +{ + int fd; +}; + +FIXTURE_SETUP(raw) +{ + struct icmp_filter filt = { .data = 0xdeadbeef }; + + self->fd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); + if (self->fd < 0) + SKIP(return, "SOCK_RAW/ICMP socket: %s", strerror(errno)); + + if (setsockopt(self->fd, SOL_RAW, ICMP_FILTER, &filt, sizeof(filt)) < 0) + SKIP(return, "set ICMP_FILTER: %s", strerror(errno)); +} + +FIXTURE_TEARDOWN(raw) +{ + if (self->fd >= 0) + close(self->fd); +} + +TEST_F(raw, icmpfilter_exact) +{ + struct icmp_filter filt = {}; + socklen_t optlen = sizeof(filt); + + ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER, + &filt, &optlen)); + ASSERT_EQ(sizeof(filt), optlen); + ASSERT_EQ(0xdeadbeef, filt.data); +} + +TEST_F(raw, icmpfilter_oversize_clamped) +{ + char buf[16] = {}; + socklen_t optlen = sizeof(buf); + + ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER, + buf, &optlen)); + ASSERT_EQ(sizeof(struct icmp_filter), optlen); +} + +/* Unlike the int/u64 options above, ICMP_FILTER clamps the length down + * to the user buffer instead of returning EINVAL: a short buffer + * succeeds and reports the truncated length back via optlen. + */ +TEST_F(raw, icmpfilter_undersize_clamped) +{ + char buf[2] = {}; + socklen_t optlen = sizeof(buf); + + ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER, + buf, &optlen)); + ASSERT_EQ(sizeof(buf), optlen); +} + +TEST_F(raw, icmpfilter_wrong_proto) +{ + struct icmp_filter filt; + socklen_t optlen = sizeof(filt); + int fd; + + fd = socket(AF_INET, SOCK_RAW, IPPROTO_UDP); + if (fd < 0) + SKIP(return, "SOCK_RAW/UDP socket: %s", strerror(errno)); + + ASSERT_EQ(-1, getsockopt(fd, SOL_RAW, ICMP_FILTER, &filt, &optlen)); + ASSERT_EQ(EOPNOTSUPP, errno); + close(fd); +} + +TEST_F(raw, bad_optname) +{ + socklen_t optlen; + int val; + + optlen = sizeof(val); + + ASSERT_EQ(-1, getsockopt(self->fd, SOL_RAW, 0x7fff, &val, &optlen)); + ASSERT_EQ(ENOPROTOOPT, errno); + ASSERT_EQ(sizeof(val), optlen); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index b3827b43782b..f13672c985bf 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -70,12 +70,28 @@ ksft_exit_status_merge() $ksft_xfail $ksft_pass $ksft_skip $ksft_fail } +timestamp_ms() +{ + local now=$(date -u +%s:%N) + local seconds=${now%:*} + local nanoseconds=${now#*:} + + if [[ $nanoseconds =~ ^[0-9]+$ ]]; then + nanoseconds=${nanoseconds:0:9} + else + nanoseconds=0 + fi + + echo $((seconds * 1000 + 10#$nanoseconds / 1000000)) +} + loopy_wait() { local sleep_cmd=$1; shift local timeout_ms=$1; shift + local current_time - local start_time="$(date -u +%s%3N)" + local start_time=$(timestamp_ms) while true do local out @@ -84,7 +100,7 @@ loopy_wait() return 0 fi - local current_time="$(date -u +%s%3N)" + local current_time=$(timestamp_ms) if ((current_time - start_time > timeout_ms)); then echo -n "$out" return 1 diff --git a/tools/testing/selftests/net/lib/py/utils.py b/tools/testing/selftests/net/lib/py/utils.py index 87eae79d01c1..184bb04343f6 100644 --- a/tools/testing/selftests/net/lib/py/utils.py +++ b/tools/testing/selftests/net/lib/py/utils.py @@ -44,7 +44,7 @@ class cmd: Use bkg() instead to run a command in the background. """ def __init__(self, comm, shell=None, fail=True, expect_fail=False, ns=None, - background=False, host=None, timeout=5, ksft_ready=None, + background=False, host=None, timeout=20, ksft_ready=None, ksft_wait=None): if ns: if hasattr(ns, 'user_ns_path'): @@ -113,7 +113,7 @@ def _process_terminate(self, terminate, timeout): return stdout, stderr - def process(self, terminate=True, fail=None, expect_fail=False, timeout=5): + def process(self, terminate=True, fail=None, expect_fail=False, timeout=20): if fail is None: fail = not terminate diff --git a/tools/testing/selftests/net/netfilter/conntrack_sctp_collision.sh b/tools/testing/selftests/net/netfilter/conntrack_sctp_collision.sh index d860f7d9744b..7261975957ef 100755 --- a/tools/testing/selftests/net/netfilter/conntrack_sctp_collision.sh +++ b/tools/testing/selftests/net/netfilter/conntrack_sctp_collision.sh @@ -2,18 +2,32 @@ # SPDX-License-Identifier: GPL-2.0 # # Testing For SCTP COLLISION SCENARIO as Below: -# +# 1. Stale INIT_ACK capture: # 14:35:47.655279 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [INIT] [init tag: 2017837359] # 14:35:48.353250 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [INIT] [init tag: 1187206187] # 14:35:48.353275 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [INIT ACK] [init tag: 2017837359] # 14:35:48.353283 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [COOKIE ECHO] # 14:35:48.353977 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [COOKIE ACK] # 14:35:48.855335 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [INIT ACK] [init tag: 164579970] +# (Delayed) +# +# 2. Stale INIT capture: +# 14:35:48.353250 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [INIT] [init tag: 1187206187] +# 14:35:48.353275 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [INIT ACK] [init tag: 2017837359] +# 14:35:48.353283 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [COOKIE ECHO] +# 14:35:48.353977 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [COOKIE ACK] +# 14:35:47.655279 IP CLIENT_IP.PORT > SERVER_IP.PORT: sctp (1) [INIT] [init tag: 2017837359] +# (Delayed) +# 14:35:48.855335 IP SERVER_IP.PORT > CLIENT_IP.PORT: sctp (1) [INIT ACK] [init tag: 164579970] # # TOPO: SERVER_NS (link0)<--->(link1) ROUTER_NS (link2)<--->(link3) CLIENT_NS source lib.sh +checktool "nft --version" "run test without nft" +checktool "tc -h" "run test without tc" +checktool "modprobe -q sctp" "load sctp module" + CLIENT_IP="198.51.200.1" CLIENT_PORT=1234 @@ -24,7 +38,8 @@ CLIENT_GW="198.51.200.2" SERVER_GW="198.51.100.2" # setup the topo -setup() { +topo_setup() { + # setup_ns cleans up existing net namespaces first. setup_ns CLIENT_NS SERVER_NS ROUTER_NS ip -n "$SERVER_NS" link add link0 type veth peer name link1 netns "$ROUTER_NS" ip -n "$CLIENT_NS" link add link3 type veth peer name link2 netns "$ROUTER_NS" @@ -38,35 +53,53 @@ setup() { ip -n "$ROUTER_NS" addr add $SERVER_GW/24 dev link1 ip -n "$ROUTER_NS" addr add $CLIENT_GW/24 dev link2 ip net exec "$ROUTER_NS" sysctl -wq net.ipv4.ip_forward=1 + sysctl -wq net.netfilter.nf_log_all_netns=1 ip -n "$CLIENT_NS" link set link3 up ip -n "$CLIENT_NS" addr add $CLIENT_IP/24 dev link3 ip -n "$CLIENT_NS" route add $SERVER_IP dev link3 via $CLIENT_GW +} + +conf_delay() +{ + # simulate the delay on OVS upcall by setting up a delay for INIT_ACK/INIT with + local ns=$1 + local link=$2 + local chunk_type=$3 - # simulate the delay on OVS upcall by setting up a delay for INIT_ACK with - # tc on $SERVER_NS side - tc -n "$SERVER_NS" qdisc add dev link0 root handle 1: htb r2q 64 - tc -n "$SERVER_NS" class add dev link0 parent 1: classid 1:1 htb rate 100mbit - tc -n "$SERVER_NS" filter add dev link0 parent 1: protocol ip u32 match ip protocol 132 \ - 0xff match u8 2 0xff at 32 flowid 1:1 - if ! tc -n "$SERVER_NS" qdisc add dev link0 parent 1:1 handle 10: netem delay 1200ms; then + # use a smaller number for assoc's max_retrans to reproduce the issue + ip net exec "$CLIENT_NS" sysctl -wq net.sctp.association_max_retrans=3 + + tc -n "$ns" qdisc add dev "$link" root handle 1: htb r2q 64 + tc -n "$ns" class add dev "$link" parent 1: classid 1:1 htb rate 100mbit + tc -n "$ns" filter add dev "$link" parent 1: protocol ip \ + u32 match ip protocol 132 0xff match u8 "$chunk_type" 0xff at 32 flowid 1:1 + if ! tc -n "$ns" qdisc add dev "$link" parent 1:1 handle 10: netem delay 1200ms; then echo "SKIP: Cannot add netem qdisc" - exit $ksft_skip + return $ksft_skip fi # simulate the ctstate check on OVS nf_conntrack - ip net exec "$ROUTER_NS" iptables -A FORWARD -m state --state INVALID,UNTRACKED -j DROP - ip net exec "$ROUTER_NS" iptables -A INPUT -p sctp -j DROP - - # use a smaller number for assoc's max_retrans to reproduce the issue - modprobe -q sctp - ip net exec "$CLIENT_NS" sysctl -wq net.sctp.association_max_retrans=3 + ip net exec "$ROUTER_NS" nft -f - <<-EOF + table ip t { + chain forward { + type filter hook forward priority filter; policy accept; + meta l4proto icmp counter accept + ct state new counter accept + ct state established,related counter accept + ct state invalid log flags all counter drop comment \ + "Expect to drop stale INIT/INIT_ACK chunks" + counter + } + } + EOF + return 0 } cleanup() { - ip net exec "$CLIENT_NS" pkill sctp_collision >/dev/null 2>&1 - ip net exec "$SERVER_NS" pkill sctp_collision >/dev/null 2>&1 + # cleanup_all_ns terminates running processes in the namespaces. cleanup_all_ns + sysctl -wq net.netfilter.nf_log_all_netns=0 } do_test() { @@ -81,7 +114,19 @@ do_test() { # run the test case trap cleanup EXIT -setup && \ -echo "Test for SCTP Collision in nf_conntrack:" && \ -do_test && echo "PASS!" -exit $? + +echo "Test for SCTP INIT_ACK Collision in nf_conntrack:" +topo_setup || exit $? +conf_delay $SERVER_NS link0 2 || exit $? + +if ! do_test; then + exit $ksft_fail +fi + +echo "Test for SCTP INIT Collision in nf_conntrack:" +topo_setup || exit $? +conf_delay $CLIENT_NS link3 1 || exit $? + +if ! do_test; then + exit $ksft_fail +fi diff --git a/tools/testing/selftests/net/netfilter/nft_flowtable.sh b/tools/testing/selftests/net/netfilter/nft_flowtable.sh index 7a34ef468975..08ad07500e8a 100755 --- a/tools/testing/selftests/net/netfilter/nft_flowtable.sh +++ b/tools/testing/selftests/net/netfilter/nft_flowtable.sh @@ -592,7 +592,7 @@ ip -net "$nsr1" link set tun0 up ip -net "$nsr1" addr add 192.168.100.1/24 dev tun0 ip netns exec "$nsr1" sysctl net.ipv4.conf.tun0.forwarding=1 > /dev/null -ip -net "$nsr1" link add name tun6 type ip6tnl local fee1:2::1 remote fee1:2::2 +ip -net "$nsr1" link add name tun6 type ip6tnl local fee1:2::1 remote fee1:2::2 encaplimit none ip -net "$nsr1" link set tun6 up ip -net "$nsr1" addr add fee1:3::1/64 dev tun6 nodad @@ -601,7 +601,7 @@ ip -net "$nsr2" link set tun0 up ip -net "$nsr2" addr add 192.168.100.2/24 dev tun0 ip netns exec "$nsr2" sysctl net.ipv4.conf.tun0.forwarding=1 > /dev/null -ip -net "$nsr2" link add name tun6 type ip6tnl local fee1:2::2 remote fee1:2::1 || ret=1 +ip -net "$nsr2" link add name tun6 type ip6tnl local fee1:2::2 remote fee1:2::1 encaplimit none || ret=1 ip -net "$nsr2" link set tun6 up ip -net "$nsr2" addr add fee1:3::2/64 dev tun6 nodad @@ -651,7 +651,7 @@ ip -net "$nsr1" route change default via 192.168.200.2 ip netns exec "$nsr1" sysctl net.ipv4.conf.tun0/10.forwarding=1 > /dev/null ip netns exec "$nsr1" nft -a insert rule inet filter forward 'meta oif tun0.10 accept' -ip -net "$nsr1" link add name tun6.10 type ip6tnl local fee1:4::1 remote fee1:4::2 +ip -net "$nsr1" link add name tun6.10 type ip6tnl local fee1:4::1 remote fee1:4::2 encaplimit none ip -net "$nsr1" link set tun6.10 up ip -net "$nsr1" addr add fee1:5::1/64 dev tun6.10 nodad ip -6 -net "$nsr1" route delete default @@ -670,7 +670,7 @@ ip -net "$nsr2" addr add 192.168.200.2/24 dev tun0.10 ip -net "$nsr2" route change default via 192.168.200.1 ip netns exec "$nsr2" sysctl net.ipv4.conf.tun0/10.forwarding=1 > /dev/null -ip -net "$nsr2" link add name tun6.10 type ip6tnl local fee1:4::2 remote fee1:4::1 || ret=1 +ip -net "$nsr2" link add name tun6.10 type ip6tnl local fee1:4::2 remote fee1:4::1 encaplimit none || ret=1 ip -net "$nsr2" link set tun6.10 up ip -net "$nsr2" addr add fee1:5::2/64 dev tun6.10 nodad ip -6 -net "$nsr2" route delete default diff --git a/tools/testing/selftests/net/netfilter/nft_queue.sh b/tools/testing/selftests/net/netfilter/nft_queue.sh index d80390848e85..7c857a2e0f34 100755 --- a/tools/testing/selftests/net/netfilter/nft_queue.sh +++ b/tools/testing/selftests/net/netfilter/nft_queue.sh @@ -85,11 +85,12 @@ ip -net "$ns3" route add default via 10.0.3.1 ip -net "$ns3" route add default via dead:3::1 load_ruleset() { - local name=$1 - local prio=$2 + local family=$1 + local name=$2 + local prio=$3 ip netns exec "$nsrouter" nft -f /dev/stdin <&2 ip netns exec "$nsrouter" nft list ruleset + echo -n "$TMPFILE0: ";cat "$TMPFILE0" + echo -n "$TMPFILE1: ";cat "$TMPFILE1" exit 1 fi done - echo "PASS: Expected and received $last" + echo "PASS: Expected and received $last ($family)" } listener_ready() @@ -400,6 +404,8 @@ EOF kill "$nfqpid" echo "PASS: icmp+nfqueue via vrf" + ip -net "$ns1" link del tvrf + ip netns exec "$ns1" nft flush ruleset } sctp_listener_ready() @@ -814,12 +820,53 @@ EOF check_tainted "queue program exiting while packets queued" } +test_queue_bridge() +{ + ip -net "$nsrouter" addr flush dev veth0 + ip -net "$nsrouter" addr flush dev veth1 + + ip -net "$nsrouter" link add br0 type bridge + ip -net "$nsrouter" link set veth0 master br0 + ip -net "$nsrouter" link set veth1 master br0 + + ip -net "$nsrouter" link set br0 up + + ip -net "$nsrouter" addr add 10.0.2.1/16 dev br0 + ip -net "$nsrouter" addr add dead:2::1/64 dev br0 nodad + + ip -net "$ns1" addr flush dev eth0 + ip -net "$ns2" addr flush dev eth0 + + ip -net "$ns1" addr add 10.0.1.1/16 dev eth0 + ip -net "$ns1" addr add dead:2::2/64 dev eth0 nodad + + ip -net "$ns2" addr add 10.0.2.99/16 dev eth0 + ip -net "$ns2" addr add dead:2::99/64 dev eth0 nodad + + ip netns exec "$nsrouter" nft flush ruleset + + ip netns exec "$nsrouter" sysctl net.ipv6.conf.all.forwarding=0 > /dev/null + ip netns exec "$nsrouter" sysctl net.ipv4.conf.veth0.forwarding=0 > /dev/null + ip netns exec "$nsrouter" sysctl net.ipv4.conf.veth1.forwarding=0 > /dev/null + + if ! test_ping;then + echo "FAIL: netns bridge connectivity" 1>&2 + exit $ret + fi + + load_ruleset "bridge" "filter" 10 + test_queue 10 "bridge" + + load_ruleset "bridge" "filter2" 20 + test_queue 20 "bridge" +} + ip netns exec "$nsrouter" sysctl net.ipv6.conf.all.forwarding=1 > /dev/null ip netns exec "$nsrouter" sysctl net.ipv4.conf.veth0.forwarding=1 > /dev/null ip netns exec "$nsrouter" sysctl net.ipv4.conf.veth1.forwarding=1 > /dev/null ip netns exec "$nsrouter" sysctl net.ipv4.conf.veth2.forwarding=1 > /dev/null -load_ruleset "filter" 0 +load_ruleset "inet" "filter" 0 if test_ping; then # queue bypass works (rules were skipped, no listener) @@ -842,11 +889,11 @@ load_counter_ruleset 10 # 1x icmp prerouting,forward,postrouting -> 3 queue events (6 incl. reply). # 1x icmp prerouting,input,output postrouting -> 4 queue events incl. reply. # so we expect that userspace program receives 10 packets. -test_queue 10 +test_queue 10 "inet" # same. We queue to a second program as well. -load_ruleset "filter2" 20 -test_queue 20 +load_ruleset "inet" "filter2" 20 +test_queue 20 "inet" ip netns exec "$ns1" nft flush ruleset test_tcp_forward @@ -863,4 +910,7 @@ test_queue_stress test_icmp_vrf test_queue_removal +# turns router into a bridge +test_queue_bridge + exit $ret diff --git a/tools/testing/selftests/net/rtnetlink.py b/tools/testing/selftests/net/rtnetlink.py index 3622413d793d..0c67c7c00d84 100755 --- a/tools/testing/selftests/net/rtnetlink.py +++ b/tools/testing/selftests/net/rtnetlink.py @@ -2,27 +2,106 @@ # SPDX-License-Identifier: GPL-2.0 import socket +import struct import time -from lib.py import bkg, ip, ksft_exit, ksft_run, ksft_ge, ksft_true, KsftSkipEx +from lib.py import bkg, ip, ksft_exit, ksft_run, ksft_eq, ksft_ge, ksft_true, KsftSkipEx from lib.py import CmdExitFailure, NetNS, NetNSEnter, RtnlAddrFamily IPV4_ALL_HOSTS_MULTICAST = b'\xe0\x00\x00\x01' +IPV4_TEST_MULTICAST = b'\xef\x01\x01\x01' +IPV6_TEST_MULTICAST = bytes.fromhex('ff020000000000000000000000000123') + + +def _users_for(rtnl: RtnlAddrFamily, family: int, grp: bytes, ifindex: int): + """Return mc-users for grp on ifindex, or 0 if absent.""" + + addrs = rtnl.getmulticast({"ifa-family": family}, dump=True) + matches = [addr for addr in addrs + if addr['multicast'] == grp and addr['ifa-index'] == ifindex] + if not matches: + return 0 + if 'mc-users' not in matches[0]: + return None + + return matches[0]['mc-users'] + def dump_mcaddr_check() -> None: """ - Verify that at least one interface has the IPv4 all-hosts multicast address. - At least the loopback interface should have this address. + Verify IPv4 multicast addresses and their user counts in RTM_GETMULTICAST. + """ + + with NetNS() as ns: + with NetNSEnter(str(ns)): + ip("link set lo up") + rtnl = RtnlAddrFamily() + lo_idx = socket.if_nametoindex('lo') + addresses = rtnl.getmulticast({"ifa-family": socket.AF_INET}, dump=True) + + all_host_multicasts = [ + addr for addr in addresses + if addr['multicast'] == IPV4_ALL_HOSTS_MULTICAST + ] + + ksft_ge(len(all_host_multicasts), 1, + "No interface found with the IPv4 all-hosts multicast address") + + mreq = IPV4_TEST_MULTICAST + socket.inet_aton('127.0.0.1') + before = _users_for(rtnl, socket.AF_INET, IPV4_TEST_MULTICAST, lo_idx) + if before is None: + raise KsftSkipEx("kernel does not expose IFA_MC_USERS") + + s1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s1.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + s2.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + + after_join = _users_for(rtnl, socket.AF_INET, + IPV4_TEST_MULTICAST, lo_idx) + if after_join is None: + raise KsftSkipEx("kernel does not expose IFA_MC_USERS") + ksft_eq(after_join - before, 2, + f"users delta != 2 after two joins " + f"(before={before}, after={after_join})") + finally: + s1.close() + s2.close() + + +def dump_mcaddr6_check() -> None: + """ + Verify IPv6 multicast addresses and their user counts in RTM_GETMULTICAST. """ - rtnl = RtnlAddrFamily() - addresses = rtnl.getmulticast({"ifa-family": socket.AF_INET}, dump=True) + with NetNS() as ns: + with NetNSEnter(str(ns)): + ip("link set lo up") + rtnl = RtnlAddrFamily() + lo_idx = socket.if_nametoindex('lo') + before = _users_for(rtnl, socket.AF_INET6, + IPV6_TEST_MULTICAST, lo_idx) + if before is None: + raise KsftSkipEx("kernel does not expose IFA_MC_USERS for IPv6") + + mreq = IPV6_TEST_MULTICAST + struct.pack('=I', lo_idx) + s1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s2 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + try: + s1.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) + s2.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) - all_host_multicasts = [ - addr for addr in addresses if addr['multicast'] == IPV4_ALL_HOSTS_MULTICAST - ] + after_join = _users_for(rtnl, socket.AF_INET6, + IPV6_TEST_MULTICAST, lo_idx) + if after_join is None: + raise KsftSkipEx("kernel does not expose IFA_MC_USERS for IPv6") + ksft_eq(after_join - before, 2, + f"IPv6 users delta != 2 after two joins " + f"(before={before}, after={after_join})") + finally: + s1.close() + s2.close() - ksft_ge(len(all_host_multicasts), 1, - "No interface found with the IPv4 all-hosts multicast address") def ipv4_devconf_notify() -> None: """ @@ -56,7 +135,7 @@ def ipv4_devconf_notify() -> None: f"No 'forwarding on' notificiation found for interface {ifname}") def main() -> None: - ksft_run([dump_mcaddr_check, ipv4_devconf_notify]) + ksft_run([dump_mcaddr_check, dump_mcaddr6_check, ipv4_devconf_notify]) ksft_exit() if __name__ == "__main__": diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index 9b9a3cb2700d..cbdd3ea28b99 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -997,6 +997,8 @@ TEST_F(tls, splice_short) char sendbuf[0x100]; char sendchar = 'S'; int pipefds[2]; + int pipe_sz; + int ret; int i; sendchar_iov.iov_base = &sendchar; @@ -1005,7 +1007,11 @@ TEST_F(tls, splice_short) memset(sendbuf, 's', sizeof(sendbuf)); ASSERT_GE(pipe2(pipefds, O_NONBLOCK), 0); - ASSERT_GE(fcntl(pipefds[0], F_SETPIPE_SZ, (MAX_FRAGS + 1) * 0x1000), 0); + pipe_sz = (MAX_FRAGS + 1) * getpagesize(); + ret = fcntl(pipefds[0], F_SETPIPE_SZ, pipe_sz); + if (ret < 0 && errno == EPERM) + SKIP(return, "insufficient pipe capacity"); + ASSERT_GE(ret, pipe_sz); for (i = 0; i < MAX_FRAGS; i++) ASSERT_GE(vmsplice(pipefds[1], &sendchar_iov, 1, 0), 0); diff --git a/tools/testing/selftests/net/vlan_bridge_binding.sh b/tools/testing/selftests/net/vlan_bridge_binding.sh index e8c02c64e03a..d04caa14202d 100755 --- a/tools/testing/selftests/net/vlan_bridge_binding.sh +++ b/tools/testing/selftests/net/vlan_bridge_binding.sh @@ -64,7 +64,7 @@ check_operstate() local expect=$1; shift local operstate - operstate=$(busywait 1000 \ + operstate=$(busywait 2000 \ operstate_is "$dev" "$expect") check_err $? "Got operstate of $operstate, expected $expect" } diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index c1c1ce43a047..996e8d13508e 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -854,6 +854,58 @@ static int test_dirent(void) return 0; } +int test_getcwd(void) +{ + char cwd_syscall[PATH_MAX]; + char cwd_proc[PATH_MAX]; + ssize_t len; + + /* Read where the link /proc/self/cwd points */ + len = readlink("/proc/self/cwd", cwd_proc, sizeof(cwd_proc) - 1); + if (len <= 0) + return __LINE__; + + /* Terminate the string from readlink() */ + cwd_proc[len] = '\0'; + + /* Get the cwd via syscall */ + if (getcwd(cwd_syscall, sizeof(cwd_syscall)) == NULL) + return __LINE__; + + /* Fail if they aren't the same */ + if (strcmp(cwd_proc, cwd_syscall) != 0) + return __LINE__; + + /* Try getcwd() with NULL for the buffer, + * should return NULL and an error in errno. + * Other libc's allow this by allocating a buffer + * internally. + */ + if (is_nolibc) { + errno = 0; + if (getcwd(NULL, 0) != NULL || !errno) + return __LINE__; + } + + /* Try getcwd() with a buffer but make the size 0, + * should return NULL and an error in errno. + */ + errno = 0; + if (getcwd(cwd_syscall, 0) != NULL || !errno) + return __LINE__; + + /* Try getcwd() with a buffer but make the size 1, + * should return NULL and an error in errno because + * the string written to the buffer is terminated + * so you need at least 2 bytes even for "/". + */ + errno = 0; + if (getcwd(cwd_syscall, 1) != NULL || !errno) + return __LINE__; + + return 0; +} + int test_getrandom(void) { uint64_t rng = 0; @@ -1555,6 +1607,7 @@ int run_syscall(int min, int max) CASE_TEST(clock_getres); EXPECT_SYSZR(1, clock_getres(CLOCK_MONOTONIC, &ts)); break; CASE_TEST(clock_gettime); EXPECT_SYSZR(1, clock_gettime(CLOCK_MONOTONIC, &ts)); break; CASE_TEST(clock_settime); EXPECT_SYSER(1, clock_settime(CLOCK_MONOTONIC, &ts), -1, EINVAL); break; + CASE_TEST(getcwd); EXPECT_SYSZR(proc, test_getcwd()); break; CASE_TEST(getpid); EXPECT_SYSNE(1, getpid(), -1); break; CASE_TEST(getppid); EXPECT_SYSNE(1, getppid(), -1); break; CASE_TEST(gettid); EXPECT_SYSNE(has_gettid, gettid(), -1); break; diff --git a/tools/testing/selftests/perf_events/watermark_signal.c b/tools/testing/selftests/perf_events/watermark_signal.c index 0f64b9b17081..a84709cabd8b 100644 --- a/tools/testing/selftests/perf_events/watermark_signal.c +++ b/tools/testing/selftests/perf_events/watermark_signal.c @@ -102,7 +102,7 @@ TEST(watermark_signal) } p = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (p == NULL) { + if (p == MAP_FAILED) { perror("mmap"); goto cleanup; } diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c index 25b9a2933c45..6f7c10fe97b3 100644 --- a/tools/testing/selftests/proc/proc-pidns.c +++ b/tools/testing/selftests/proc/proc-pidns.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h index d3133587d996..5a29d6bec51f 100644 --- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -8,13 +8,11 @@ #ifndef _RSEQ_X86_THREAD_POINTER #define _RSEQ_X86_THREAD_POINTER -#include - #ifdef __cplusplus extern "C" { #endif -#if __GNUC_PREREQ (11, 1) +#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1) static inline void *rseq_thread_pointer(void) { return __builtin_thread_pointer(); diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ct.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ct.json index 33bb8f3ff8ed..da65f838bd52 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/ct.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ct.json @@ -664,5 +664,43 @@ "teardown": [ "$TC qdisc del dev $DEV1 ingress_block 21 clsact" ] + }, + { + "id": "9c2a", + "name": "Act_ct preserves skb cb across defrag before prio dequeue", + "category": [ + "actions", + "ct", + "scapy" + ], + "plugins": { + "requires": [ + "nsPlugin", + "scapyPlugin" + ] + }, + "setup": [ + "$TC qdisc add dev $DUMMY root handle 1: prio", + "$TC qdisc add dev $DUMMY clsact", + "$TC qdisc add dev $DEV1 clsact", + "$TC filter add dev $DEV1 ingress protocol ip prio 1 matchall action mirred egress redirect dev $DUMMY" + ], + "cmdUnderTest": "$TC filter add dev $DUMMY egress protocol ip prio 1 matchall action ct zone 1 pipe", + "scapy": [ + { + "iface": "$DEV0", + "count": 1, + "packet": "[Ether()/frag for frag in fragment(IP(src='10.0.0.10', dst='10.0.0.1', id=1)/UDP(sport=12345, dport=9)/Raw(b'A' * 4000), fragsize=1400)]" + } + ], + "expExitCode": "0", + "verifyCmd": "$TC -s qdisc show dev $DUMMY | grep -A 1 '^qdisc prio 1:'", + "matchPattern": "Sent [1-9][0-9]* bytes [1-9][0-9]* pkt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 clsact", + "$TC qdisc del dev $DUMMY clsact", + "$TC qdisc del dev $DUMMY root handle 1:" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json index a1f97a4b606e..0cf12c50fb74 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json +++ b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json @@ -1540,5 +1540,169 @@ "$TC qdisc del dev $DUMMY root", "$IP addr del 10.10.10.10/24 dev $DUMMY || true" ] + }, + { + "id": "fb6c", + "name": "Force multiq to dequeue from its child's gso_skb with qfq leaf", + "category": [ + "qdisc", + "tbf", + "multiq", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device", + "$IP link set dev $ETH up || true", + "$IP l set addr 01:02:03:04:05:06 dev $ETH || true", + "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true", + "$IP addr add 10.10.11.10/24 dev $ETH || true", + "$TC qdisc add dev $ETH root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b", + "$TC qdisc add dev $ETH parent 1: handle 2: multiq", + "$TC qdisc add dev $ETH parent 2:1 handle 3: qfq", + "$TC class add dev $ETH classid 3:1 parent 3: qfq maxpkt 512 weight 1", + "$TC filter add dev $ETH parent 2: protocol all prio 1 matchall action skbedit queue_mapping 0", + "$TC filter add dev $ETH parent 3: protocol all prio 1 matchall classid 3:1 action ok" + ], + "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true", + "expExitCode": "0", + "verifyCmd": "$TC -s -j qdisc ls dev $ETH parent 1:", + "matchJSON": [ + { + "kind": "multiq", + "handle": "2:", + "bytes": 98, + "packets": 1, + "backlog": 0, + "qlen": 0 + } + ], + "teardown": [ + "$TC qdisc del dev $ETH handle 1: root", + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "1922", + "name": "Force multiq to dequeue from its child's gso_skb with dualpi2 leaf", + "category": [ + "qdisc", + "tbf", + "multiq", + "dualpi2" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device", + "$IP link set dev $ETH up || true", + "$IP l set addr 01:02:03:04:05:06 dev $ETH || true", + "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true", + "$IP addr add 10.10.11.10/24 dev $ETH || true", + "$TC qdisc add dev $ETH root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b", + "$TC qdisc add dev $ETH parent 1: handle 2: multiq", + "$TC qdisc add dev $ETH parent 2:1 handle 3: dualpi2", + "$TC filter add dev $ETH parent 2: protocol ip prio 1 u32 match ip dst 10.10.11.1 action skbedit queue_mapping 0", + "$TC filter add dev $ETH parent 3: protocol ip prio 1 u32 match ip dst 10.10.11.1 classid 3:1 action ok" + ], + "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true", + "expExitCode": "0", + "verifyCmd": "$TC -j -s qdisc ls dev $ETH handle 3:", + "matchJSON": [ + { + "kind": "dualpi2", + "handle": "3:", + "bytes": 98, + "packets": 1, + "backlog": 0, + "qlen": 0 + } + ], + "teardown": [ + "$TC qdisc del dev $ETH handle 1: root", + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "476f", + "name": "Force taprio to dequeue from its child's gso_skb with qfq leaf", + "category": [ + "qdisc", + "tbf", + "multiq", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device", + "$IP link set dev $ETH up || true", + "$IP l set addr 01:02:03:04:05:06 dev $ETH || true", + "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true", + "$TC qdisc add dev $ETH root handle 1: taprio num_tc 2 map 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 queues 1@0 1@1 base-time 9000000000000000000 sched-entry S 03 200000 flags 0x0 clockid CLOCK_TAI", + "$TC qdisc add dev $ETH parent 1:1 handle 3: qfq", + "$TC class add dev $ETH classid 3:1 parent 3: qfq maxpkt 512 weight 1", + "$TC filter add dev $ETH parent 3: protocol all prio 1 matchall classid 3:1 action ok" + ], + "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true", + "expExitCode": "0", + "verifyCmd": "$TC -s -j qdisc ls dev $ETH", + "matchJSON": [ + { + "kind": "taprio", + "handle": "1:", + "bytes": 98, + "packets": 1, + "backlog": 0, + "qlen": 0 + } + ], + "teardown": [ + "$TC qdisc del dev $ETH handle 1: root", + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "0235", + "name": "Force taprio to dequeue from its child's gso_skb with dualpi2 leaf", + "category": [ + "qdisc", + "tbf", + "taprio", + "dualpi2" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device", + "$IP link set dev $ETH up || true", + "$IP l set addr 01:02:03:04:05:06 dev $ETH || true", + "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true", + "$TC qdisc add dev $ETH root handle 1: taprio num_tc 2 map 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 queues 1@0 1@1 base-time 9000000000000000000 sched-entry S 03 200000 flags 0x0 clockid CLOCK_TAI", + "$TC qdisc replace dev $ETH parent 1:1 handle 3: dualpi2", + "$TC filter add dev $ETH parent 3: protocol ip prio 1 u32 match ip dst 10.10.11.1 classid 3:1 action ok" + ], + "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true", + "expExitCode": "0", + "verifyCmd": "$TC -j -s qdisc ls dev $ETH handle 3:", + "matchJSON": [ + { + "kind": "dualpi2", + "handle": "3:", + "bytes": 98, + "packets": 1, + "backlog": 0, + "qlen": 0 + } + ], + "teardown": [ + "$TC qdisc del dev $ETH handle 1: root", + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dualpi2.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dualpi2.json index cd1f2ee8f354..ed6a900bb568 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dualpi2.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dualpi2.json @@ -250,5 +250,49 @@ "teardown": [ "$TC qdisc del dev $DUMMY handle 1: root" ] + }, + { + "id": "891f", + "name": "Verify DualPI2 GSO backlog accounting with QFQ parent", + "category": [ + "qdisc", + "dualpi2", + "qfq", + "gso" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link set dev $DUMMY up || true", + "$IP addr add 10.10.10.10/24 dev $DUMMY || true", + "$TC qdisc add dev $DUMMY root handle 1: qfq", + "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 1 maxpkt 4096", + "$TC qdisc add dev $DUMMY parent 1:1 handle 2: dualpi2", + "$TC filter add dev $DUMMY parent 1: matchall classid 1:1" + ], + "cmdUnderTest": "./tdc_gso.py 10.10.10.10 10.10.10.1 9000 1200 2400", + "expExitCode": "0", + "verifyCmd": "$TC -j -s qdisc ls dev $DUMMY", + "matchJSON": [ + { + "kind": "qfq", + "handle": "1:", + "packets": 2, + "backlog": 0, + "qlen": 0 + }, + { + "kind": "dualpi2", + "handle": "2:", + "packets": 2, + "backlog": 0, + "qlen": 0 + } + ], + "teardown": [ + "$TC qdisc del dev $DUMMY root", + "$IP addr del 10.10.10.10/24 dev $DUMMY || true" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tdc_gso.py b/tools/testing/selftests/tc-testing/tdc_gso.py new file mode 100755 index 000000000000..b66528ea4b68 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tdc_gso.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +""" +tdc_gso.py - send a UDP GSO datagram + +Copyright (C) 2026 Xingquan Liu +""" + +import argparse +import socket +import struct +import sys + +UDP_MAX_SEGMENTS = 1 << 7 + + +parser = argparse.ArgumentParser(description="UDP GSO datagram sender") +parser.add_argument("src", help="source IPv4 address") +parser.add_argument("dst", help="destination IPv4 address") +parser.add_argument("port", type=int, help="destination UDP port") +parser.add_argument("gso_size", type=int, help="UDP GSO segment payload size") +parser.add_argument("payload_len", type=int, help="total UDP payload length") +args = parser.parse_args() + +if args.gso_size <= 0 or args.gso_size > 0xFFFF: + parser.error("gso_size must fit in an unsigned 16-bit integer") +if args.payload_len <= args.gso_size: + parser.error("payload_len must be larger than gso_size") +if args.payload_len > args.gso_size * UDP_MAX_SEGMENTS: + parser.error("payload_len exceeds UDP_MAX_SEGMENTS") + +SOL_UDP = getattr(socket, "SOL_UDP", socket.IPPROTO_UDP) +UDP_SEGMENT = getattr(socket, "UDP_SEGMENT", 103) + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.bind((args.src, 0)) + +payload = b"b" * args.payload_len +cmsg = [(SOL_UDP, UDP_SEGMENT, struct.pack("=H", args.gso_size))] + +sent = sock.sendmsg([payload], cmsg, 0, (args.dst, args.port)) +sys.exit(sent != len(payload)) diff --git a/tools/testing/selftests/uevent/uevent_filtering.c b/tools/testing/selftests/uevent/uevent_filtering.c index 974b076f9235..33a09f66d7e2 100644 --- a/tools/testing/selftests/uevent/uevent_filtering.c +++ b/tools/testing/selftests/uevent/uevent_filtering.c @@ -22,7 +22,7 @@ #include "kselftest_harness.h" #define __DEV_FULL "/sys/devices/virtual/mem/full/uevent" -#define __UEVENT_BUFFER_SIZE (2048 * 2) +#define __UEVENT_BUFFER_SIZE (1024 * 1024) #define __UEVENT_HEADER "add@/devices/virtual/mem/full" #define __UEVENT_HEADER_LEN sizeof("add@/devices/virtual/mem/full") #define __UEVENT_LISTEN_ALL -1 diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile index e6e8cb52ab03..2c32c48db509 100644 --- a/tools/testing/selftests/vfio/Makefile +++ b/tools/testing/selftests/vfio/Makefile @@ -29,8 +29,6 @@ CFLAGS += $(EXTRA_CFLAGS) LDFLAGS += -pthread -LDLIBS += -luuid - $(TEST_GEN_PROGS): $(OUTPUT)/%: $(OUTPUT)/%.o $(LIBVFIO_O) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $< $(LIBVFIO_O) $(LDLIBS) -o $@ diff --git a/tools/testing/selftests/vfio/lib/libvfio.mk b/tools/testing/selftests/vfio/lib/libvfio.mk index 2b8d73b7d329..67942b085068 100644 --- a/tools/testing/selftests/vfio/lib/libvfio.mk +++ b/tools/testing/selftests/vfio/lib/libvfio.mk @@ -26,6 +26,8 @@ $(LIBVFIO_O_DIRS): CFLAGS += -I$(LIBVFIO_SRCDIR)/include +LDLIBS += -luuid + $(LIBVFIO_O): $(LIBVFIO_OUTPUT)/%.o : $(LIBVFIO_SRCDIR)/%.c | $(LIBVFIO_O_DIRS) $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@ diff --git a/tools/testing/selftests/x86/test_shadow_stack.c b/tools/testing/selftests/x86/test_shadow_stack.c index 21af54d5f4ea..3d6ca33edba4 100644 --- a/tools/testing/selftests/x86/test_shadow_stack.c +++ b/tools/testing/selftests/x86/test_shadow_stack.c @@ -873,6 +873,86 @@ static int test_uretprobe(void) return err; } +/* Keep the CALL first so the function address is exactly the probed CALL. */ +extern void uprobe_call_trigger(void); +asm (".pushsection .text\n" + ".global uprobe_call_target\n" + ".type uprobe_call_target, @function\n" + "uprobe_call_target:\n" + " ret\n" + ".size uprobe_call_target, .-uprobe_call_target\n" + + ".global uprobe_call_trigger\n" + ".type uprobe_call_trigger, @function\n" + "uprobe_call_trigger:\n" + " call uprobe_call_target\n" + " ret\n" + ".size uprobe_call_trigger, .-uprobe_call_trigger\n" + ".popsection\n" +); + +/* If CALL emulation misses the shadow stack update, this exits via SIGSEGV. */ +static int test_uprobe_call(void) +{ + const size_t attr_sz = sizeof(struct perf_event_attr); + const char *file = "/proc/self/exe"; + int fd = -1, type, err = 1; + struct perf_event_attr attr; + struct sigaction sa = {}; + ssize_t offset; + + type = determine_uprobe_perf_type(); + if (type < 0) { + if (type == -ENOENT) + printf("[SKIP]\tUprobe on CALL test, uprobes are not available\n"); + return 0; + } + + offset = get_uprobe_offset(uprobe_call_trigger); + if (offset < 0) + return 1; + + sa.sa_sigaction = segv_gp_handler; + sa.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &sa, NULL)) + return 1; + + /* Setup entry uprobe through perf event interface. */ + memset(&attr, 0, attr_sz); + attr.size = attr_sz; + attr.type = type; + attr.config = 0; + attr.config1 = (__u64)(unsigned long)file; + attr.config2 = offset; + + fd = syscall(__NR_perf_event_open, &attr, 0 /* pid */, -1 /* cpu */, + -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); + if (fd < 0) + goto out; + + if (sigsetjmp(jmp_buffer, 1)) + goto out; + + if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) + goto out; + + /* + * This either segfaults and goes through sigsetjmp above + * or succeeds and we're good. + */ + uprobe_call_trigger(); + + printf("[OK]\tUprobe on CALL test\n"); + err = 0; + +out: + ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK); + signal(SIGSEGV, SIG_DFL); + if (fd >= 0) + close(fd); + return err; +} + void segv_handler_ptrace(int signum, siginfo_t *si, void *uc) { /* The SSP adjustment caused a segfault. */ @@ -1071,6 +1151,12 @@ int main(int argc, char *argv[]) goto out; } + if (test_uprobe_call()) { + ret = 1; + printf("[FAIL]\tuprobe on CALL test\n"); + goto out; + } + return ret; out: diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index bf26b3f48d3a..cf73bcd9bb9d 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -243,7 +243,7 @@ enum { #define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE) #define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE) #define VM_STACK INIT_VM_FLAG(STACK) -#ifdef CONFIG_STACK_GROWS_UP +#ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) #else #define VM_STACK_EARLY VM_NONE diff --git a/tools/tracing/rtla/.gitignore b/tools/tracing/rtla/.gitignore index c7b4bf1c8ba9..2a1c99ddd1bf 100644 --- a/tools/tracing/rtla/.gitignore +++ b/tools/tracing/rtla/.gitignore @@ -1,14 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only -rtla -rtla-static -unit_tests +/rtla +/rtla-static +/unit_tests fixdep feature FEATURE-DUMP *.skel.h -custom_filename.txt -osnoise_irq_noise_hist.txt -osnoise_trace.txt -timerlat_trace.txt -libsubcmd/ -lib/ +/libsubcmd/ diff --git a/tools/tracing/rtla/tests/scripts/lib/get_workload_pids.sh b/tools/tracing/rtla/tests/scripts/lib/get_workload_pids.sh index 8aff98cd2c1f..d10a4e3b321d 100644 --- a/tools/tracing/rtla/tests/scripts/lib/get_workload_pids.sh +++ b/tools/tracing/rtla/tests/scripts/lib/get_workload_pids.sh @@ -5,7 +5,7 @@ get_workload_pids() { local rtla_pid=$(ps -o ppid= $shell_pid) # kernel threads - pgrep -P $(pgrep ^kthreadd$) -f '^(osnoise|timerlat)/[0-9]+$' + pgrep -P $(pgrep ^kthreadd$) -f '^\[?(osnoise|timerlat)/[0-9]+\]?$' # user threads pgrep -P $rtla_pid | grep -v "^$shell_pid$" } diff --git a/tools/virtio/asm/percpu_types.h b/tools/virtio/asm/percpu_types.h new file mode 100644 index 000000000000..4eb53d93c099 --- /dev/null +++ b/tools/virtio/asm/percpu_types.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PERCPU_TYPES_H +#define _ASM_PERCPU_TYPES_H + +#define __percpu_qual + +#endif /* _ASM_PERCPU_TYPES_H */ diff --git a/tools/virtio/linux/completion.h b/tools/virtio/linux/completion.h new file mode 100644 index 000000000000..5e54b679721b --- /dev/null +++ b/tools/virtio/linux/completion.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_COMPLETION_H +#define _LINUX_COMPLETION_H + +struct completion { + unsigned int done; +}; + +#endif /* _LINUX_COMPLETION_H */ diff --git a/tools/virtio/linux/device.h b/tools/virtio/linux/device.h index 075c2140d975..abf100cb0023 100644 --- a/tools/virtio/linux/device.h +++ b/tools/virtio/linux/device.h @@ -1,4 +1,5 @@ #ifndef LINUX_DEVICE_H +#define LINUX_DEVICE_H struct device { void *parent; diff --git a/tools/virtio/linux/dma-mapping.h b/tools/virtio/linux/dma-mapping.h index 8d1a16cb20db..b9fc5e8338e3 100644 --- a/tools/virtio/linux/dma-mapping.h +++ b/tools/virtio/linux/dma-mapping.h @@ -61,5 +61,6 @@ enum dma_data_direction { #define DMA_MAPPING_ERROR (~(dma_addr_t)0) #define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11) +#define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES 0 #endif diff --git a/tools/virtio/linux/mod_devicetable.h b/tools/virtio/linux/mod_devicetable.h new file mode 100644 index 000000000000..3ba594b8229d --- /dev/null +++ b/tools/virtio/linux/mod_devicetable.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_MOD_DEVICETABLE_H +#define _LINUX_MOD_DEVICETABLE_H + +#include + +struct virtio_device_id { + __u32 device; + __u32 vendor; +}; + +#define VIRTIO_DEV_ANY_ID 0xffffffff + +#endif /* _LINUX_MOD_DEVICETABLE_H */ diff --git a/tools/virtio/linux/virtio_features.h b/tools/virtio/linux/virtio_features.h new file mode 100644 index 000000000000..04cbb9622ec7 --- /dev/null +++ b/tools/virtio/linux/virtio_features.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VIRTIO_FEATURES_H +#define _LINUX_VIRTIO_FEATURES_H + +#include +#include +#include + +#define VIRTIO_FEATURES_U64S 2 +#define VIRTIO_FEATURES_BITS (VIRTIO_FEATURES_U64S * 64) + +#define VIRTIO_BIT(b) (1ULL << ((b) & 0x3f)) +#define VIRTIO_U64(b) ((b) >> 6) + +#define VIRTIO_DECLARE_FEATURES(name) \ + union { \ + u64 name; \ + u64 name##_array[VIRTIO_FEATURES_U64S];\ + } + +static inline bool virtio_features_chk_bit(unsigned int bit) +{ + return bit < VIRTIO_FEATURES_BITS; +} + +static inline bool virtio_features_test_bit(const u64 *features, + unsigned int bit) +{ + return virtio_features_chk_bit(bit) && + !!(features[VIRTIO_U64(bit)] & VIRTIO_BIT(bit)); +} + +static inline void virtio_features_set_bit(u64 *features, unsigned int bit) +{ + if (virtio_features_chk_bit(bit)) + features[VIRTIO_U64(bit)] |= VIRTIO_BIT(bit); +} + +static inline void virtio_features_clear_bit(u64 *features, unsigned int bit) +{ + if (virtio_features_chk_bit(bit)) + features[VIRTIO_U64(bit)] &= ~VIRTIO_BIT(bit); +} + +static inline void virtio_features_zero(u64 *features) +{ + memset(features, 0, sizeof(features[0]) * VIRTIO_FEATURES_U64S); +} + +static inline void virtio_features_from_u64(u64 *features, u64 from) +{ + virtio_features_zero(features); + features[0] = from; +} + +static inline bool virtio_features_equal(const u64 *f1, const u64 *f2) +{ + int i; + + for (i = 0; i < VIRTIO_FEATURES_U64S; ++i) + if (f1[i] != f2[i]) + return false; + return true; +} + +static inline void virtio_features_copy(u64 *to, const u64 *from) +{ + memcpy(to, from, sizeof(to[0]) * VIRTIO_FEATURES_U64S); +} + +static inline void virtio_features_andnot(u64 *to, const u64 *f1, const u64 *f2) +{ + int i; + + for (i = 0; i < VIRTIO_FEATURES_U64S; i++) + to[i] = f1[i] & ~f2[i]; +} + +#endif /* _LINUX_VIRTIO_FEATURES_H */ diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index ce4161f52f2f..a0c72237531f 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -46,6 +46,11 @@ import sys +import argparse +parser = argparse.ArgumentParser(description=desc, + formatter_class=argparse.RawTextHelpFormatter) +args = parser.parse_args() + import drgn from drgn.helpers.linux.list import list_for_each_entry,list_empty from drgn.helpers.linux.percpu import per_cpu_ptr @@ -53,11 +58,6 @@ from drgn.helpers.linux.nodemask import for_each_node from drgn.helpers.linux.idr import idr_for_each -import argparse -parser = argparse.ArgumentParser(description=desc, - formatter_class=argparse.RawTextHelpFormatter) -args = parser.parse_args() - def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py index 9e964c5be40c..7f47fa398e3c 100644 --- a/tools/workqueue/wq_monitor.py +++ b/tools/workqueue/wq_monitor.py @@ -37,9 +37,6 @@ import time import json -import drgn -from drgn.helpers.linux.list import list_for_each_entry - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -51,6 +48,9 @@ help='Output in json') args = parser.parse_args() +import drgn +from drgn.helpers.linux.list import list_for_each_entry + workqueues = prog['workqueues'] WQ_UNBOUND = prog['WQ_UNBOUND'] diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 0e8b8a2c5b79..93ad2ebc963f 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -779,21 +780,18 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) return true; /* otherwise, we have to actually compare the data */ - - BUG_ON(!IS_ALIGNED((unsigned long)val, len)); - switch (len) { case 1: - _val = *(u8 *)val; + _val = get_unaligned((u8 *)val); break; case 2: - _val = *(u16 *)val; + _val = get_unaligned((u16 *)val); break; case 4: - _val = *(u32 *)val; + _val = get_unaligned((u32 *)val); break; case 8: - _val = *(u64 *)val; + _val = get_unaligned((u64 *)val); break; default: return false;